use crate::{
blocks::block::BlockType,
common::MAX_BLOCK_SIZE,
encoding::{
CompressionLevel, Matcher,
block_header::BlockHeader,
blocks::{compress_block, compress_block_with_post_split},
frame_compressor::CompressState,
incompressible::{
block_looks_incompressible, block_looks_incompressible_strict,
compression_level_allows_raw_fast_path,
},
match_generator::MatchGeneratorDriver,
},
};
use alloc::vec::Vec;
pub(crate) enum BlockInput {
Staged(Vec<u8>),
InPlace(usize),
}
impl BlockInput {
fn len(&self) -> usize {
match self {
BlockInput::Staged(v) => v.len(),
BlockInput::InPlace(n) => *n,
}
}
fn bytes<'a, M: Matcher>(&'a self, matcher: &'a M) -> &'a [u8] {
match self {
BlockInput::Staged(v) => v,
BlockInput::InPlace(n) => &matcher.uncommitted_input()[..*n],
}
}
fn commit<M: Matcher>(self, matcher: &mut M) {
match self {
BlockInput::Staged(v) => matcher.commit_space(v),
BlockInput::InPlace(n) => matcher.commit_filled(n),
}
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
pub(crate) fn compress_block_encoded<M: Matcher>(
state: &mut CompressState<M>,
compression_level: CompressionLevel,
last_block: bool,
uncompressed_data: BlockInput,
output: &mut Vec<u8>,
dict_active: bool,
#[cfg(feature = "lsm")] block_decompressed_sizes: Option<&mut Vec<u32>>,
#[cfg(all(feature = "lsm", feature = "hash"))] block_checksums: Option<&mut Vec<u32>>,
) -> BlockType {
let block_size = uncompressed_data.len() as u32;
let bytes = uncompressed_data.bytes(&state.matcher);
let rle_byte_opt = bytes
.first()
.copied()
.filter(|f| bytes.iter().all(|x| x == f));
let window_size = state.matcher.window_size();
let dict_rejects_raw = dict_active;
let raw_skip_reachable = compression_level_allows_raw_fast_path(compression_level, window_size);
let looks_incompressible = rle_byte_opt.is_none()
&& !dict_rejects_raw
&& raw_skip_reachable
&& should_emit_raw_fast_path(compression_level, bytes);
let repeats_earlier_content = if looks_incompressible {
state
.seen_content
.record_and_report_repeat(bytes, window_size as usize)
} else {
if raw_skip_reachable && rle_byte_opt.is_none() {
state
.seen_content
.record_searched(bytes, window_size as usize);
} else if raw_skip_reachable {
state.seen_content.skip_recording(bytes.len());
}
false
};
let raw_fast_path = looks_incompressible && !repeats_earlier_content;
#[cfg(all(feature = "lsm", feature = "hash"))]
let post_split_path = rle_byte_opt.is_none()
&& !raw_fast_path
&& matches!(compression_level, CompressionLevel::Level(16..=22))
&& state.matcher.window_size() >= (1 << 17);
#[cfg(all(feature = "lsm", feature = "hash"))]
let precomputed_checksum = block_checksums
.as_ref()
.filter(|_| !post_split_path)
.map(|_| crate::encoding::frame_compressor::xxh64_block_low32(bytes));
if let Some(rle_byte) = rle_byte_opt {
#[cfg(feature = "lsm")]
if let Some(sink) = block_decompressed_sizes {
sink.push(block_size);
}
#[cfg(all(feature = "lsm", feature = "hash"))]
if let Some(sink) = block_checksums {
sink.push(precomputed_checksum.expect("checksum is hashed whenever a sink exists"));
}
uncompressed_data.commit(&mut state.matcher);
state.matcher.skip_matching_with_hint(Some(false));
let header = BlockHeader {
last_block,
block_type: BlockType::RLE,
block_size,
};
header.serialize(output);
output.push(rle_byte);
BlockType::RLE
} else if raw_fast_path {
#[cfg(feature = "lsm")]
if let Some(sink) = block_decompressed_sizes {
sink.push(block_size);
}
#[cfg(all(feature = "lsm", feature = "hash"))]
if let Some(sink) = block_checksums {
sink.push(precomputed_checksum.expect("checksum is hashed whenever a sink exists"));
}
uncompressed_data.commit(&mut state.matcher);
state.matcher.skip_matching_with_hint(Some(true));
let header = BlockHeader {
last_block,
block_type: BlockType::Raw,
block_size,
};
header.serialize(output);
output.extend_from_slice(state.matcher.get_last_space());
BlockType::Raw
} else {
uncompressed_data.commit(&mut state.matcher);
if matches!(compression_level, CompressionLevel::Level(16..=22))
&& state.matcher.window_size() >= (1 << 17)
{
#[cfg(all(feature = "lsm", feature = "hash"))]
compress_block_with_post_split(
state,
last_block,
output,
block_decompressed_sizes,
block_checksums,
);
#[cfg(all(feature = "lsm", not(feature = "hash")))]
compress_block_with_post_split(state, last_block, output, block_decompressed_sizes);
#[cfg(not(feature = "lsm"))]
compress_block_with_post_split(state, last_block, output);
return BlockType::Compressed;
}
#[cfg(feature = "lsm")]
if let Some(sink) = block_decompressed_sizes {
sink.push(block_size);
}
#[cfg(all(feature = "lsm", feature = "hash"))]
if let Some(sink) = block_checksums {
sink.push(precomputed_checksum.expect("checksum is hashed whenever a sink exists"));
}
let saved_offset_hist = state.offset_hist;
let mut saved_huff_table = core::mem::take(&mut state.block_scratch.huff_rollback);
let had_prior_huff_table = state.last_huff_table.is_some();
if let Some(prior) = state.last_huff_table.as_ref() {
match saved_huff_table.as_mut() {
Some(slot) => slot.clone_from(prior),
None => saved_huff_table = Some(prior.clone()),
}
}
let saved_ll_previous = state.fse_tables.ll_previous.clone();
let saved_ml_previous = state.fse_tables.ml_previous.clone();
let saved_of_previous = state.fse_tables.of_previous.clone();
let hdr_off = output.len();
output.extend_from_slice(&[0u8; 3]);
let payload_off = output.len();
compress_block(state, output);
let payload_len = output.len() - payload_off;
if payload_len >= MAX_BLOCK_SIZE as usize || payload_len >= block_size as usize {
output.truncate(hdr_off);
state.offset_hist = saved_offset_hist;
if had_prior_huff_table {
core::mem::swap(&mut state.last_huff_table, &mut saved_huff_table);
} else if let Some(built) = state.last_huff_table.take() {
saved_huff_table = Some(built);
}
if saved_huff_table.is_none() {
saved_huff_table = state.huff_table_spare.take();
}
state.fse_tables.roll_back_confirmation([
saved_ll_previous,
saved_ml_previous,
saved_of_previous,
]);
state.block_scratch.huff_rollback = saved_huff_table;
let header = BlockHeader {
last_block,
block_type: BlockType::Raw,
block_size,
};
header.serialize(output);
output.extend_from_slice(state.matcher.get_last_space());
BlockType::Raw
} else {
state.block_scratch.huff_rollback = saved_huff_table;
let header = BlockHeader {
last_block,
block_type: BlockType::Compressed,
block_size: payload_len as u32,
};
output[hdr_off..hdr_off + 3].copy_from_slice(&header.to_le_bytes());
BlockType::Compressed
}
}
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn compress_block_encoded_borrowed(
state: &mut CompressState<MatchGeneratorDriver>,
compression_level: CompressionLevel,
last_block: bool,
block: &[u8],
block_start: usize,
block_end: usize,
output: &mut Vec<u8>,
dict_active: bool,
#[cfg(feature = "lsm")] block_decompressed_sizes: Option<&mut Vec<u32>>,
#[cfg(all(feature = "lsm", feature = "hash"))] block_checksums: Option<&mut Vec<u32>>,
) -> BlockType {
debug_assert!(
state.matcher.borrowed_supported(),
"borrowed one-shot path reached for an unsupported backend/search config",
);
let block_size = block.len() as u32;
let is_rle = !block.is_empty() && block.iter().all(|x| block[0].eq(x));
let window_size = state.matcher.window_size();
let dict_rejects_raw = dict_active;
let raw_skip_reachable = compression_level_allows_raw_fast_path(compression_level, window_size);
let looks_incompressible = !is_rle
&& !dict_rejects_raw
&& raw_skip_reachable
&& should_emit_raw_fast_path(compression_level, block);
let repeats_earlier_content = if looks_incompressible {
state
.seen_content
.record_and_report_repeat(block, window_size as usize)
} else {
if raw_skip_reachable && !is_rle {
state
.seen_content
.record_searched(block, window_size as usize);
} else if raw_skip_reachable {
state.seen_content.skip_recording(block.len());
}
false
};
if is_rle {
let rle_byte = block[0];
#[cfg(feature = "lsm")]
if let Some(sink) = block_decompressed_sizes {
sink.push(block_size);
}
#[cfg(all(feature = "lsm", feature = "hash"))]
if let Some(sink) = block_checksums {
sink.push(crate::encoding::frame_compressor::xxh64_block_low32(block));
}
state.matcher.set_borrowed_block(block_start, block_end);
state.matcher.skip_matching_with_hint(Some(false));
let header = BlockHeader {
last_block,
block_type: BlockType::RLE,
block_size,
};
header.serialize(output);
output.push(rle_byte);
BlockType::RLE
} else if looks_incompressible && !repeats_earlier_content {
#[cfg(feature = "lsm")]
if let Some(sink) = block_decompressed_sizes {
sink.push(block_size);
}
#[cfg(all(feature = "lsm", feature = "hash"))]
if let Some(sink) = block_checksums {
sink.push(crate::encoding::frame_compressor::xxh64_block_low32(block));
}
state.matcher.set_borrowed_block(block_start, block_end);
state.matcher.skip_matching_with_hint(Some(true));
let header = BlockHeader {
last_block,
block_type: BlockType::Raw,
block_size,
};
header.serialize(output);
output.extend_from_slice(block);
BlockType::Raw
} else {
state.matcher.set_borrowed_block(block_start, block_end);
#[cfg(feature = "lsm")]
if let Some(sink) = block_decompressed_sizes {
sink.push(block_size);
}
#[cfg(all(feature = "lsm", feature = "hash"))]
if let Some(sink) = block_checksums {
sink.push(crate::encoding::frame_compressor::xxh64_block_low32(block));
}
let saved_offset_hist = state.offset_hist;
let mut saved_huff_table = core::mem::take(&mut state.block_scratch.huff_rollback);
let had_prior_huff_table = state.last_huff_table.is_some();
if let Some(prior) = state.last_huff_table.as_ref() {
match saved_huff_table.as_mut() {
Some(slot) => slot.clone_from(prior),
None => saved_huff_table = Some(prior.clone()),
}
}
let saved_ll_previous = state.fse_tables.ll_previous.clone();
let saved_ml_previous = state.fse_tables.ml_previous.clone();
let saved_of_previous = state.fse_tables.of_previous.clone();
let hdr_off = output.len();
output.extend_from_slice(&[0u8; 3]);
let payload_off = output.len();
compress_block(state, output);
let payload_len = output.len() - payload_off;
if payload_len >= MAX_BLOCK_SIZE as usize || payload_len >= block_size as usize {
output.truncate(hdr_off);
state.offset_hist = saved_offset_hist;
if had_prior_huff_table {
core::mem::swap(&mut state.last_huff_table, &mut saved_huff_table);
} else if let Some(built) = state.last_huff_table.take() {
saved_huff_table = Some(built);
}
if saved_huff_table.is_none() {
saved_huff_table = state.huff_table_spare.take();
}
state.fse_tables.roll_back_confirmation([
saved_ll_previous,
saved_ml_previous,
saved_of_previous,
]);
state.block_scratch.huff_rollback = saved_huff_table;
let header = BlockHeader {
last_block,
block_type: BlockType::Raw,
block_size,
};
header.serialize(output);
output.extend_from_slice(block);
BlockType::Raw
} else {
state.block_scratch.huff_rollback = saved_huff_table;
let header = BlockHeader {
last_block,
block_type: BlockType::Compressed,
block_size: payload_len as u32,
};
output[hdr_off..hdr_off + 3].copy_from_slice(&header.to_le_bytes());
BlockType::Compressed
}
}
}
#[inline]
fn should_emit_raw_fast_path(level: CompressionLevel, block: &[u8]) -> bool {
if matches!(level, CompressionLevel::Best) {
return block_looks_incompressible_strict(block);
}
block_looks_incompressible(block)
}
#[cfg(test)]
mod tests;