macro_rules! dispatch_decoder {
($params:expr, $base:expr, $F:ty, $decoder:ident => $body:expr) => {{
let (exp_factor, fac_int, frac_flt) = $params.factors::<$F>();
if $params.use_div {
let $decoder = $crate::bitpack::AlpDivDecoder {
base: $base,
exp_factor,
};
$body
} else if fac_int == 1 {
let $decoder = $crate::bitpack::AlpFac1Decoder {
base: $base,
frac_flt,
};
$body
} else {
let $decoder = $crate::bitpack::AlpMulDecoder {
base: $base,
fac_int,
frac_flt,
};
$body
}
}};
}
mod delta;
mod standard;
use core::{
marker::PhantomData,
mem::{MaybeUninit, size_of},
ptr::copy_nonoverlapping,
slice::from_raw_parts_mut,
};
use delta::decode_delta_raw;
use standard::decode_standard_raw;
use crate::{
bitpack::{
AlpDictDecoder, AlpRdConstantDecoder, bitunpack_core_generic, bitunpack_u64_raw,
bitunpack_u64_slice, packed_byte_size,
},
constants::{EXC_COUNT_LEN, EXC_COUNT_LEN_U32, MAX_DICT_ENTRIES},
error::{Error, Result},
float::AlpFloat,
header::{ParsedHeader, read_count, read_header},
params::AlpParams,
};
#[inline(always)]
fn count_bitmap_ones(bitmap: &[u8], count: usize) -> usize {
let full_bytes = count / 8;
let full_words = full_bytes / 8;
let mut ones = 0usize;
let ptr = bitmap.as_ptr().cast::<u64>();
for i in 0..full_words {
let word = unsafe { ptr.add(i).read_unaligned() };
ones += word.count_ones() as usize;
}
for &b in &bitmap[full_words * 8..full_bytes] {
ones += b.count_ones() as usize;
}
let rem_bits = count % 8;
if rem_bits > 0 {
let mask = (1u8 << rem_bits) - 1;
ones += (bitmap[full_bytes] & mask).count_ones() as usize;
}
ones
}
#[inline(always)]
unsafe fn expand_byte<F: AlpFloat>(
byte: u8,
out_pos: usize,
src_idx: &mut usize,
prev: &mut F,
non_repeats: *const F,
dst_ptr: *mut F,
) {
unsafe {
if byte == 0x00 {
copy_nonoverlapping(non_repeats.add(*src_idx + 1), dst_ptr.add(out_pos), 8);
*src_idx += 8;
*prev = *dst_ptr.add(out_pos + 7);
} else if byte == 0xFF {
let p = *prev;
for k in 0..8 {
dst_ptr.add(out_pos + k).write(p);
}
} else {
for k in 0..8 {
if (byte & (1 << k)) == 0 {
*src_idx += 1;
*prev = *non_repeats.add(*src_idx);
}
*dst_ptr.add(out_pos + k) = *prev;
}
}
}
}
#[inline]
pub(crate) unsafe fn expand_repeats<F: AlpFloat>(
bitmap: &[u8],
count: usize,
non_repeats: *const F,
dst_ptr: *mut F,
) {
if count == 0 {
return;
}
unsafe {
let mut src_idx = 0usize;
let mut prev = *non_repeats;
*dst_ptr = prev;
let full_words = count / 64;
let ptr_u64 = bitmap.as_ptr().cast::<u64>();
for w in 0..full_words {
let word = u64::from_le(ptr_u64.add(w).read_unaligned());
let base_out = w * 64;
if w == 0 {
for k in 1..8 {
if (bitmap[0] & (1 << k)) == 0 {
src_idx += 1;
prev = *non_repeats.add(src_idx);
}
*dst_ptr.add(k) = prev;
}
#[allow(clippy::needless_range_loop)]
for b in 1..8 {
expand_byte(
bitmap[b],
b * 8,
&mut src_idx,
&mut prev,
non_repeats,
dst_ptr,
);
}
} else if word == 0 {
copy_nonoverlapping(non_repeats.add(src_idx + 1), dst_ptr.add(base_out), 64);
src_idx += 64;
prev = *dst_ptr.add(base_out + 63);
} else if word == u64::MAX {
for k in 0..64 {
dst_ptr.add(base_out + k).write(prev);
}
} else {
let bytes_ptr = bitmap.as_ptr().add(base_out / 8);
for b in 0..8 {
expand_byte(
*bytes_ptr.add(b),
base_out + b * 8,
&mut src_idx,
&mut prev,
non_repeats,
dst_ptr,
);
}
}
}
let rem_start = full_words * 64;
let start_j = if full_words == 0 { 1 } else { 0 };
for i in (rem_start + start_j)..count {
let byte_idx = i / 8;
let bit_idx = i % 8;
if (bitmap[byte_idx] & (1 << bit_idx)) != 0 {
*dst_ptr.add(i) = prev;
} else {
src_idx += 1;
prev = *non_repeats.add(src_idx);
*dst_ptr.add(i) = prev;
}
}
}
}
#[inline(always)]
unsafe fn decode_dict_raw<F: AlpFloat>(
payload: &[u8],
count: usize,
dst_ptr: *mut F,
) -> Result<()> {
if payload.len() < 2 {
return Err(Error::UnexpectedEof {
needed: 2,
available: payload.len(),
});
}
let dict_len = payload[0] as usize;
let bit_width = payload[1];
if dict_len == 0 || dict_len > MAX_DICT_ENTRIES || bit_width > 6 {
return Err(Error::InvalidHeader);
}
let elem_size = size_of::<F>();
let dict_bytes = dict_len * elem_size;
if payload.len() < 2 + dict_bytes {
return Err(Error::UnexpectedEof {
needed: 2 + dict_bytes,
available: payload.len(),
});
}
let mut dict = [F::ZERO; MAX_DICT_ENTRIES];
let dict_slice = &payload[2..2 + dict_bytes];
for (entry, chunk) in dict.iter_mut().zip(dict_slice.chunks_exact(elem_size)) {
*entry = F::read_raw(chunk);
}
if dict_len > 0 {
let pad = dict[0];
for entry in &mut dict[dict_len..MAX_DICT_ENTRIES] {
*entry = pad;
}
}
if bit_width == 0 {
let single_val = dict[0];
unsafe {
for i in 0..count {
dst_ptr.add(i).write(single_val);
}
}
return Ok(());
}
let indices_offset = 2 + dict_bytes;
let packed_bytes = packed_byte_size(count, bit_width);
if payload.len() < indices_offset + packed_bytes {
return Err(Error::UnexpectedEof {
needed: indices_offset + packed_bytes,
available: payload.len(),
});
}
let decoder = AlpDictDecoder { dict: &dict };
unsafe {
bitunpack_core_generic(
&payload[indices_offset..indices_offset + packed_bytes],
count,
bit_width,
decoder,
dst_ptr,
);
}
Ok(())
}
#[inline(always)]
unsafe fn decode_rd_raw<F: AlpFloat>(payload: &[u8], count: usize, dst_ptr: *mut F) -> Result<()> {
if payload.len() < 5 {
return Err(Error::UnexpectedEof {
needed: 5,
available: payload.len(),
});
}
let right_bw = payload[0];
let left_bw = payload[1];
let actual_dict_size = payload[2] as usize;
if actual_dict_size > 8 || right_bw == 0 || right_bw >= F::RD_TOTAL_BITS || left_bw > 3 {
return Err(Error::InvalidHeader);
}
let exc_count = u16::from_le_bytes([payload[3], payload[4]]) as usize;
let dict_bytes = actual_dict_size * 2;
let mut cursor = 5;
if payload.len() < cursor + dict_bytes {
return Err(Error::UnexpectedEof {
needed: cursor + dict_bytes,
available: payload.len(),
});
}
let mut dict = [0u16; 8];
unsafe {
let dict_ptr = payload.as_ptr().add(cursor).cast::<u16>();
for (i, entry) in dict.iter_mut().take(actual_dict_size).enumerate() {
*entry = u16::from_le(dict_ptr.add(i).read_unaligned());
}
}
cursor += dict_bytes;
let left_bytes = if left_bw > 0 {
packed_byte_size(count, left_bw)
} else {
0
};
let right_bytes = packed_byte_size(count, right_bw);
let exc_bytes = exc_count * 4;
if payload.len() < cursor + left_bytes + right_bytes + exc_bytes {
return Err(Error::UnexpectedEof {
needed: cursor + left_bytes + right_bytes + exc_bytes,
available: payload.len(),
});
}
let shift = right_bw as u64;
let mut shifted_dict = [0u64; 8];
for (i, &entry) in dict.iter().take(actual_dict_size).enumerate() {
shifted_dict[i] = (entry as u64) << shift;
}
let right_cursor = cursor + left_bytes;
let exc_cursor = right_cursor + right_bytes;
if left_bw == 0 {
let decoder = AlpRdConstantDecoder {
high_bits: shifted_dict[0],
_phantom: PhantomData,
};
unsafe {
bitunpack_core_generic(
&payload[right_cursor..right_cursor + right_bytes],
count,
right_bw,
decoder,
dst_ptr,
);
}
} else {
let mut block_offset = 0;
let mut cur_left_cursor = cursor;
let mut cur_right_cursor = right_cursor;
let mut left_buf = [0u64; 1024];
let mut right_buf = [0u64; 1024];
while block_offset < count {
let cur_count = (count - block_offset).min(1024);
let cur_left_bytes = packed_byte_size(cur_count, left_bw);
let cur_right_bytes = packed_byte_size(cur_count, right_bw);
bitunpack_u64_slice(
&payload[cur_left_cursor..cur_left_cursor + cur_left_bytes],
cur_count,
left_bw,
&mut left_buf[..cur_count],
)?;
cur_left_cursor += cur_left_bytes;
if size_of::<F>() == 8 {
let dst_u64_ptr = unsafe { dst_ptr.add(block_offset).cast::<u64>() };
unsafe {
bitunpack_u64_raw(
&payload[cur_right_cursor..cur_right_cursor + cur_right_bytes],
cur_count,
right_bw,
dst_u64_ptr,
)?;
}
cur_right_cursor += cur_right_bytes;
let dst_u64 = unsafe { from_raw_parts_mut(dst_u64_ptr, cur_count) };
let (dst_chunks, dst_rem) = dst_u64.as_chunks_mut::<8>();
let (left_chunks, left_rem) = left_buf[..cur_count].as_chunks::<8>();
for (dc, lc) in dst_chunks.iter_mut().zip(left_chunks.iter()) {
unroll_8!(k => {
dc[k] |= shifted_dict[lc[k] as usize & 7];
});
}
for (d, l) in dst_rem.iter_mut().zip(left_rem.iter()) {
*d |= shifted_dict[*l as usize & 7];
}
} else {
bitunpack_u64_slice(
&payload[cur_right_cursor..cur_right_cursor + cur_right_bytes],
cur_count,
right_bw,
&mut right_buf[..cur_count],
)?;
cur_right_cursor += cur_right_bytes;
unsafe {
for i in 0..cur_count {
*dst_ptr.add(block_offset + i) =
F::from_u64_raw(shifted_dict[left_buf[i] as usize & 7] | right_buf[i]);
}
}
}
block_offset += cur_count;
}
}
unsafe {
let exc_ptr = payload.as_ptr().add(exc_cursor);
for i in 0..exc_count {
let pos = u16::from_le(exc_ptr.add(i * 4).cast::<u16>().read_unaligned()) as usize;
let left_val = u16::from_le(exc_ptr.add(i * 4 + 2).cast::<u16>().read_unaligned()) as u64;
if pos >= count {
return Err(Error::CorruptedData { index: pos, count });
}
let cur = (*dst_ptr.add(pos)).to_u64_key();
let right = cur & ((1u64 << right_bw) - 1);
let raw = (left_val << shift) | right;
*dst_ptr.add(pos) = F::from_u64_raw(raw);
}
}
Ok(())
}
#[inline(always)]
unsafe fn decompress_into_raw_direct<F: AlpFloat>(
src: &[u8],
cursor: usize,
type_byte: u8,
count: usize,
params: Option<AlpParams>,
dst_ptr: *mut F,
) -> Result<()> {
if type_byte == F::TYPE_RAW_BYTE {
let raw_bytes_needed = count
.checked_mul(size_of::<F>())
.ok_or(Error::InvalidHeader)?;
if src.len() < cursor + raw_bytes_needed {
return Err(Error::UnexpectedEof {
needed: cursor + raw_bytes_needed,
available: src.len(),
});
}
unsafe {
copy_nonoverlapping(
src.as_ptr().add(cursor),
dst_ptr.cast::<u8>(),
raw_bytes_needed,
);
}
return Ok(());
}
if type_byte == F::TYPE_DICT_BYTE {
unsafe {
decode_dict_raw::<F>(&src[cursor..], count, dst_ptr)?;
}
return Ok(());
}
if type_byte == F::TYPE_RD_BYTE {
unsafe {
decode_rd_raw::<F>(&src[cursor..], count, dst_ptr)?;
}
return Ok(());
}
let is_delta = type_byte == F::TYPE_DELTA_BYTE || type_byte == F::TYPE_DEC_DELTA_BYTE;
let is_standard = type_byte == F::TYPE_BYTE || type_byte == F::TYPE_DEC_BYTE;
if !is_standard && !is_delta {
return Err(Error::InvalidHeader);
}
let alp_params = match params {
Some(p) => p,
None => return Err(Error::InvalidHeader),
};
if !alp_params.validate::<F>() {
return Err(Error::UnsupportedParams {
exp: alp_params.exp,
fac: alp_params.fac,
bit_width: alp_params.bit_width,
});
}
let payload = &src[cursor..];
if is_delta {
unsafe {
decode_delta_raw::<F>(payload, count, alp_params, dst_ptr)?;
}
} else {
unsafe {
decode_standard_raw::<F>(payload, count, alp_params, dst_ptr)?;
}
}
Ok(())
}
pub unsafe fn decompress_into_raw<F: AlpFloat>(
src: &[u8],
dst_ptr: *mut F,
dst_cap: usize,
) -> Result<usize> {
let ParsedHeader {
type_byte,
count,
params,
mut cursor,
has_repeat,
..
} = read_header(src)?;
if count == 0 {
return Ok(0);
}
if dst_cap < count {
return Err(Error::BufferTooSmall {
needed: count,
available: dst_cap,
});
}
if !has_repeat {
unsafe {
decompress_into_raw_direct::<F>(src, cursor, type_byte, count, params, dst_ptr)?;
}
return Ok(count);
}
let bitmap_len = count.div_ceil(8);
if src.len() < cursor + bitmap_len {
return Err(Error::UnexpectedEof {
needed: cursor + bitmap_len,
available: src.len(),
});
}
let bitmap = &src[cursor..cursor + bitmap_len];
cursor += bitmap_len;
let repeats = count_bitmap_ones(bitmap, count);
let non_repeat_count = count - repeats;
if non_repeat_count == 0 {
return Err(Error::InvalidHeader);
}
let mut stack_buf = MaybeUninit::<[F; 1024]>::uninit();
let mut heap_buf: Vec<F> = Vec::new();
let tmp_ptr: *mut F = if non_repeat_count <= 1024 {
stack_buf.as_mut_ptr().cast::<F>()
} else {
heap_buf.reserve(non_repeat_count);
heap_buf.as_mut_ptr()
};
unsafe {
decompress_into_raw_direct::<F>(src, cursor, type_byte, non_repeat_count, params, tmp_ptr)?;
expand_repeats::<F>(bitmap, count, tmp_ptr, dst_ptr);
}
Ok(count)
}
#[inline(always)]
pub fn decompress_into_slice<F: AlpFloat>(src: &[u8], dst: &mut [F]) -> Result<usize> {
unsafe { decompress_into_raw(src, dst.as_mut_ptr(), dst.len()) }
}
pub fn decompress_into<F: AlpFloat>(src: &[u8], dst: &mut Vec<F>) -> Result<()> {
let count = read_count(src)?;
if count == 0 {
return Ok(());
}
let old_len = dst.len();
dst.reserve(count);
unsafe {
let written = decompress_into_raw(src, dst.as_mut_ptr().add(old_len), count)?;
dst.set_len(old_len + written);
}
Ok(())
}
#[inline]
pub fn decompress<F: AlpFloat>(src: &[u8]) -> Result<Vec<F>> {
let mut dst = Vec::new();
decompress_into(src, &mut dst)?;
Ok(dst)
}
#[inline]
pub(crate) unsafe fn patch_exceptions<F: AlpFloat>(
src: &[u8],
count: usize,
dst_ptr: *mut F,
) -> Result<()> {
if src.is_empty() {
return Ok(());
}
let mut cursor = 0;
let is_large = count > u16::MAX as usize;
let (exc_count, exc_count_len) = if is_large {
if src.len() < cursor + EXC_COUNT_LEN_U32 {
return Err(Error::UnexpectedEof {
needed: cursor + EXC_COUNT_LEN_U32,
available: src.len(),
});
}
let c = u32::from_le_bytes(
src[cursor..cursor + 4]
.try_into()
.map_err(|_| Error::InvalidHeader)?,
) as usize;
(c, EXC_COUNT_LEN_U32)
} else {
if src.len() < cursor + EXC_COUNT_LEN {
return Err(Error::UnexpectedEof {
needed: cursor + EXC_COUNT_LEN,
available: src.len(),
});
}
let c = u16::from_le_bytes([src[cursor], src[cursor + 1]]) as usize;
(c, EXC_COUNT_LEN)
};
cursor += exc_count_len;
let entry_size = if is_large {
F::EXC_ENTRY_SIZE_U32
} else {
F::EXC_ENTRY_SIZE
};
let exc_bytes_needed = exc_count
.checked_mul(entry_size)
.ok_or(Error::InvalidHeader)?;
if src.len() < cursor + exc_bytes_needed {
return Err(Error::UnexpectedEof {
needed: cursor + exc_bytes_needed,
available: src.len(),
});
}
let exc_slice = &src[cursor..cursor + exc_bytes_needed];
if is_large {
for chunk in exc_slice.chunks_exact(entry_size) {
let (pos, val) = F::read_exception_u32(chunk);
if pos >= count {
return Err(Error::CorruptedData { index: pos, count });
}
unsafe {
*dst_ptr.add(pos) = val;
}
}
} else {
for chunk in exc_slice.chunks_exact(entry_size) {
let (pos, val) = F::read_exception(chunk);
if pos >= count {
return Err(Error::CorruptedData { index: pos, count });
}
unsafe {
*dst_ptr.add(pos) = val;
}
}
}
Ok(())
}