use core::arch::wasm32::{
u32x4_shl, u32x4_shr, u32x4_splat, v128, v128_and, v128_load, v128_or, v128_store,
};
const BLOCK_LEN: usize = 128;
const LANES: usize = 4;
const ROWS: usize = BLOCK_LEN / LANES;
macro_rules! decode_bitwidth {
($name:ident, $bits:literal) => {
#[doc = concat!(
"Decodes `BLOCK_LEN` (128) values packed at ",
stringify!($bits),
" bit(s) each, matching [`Scalar`](crate::Scalar)'s 4-way interleaved ",
"layout exactly. See the [module-level algorithm docs](self) for how."
)]
#[allow(clippy::cast_possible_truncation, clippy::cast_ptr_alignment)]
pub(crate) unsafe fn $name(compressed: &[u8], decompressed: &mut [u32]) {
const NUM_BITS: usize = $bits;
const MASK: u32 = ((1u64 << NUM_BITS) - 1) as u32;
let mask_v = u32x4_splat(MASK);
let mut in_register = unsafe { v128_load(compressed.as_ptr().cast::<v128>()) };
let mut next_word = 1usize;
unsafe {
v128_store(
decompressed[0..4].as_mut_ptr().cast::<v128>(),
v128_and(in_register, mask_v),
);
}
for row in 1..ROWS {
let inner_cursor = (row * NUM_BITS) % 32;
let inner_capacity = 32 - inner_cursor;
let shifted = if inner_cursor == 0 {
in_register
} else {
u32x4_shr(in_register, inner_cursor as u32)
};
let mut out = v128_and(shifted, mask_v);
if inner_capacity <= NUM_BITS && row != ROWS - 1 {
in_register =
unsafe { v128_load(compressed[16 * next_word..].as_ptr().cast::<v128>()) };
next_word += 1;
if inner_capacity < NUM_BITS {
let filled =
v128_and(u32x4_shl(in_register, inner_capacity as u32), mask_v);
out = v128_or(out, filled);
}
}
unsafe {
v128_store(
decompressed[4 * row..4 * row + 4]
.as_mut_ptr()
.cast::<v128>(),
out,
);
}
}
}
};
}
decode_bitwidth!(decode_1bit, 1);
decode_bitwidth!(decode_2bit, 2);
decode_bitwidth!(decode_3bit, 3);
decode_bitwidth!(decode_4bit, 4);
decode_bitwidth!(decode_5bit, 5);
decode_bitwidth!(decode_6bit, 6);
decode_bitwidth!(decode_7bit, 7);
decode_bitwidth!(decode_8bit, 8);
decode_bitwidth!(decode_9bit, 9);
decode_bitwidth!(decode_10bit, 10);
decode_bitwidth!(decode_11bit, 11);
decode_bitwidth!(decode_12bit, 12);
decode_bitwidth!(decode_13bit, 13);
decode_bitwidth!(decode_14bit, 14);
decode_bitwidth!(decode_15bit, 15);
decode_bitwidth!(decode_16bit, 16);
decode_bitwidth!(decode_17bit, 17);
decode_bitwidth!(decode_18bit, 18);
decode_bitwidth!(decode_19bit, 19);
decode_bitwidth!(decode_20bit, 20);
decode_bitwidth!(decode_21bit, 21);
decode_bitwidth!(decode_22bit, 22);
decode_bitwidth!(decode_23bit, 23);
decode_bitwidth!(decode_24bit, 24);
decode_bitwidth!(decode_25bit, 25);
decode_bitwidth!(decode_26bit, 26);
decode_bitwidth!(decode_27bit, 27);
decode_bitwidth!(decode_28bit, 28);
decode_bitwidth!(decode_29bit, 29);
decode_bitwidth!(decode_30bit, 30);
decode_bitwidth!(decode_31bit, 31);
#[allow(clippy::cast_ptr_alignment)]
pub(crate) unsafe fn decode_32bit(compressed: &[u8], decompressed: &mut [u32]) {
for row in 0..ROWS {
let v = unsafe { v128_load(compressed[16 * row..].as_ptr().cast::<v128>()) };
unsafe {
v128_store(
decompressed[4 * row..4 * row + 4]
.as_mut_ptr()
.cast::<v128>(),
v,
);
}
}
}
pub(crate) unsafe fn decode(num_bits: u8, compressed: &[u8], decompressed: &mut [u32]) {
unsafe {
match num_bits {
1 => decode_1bit(compressed, decompressed),
2 => decode_2bit(compressed, decompressed),
3 => decode_3bit(compressed, decompressed),
4 => decode_4bit(compressed, decompressed),
5 => decode_5bit(compressed, decompressed),
6 => decode_6bit(compressed, decompressed),
7 => decode_7bit(compressed, decompressed),
8 => decode_8bit(compressed, decompressed),
9 => decode_9bit(compressed, decompressed),
10 => decode_10bit(compressed, decompressed),
11 => decode_11bit(compressed, decompressed),
12 => decode_12bit(compressed, decompressed),
13 => decode_13bit(compressed, decompressed),
14 => decode_14bit(compressed, decompressed),
15 => decode_15bit(compressed, decompressed),
16 => decode_16bit(compressed, decompressed),
17 => decode_17bit(compressed, decompressed),
18 => decode_18bit(compressed, decompressed),
19 => decode_19bit(compressed, decompressed),
20 => decode_20bit(compressed, decompressed),
21 => decode_21bit(compressed, decompressed),
22 => decode_22bit(compressed, decompressed),
23 => decode_23bit(compressed, decompressed),
24 => decode_24bit(compressed, decompressed),
25 => decode_25bit(compressed, decompressed),
26 => decode_26bit(compressed, decompressed),
27 => decode_27bit(compressed, decompressed),
28 => decode_28bit(compressed, decompressed),
29 => decode_29bit(compressed, decompressed),
30 => decode_30bit(compressed, decompressed),
31 => decode_31bit(compressed, decompressed),
32 => decode_32bit(compressed, decompressed),
_ => unreachable!("num_bits must be in 1..=32, checked by the caller"),
}
}
}