#![allow(clippy::similar_names, reason = "XXX")]
#[cfg(target_arch = "wasm32")]
use core::arch::wasm32::*;
use core::mem::MaybeUninit;
use core::{ptr, slice};
use crate::backend::generic::{decode_generic_unchecked, encode_generic_unchecked};
use crate::error::InvalidInput;
use crate::util::lut16;
#[inline]
unsafe fn interleave(a: v128, b: v128) -> (v128, v128) {
(
i8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(a, b),
i8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(a, b),
)
}
pub(crate) unsafe fn encode_simd128_unchecked<const UPPER: bool>(
src: &[u8],
dst: &mut [[MaybeUninit<u8>; 2]],
) {
#[allow(clippy::identity_op, reason = "XXX")]
const BATCH_ELEMS_V128_X1: usize = size_of::<v128>() * 1;
debug_assert!(src.len() <= dst.len());
if src.len() >= BATCH_ELEMS_V128_X1 {
let mask = u8x16_splat(0b_0000_1111);
let lut = v128_load(lut16::<UPPER>().as_ptr().cast::<v128>());
let encode_v128 = |src: *const v128, dst: *mut v128| {
let chunk = v128_load(src);
let hi = i8x16_shr(chunk, 4);
let lo = v128_and(chunk, mask);
let (a, b) = interleave(hi, lo);
let a = i8x16_swizzle(lut, a);
let b = i8x16_swizzle(lut, b);
v128_store(dst.add(0), a);
v128_store(dst.add(1), b);
};
let batches = src.len() / BATCH_ELEMS_V128_X1;
let remainder = src.len() % BATCH_ELEMS_V128_X1;
for i in 0..batches {
encode_v128(
src.as_ptr().cast::<v128>().add(i),
dst.as_mut_ptr().cast::<v128>().add(i * 2),
);
}
encode_generic_unchecked::<UPPER>(
slice::from_raw_parts(src.as_ptr().add(batches * BATCH_ELEMS_V128_X1), remainder),
slice::from_raw_parts_mut(
dst.as_mut_ptr().add(batches * BATCH_ELEMS_V128_X1),
remainder,
),
);
} else {
encode_generic_unchecked::<UPPER>(src, dst);
}
}
#[allow(clippy::cast_possible_wrap, reason = "XXX")]
pub(crate) unsafe fn decode_simd128_unchecked(
src: *const [[u8; 2]],
dst: *mut [MaybeUninit<u8>],
) -> Result<(), InvalidInput> {
const BATCH_ELEMS_V128_X2: usize = size_of::<v128>() / 2 * 2;
debug_assert!(src.len() <= dst.len());
if src.len() >= BATCH_ELEMS_V128_X2 {
let n_c6 = i8x16_splat((0xFF_u8 - b'9').cast_signed());
let n_06 = i8x16_splat(0x06);
let n_f0 = i8x16_splat(0xF0_u8.cast_signed());
let n_df = i8x16_splat(0xDF_u8.cast_signed());
let u_a = i8x16_splat(b'A'.cast_signed());
let n_0a = i8x16_splat(0x0A);
let trick = i8x16_splat(127 - 15);
let batches = src.len() / BATCH_ELEMS_V128_X2;
let remainder = src.len() % BATCH_ELEMS_V128_X2;
let mut invalid = 0;
let decode_chunk = |chunk: v128| -> v128 {
let d = i8x16_sub(u8x16_sub_sat(i8x16_add(chunk, n_c6), n_06), n_f0);
let a = u8x16_add_sat(i8x16_sub(v128_and(chunk, n_df), u_a), n_0a);
u8x16_min(d, a)
};
let mut decode_v128x2 = |src: *const v128, dst: *mut v128| {
let chunk0 = v128_load(src.cast::<v128>().add(0));
let chunk1 = v128_load(src.cast::<v128>().add(1));
let n0 = decode_chunk(chunk0);
let n1 = decode_chunk(chunk1);
invalid |= i8x16_bitmask(v128_or(u8x16_add_sat(n0, trick), u8x16_add_sat(n1, trick)));
let out = {
let hi01 = i8x16_shuffle::<0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30>(
n0, n1,
);
let lo01 = i8x16_shuffle::<1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31>(
n0, n1,
);
v128_or(i8x16_shl(hi01, 4), lo01)
};
v128_store(dst, out);
};
for i in 0..batches {
decode_v128x2(src.cast::<v128>().add(i * 2), dst.cast::<v128>().add(i));
}
if invalid != 0 {
return Err(InvalidInput);
}
decode_generic_unchecked::<false>(
ptr::slice_from_raw_parts(
src.cast::<[u8; 2]>().add(batches * BATCH_ELEMS_V128_X2),
remainder,
),
ptr::slice_from_raw_parts_mut(
dst.cast::<MaybeUninit<u8>>()
.add(batches * BATCH_ELEMS_V128_X2),
remainder,
),
)
} else {
decode_generic_unchecked::<false>(src, dst)
}
}
#[cfg(test)]
mod smoking {
use super::*;
use crate::backend::tests::{
check_decode_validation_any_backend, check_encode_decode_any_backend,
};
#[test]
#[cfg_attr(not(target_feature = "simd128"), ignore)]
fn test_encode_decode_simd128() {
check_encode_decode_any_backend::<true>(
encode_simd128_unchecked::<true>,
decode_generic_unchecked::<false>,
);
check_encode_decode_any_backend::<false>(
encode_simd128_unchecked::<false>,
decode_generic_unchecked::<false>,
);
check_encode_decode_any_backend::<true>(
encode_simd128_unchecked::<true>,
decode_simd128_unchecked,
);
check_encode_decode_any_backend::<false>(
encode_simd128_unchecked::<false>,
decode_simd128_unchecked,
);
}
#[test]
#[cfg_attr(not(target_feature = "simd128"), ignore)]
fn test_decode_validation_simd128() {
check_decode_validation_any_backend(decode_simd128_unchecked);
}
}