#![allow(dead_code)]
use super::*;
const HOST_NATIVE_BE: bool = cfg!(target_endian = "big");
#[inline(always)]
unsafe fn byteswap32_if_be<const BE: bool>(v: __m128i) -> __m128i {
if BE != HOST_NATIVE_BE {
const MASK: __m128i =
unsafe { core::mem::transmute([3u8, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12]) };
unsafe { _mm_shuffle_epi8(v, MASK) }
} else {
v
}
}
#[inline(always)]
unsafe fn deinterleave_rgb96_4px(
v0: __m128i,
v1: __m128i,
v2: __m128i,
) -> (__m128i, __m128i, __m128i) {
unsafe {
let r_v0 = _mm_setr_epi8(0, 1, 2, 3, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1);
let r_v1 = _mm_setr_epi8(-1, -1, -1, -1, -1, -1, -1, -1, 8, 9, 10, 11, -1, -1, -1, -1);
let r_v2 = _mm_setr_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 4, 5, 6, 7);
let r = _mm_or_si128(
_mm_or_si128(_mm_shuffle_epi8(v0, r_v0), _mm_shuffle_epi8(v1, r_v1)),
_mm_shuffle_epi8(v2, r_v2),
);
let g_v0 = _mm_setr_epi8(4, 5, 6, 7, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1);
let g_v1 = _mm_setr_epi8(-1, -1, -1, -1, 0, 1, 2, 3, 12, 13, 14, 15, -1, -1, -1, -1);
let g_v2 = _mm_setr_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 8, 9, 10, 11);
let g = _mm_or_si128(
_mm_or_si128(_mm_shuffle_epi8(v0, g_v0), _mm_shuffle_epi8(v1, g_v1)),
_mm_shuffle_epi8(v2, g_v2),
);
let b_v0 = _mm_setr_epi8(8, 9, 10, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1);
let b_v1 = _mm_setr_epi8(-1, -1, -1, -1, 4, 5, 6, 7, -1, -1, -1, -1, -1, -1, -1, -1);
let b_v2 = _mm_setr_epi8(-1, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 12, 13, 14, 15);
let b = _mm_or_si128(
_mm_or_si128(_mm_shuffle_epi8(v0, b_v0), _mm_shuffle_epi8(v1, b_v1)),
_mm_shuffle_epi8(v2, b_v2),
);
(r, g, b)
}
}
#[inline(always)]
unsafe fn load_deint_rgb96_4px<const BE: bool>(ptr: *const u32) -> (__m128i, __m128i, __m128i) {
unsafe {
let v0 = byteswap32_if_be::<BE>(_mm_loadu_si128(ptr.cast()));
let v1 = byteswap32_if_be::<BE>(_mm_loadu_si128(ptr.add(4).cast()));
let v2 = byteswap32_if_be::<BE>(_mm_loadu_si128(ptr.add(8).cast()));
deinterleave_rgb96_4px(v0, v1, v2)
}
}
#[inline(always)]
unsafe fn pack_u32x4_quad_to_u8x16(v0: __m128i, v1: __m128i, v2: __m128i, v3: __m128i) -> __m128i {
unsafe { _mm_packus_epi16(_mm_packus_epi32(v0, v1), _mm_packus_epi32(v2, v3)) }
}
#[inline(always)]
unsafe fn block_rgb96_to_rgb_8px<const BE: bool>(src_ptr: *const u32, out_ptr: *mut u8) {
unsafe {
let zero = _mm_setzero_si128();
let (r0, g0, b0) = load_deint_rgb96_4px::<BE>(src_ptr);
let (r1, g1, b1) = load_deint_rgb96_4px::<BE>(src_ptr.add(12));
let r = pack_u32x4_quad_to_u8x16(
_mm_srli_epi32::<24>(r0),
_mm_srli_epi32::<24>(r1),
zero,
zero,
);
let g = pack_u32x4_quad_to_u8x16(
_mm_srli_epi32::<24>(g0),
_mm_srli_epi32::<24>(g1),
zero,
zero,
);
let b = pack_u32x4_quad_to_u8x16(
_mm_srli_epi32::<24>(b0),
_mm_srli_epi32::<24>(b1),
zero,
zero,
);
let mut tmp = [0u8; 48];
write_rgb_16(r, g, b, tmp.as_mut_ptr());
core::ptr::copy_nonoverlapping(tmp.as_ptr(), out_ptr, 24);
}
}
#[inline(always)]
unsafe fn block_rgb96_to_rgba_8px<const BE: bool>(src_ptr: *const u32, out_ptr: *mut u8) {
unsafe {
let zero = _mm_setzero_si128();
let opaque = _mm_set1_epi8(-1i8);
let (r0, g0, b0) = load_deint_rgb96_4px::<BE>(src_ptr);
let (r1, g1, b1) = load_deint_rgb96_4px::<BE>(src_ptr.add(12));
let r = pack_u32x4_quad_to_u8x16(
_mm_srli_epi32::<24>(r0),
_mm_srli_epi32::<24>(r1),
zero,
zero,
);
let g = pack_u32x4_quad_to_u8x16(
_mm_srli_epi32::<24>(g0),
_mm_srli_epi32::<24>(g1),
zero,
zero,
);
let b = pack_u32x4_quad_to_u8x16(
_mm_srli_epi32::<24>(b0),
_mm_srli_epi32::<24>(b1),
zero,
zero,
);
let mut tmp = [0u8; 64];
write_rgba_16(r, g, b, opaque, tmp.as_mut_ptr());
core::ptr::copy_nonoverlapping(tmp.as_ptr(), out_ptr, 32);
}
}
#[inline(always)]
unsafe fn block_rgb96_to_rgb_u16_8px<const BE: bool>(src_ptr: *const u32, out_ptr: *mut u16) {
unsafe {
let (r0, g0, b0) = load_deint_rgb96_4px::<BE>(src_ptr);
let (r1, g1, b1) = load_deint_rgb96_4px::<BE>(src_ptr.add(12));
let r = _mm_packus_epi32(_mm_srli_epi32::<16>(r0), _mm_srli_epi32::<16>(r1));
let g = _mm_packus_epi32(_mm_srli_epi32::<16>(g0), _mm_srli_epi32::<16>(g1));
let b = _mm_packus_epi32(_mm_srli_epi32::<16>(b0), _mm_srli_epi32::<16>(b1));
write_rgb_u16_8(r, g, b, out_ptr);
}
}
#[inline(always)]
unsafe fn block_rgb96_to_rgba_u16_8px<const BE: bool>(src_ptr: *const u32, out_ptr: *mut u16) {
unsafe {
let opaque = _mm_set1_epi16(0xFFFFu16 as i16);
let (r0, g0, b0) = load_deint_rgb96_4px::<BE>(src_ptr);
let (r1, g1, b1) = load_deint_rgb96_4px::<BE>(src_ptr.add(12));
let r = _mm_packus_epi32(_mm_srli_epi32::<16>(r0), _mm_srli_epi32::<16>(r1));
let g = _mm_packus_epi32(_mm_srli_epi32::<16>(g0), _mm_srli_epi32::<16>(g1));
let b = _mm_packus_epi32(_mm_srli_epi32::<16>(b0), _mm_srli_epi32::<16>(b1));
write_rgba_u16_8(r, g, b, opaque, out_ptr);
}
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) unsafe fn avx2_rgb96_to_rgb_row<const BE: bool>(
rgb96: &[u32],
rgb_out: &mut [u8],
width: usize,
) {
debug_assert!(rgb96.len() >= width * 3, "rgb96 row too short");
debug_assert!(rgb_out.len() >= width * 3, "rgb_out row too short");
unsafe {
let mut x = 0usize;
while x + 16 <= width {
block_rgb96_to_rgb_8px::<BE>(rgb96.as_ptr().add(x * 3), rgb_out.as_mut_ptr().add(x * 3));
block_rgb96_to_rgb_8px::<BE>(
rgb96.as_ptr().add((x + 8) * 3),
rgb_out.as_mut_ptr().add((x + 8) * 3),
);
x += 16;
}
while x + 8 <= width {
block_rgb96_to_rgb_8px::<BE>(rgb96.as_ptr().add(x * 3), rgb_out.as_mut_ptr().add(x * 3));
x += 8;
}
if x < width {
scalar::rgb96_to_rgb_row::<BE>(&rgb96[x * 3..], &mut rgb_out[x * 3..], width - x);
}
}
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) unsafe fn avx2_rgb96_to_rgba_row<const BE: bool>(
rgb96: &[u32],
rgba_out: &mut [u8],
width: usize,
) {
debug_assert!(rgb96.len() >= width * 3, "rgb96 row too short");
debug_assert!(rgba_out.len() >= width * 4, "rgba_out row too short");
unsafe {
let mut x = 0usize;
while x + 16 <= width {
block_rgb96_to_rgba_8px::<BE>(rgb96.as_ptr().add(x * 3), rgba_out.as_mut_ptr().add(x * 4));
block_rgb96_to_rgba_8px::<BE>(
rgb96.as_ptr().add((x + 8) * 3),
rgba_out.as_mut_ptr().add((x + 8) * 4),
);
x += 16;
}
while x + 8 <= width {
block_rgb96_to_rgba_8px::<BE>(rgb96.as_ptr().add(x * 3), rgba_out.as_mut_ptr().add(x * 4));
x += 8;
}
if x < width {
scalar::rgb96_to_rgba_row::<BE>(&rgb96[x * 3..], &mut rgba_out[x * 4..], width - x);
}
}
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) unsafe fn avx2_rgb96_to_rgb_u16_row<const BE: bool>(
rgb96: &[u32],
rgb_out: &mut [u16],
width: usize,
) {
debug_assert!(rgb96.len() >= width * 3, "rgb96 row too short");
debug_assert!(rgb_out.len() >= width * 3, "rgb_out row too short");
unsafe {
let mut x = 0usize;
while x + 16 <= width {
block_rgb96_to_rgb_u16_8px::<BE>(rgb96.as_ptr().add(x * 3), rgb_out.as_mut_ptr().add(x * 3));
block_rgb96_to_rgb_u16_8px::<BE>(
rgb96.as_ptr().add((x + 8) * 3),
rgb_out.as_mut_ptr().add((x + 8) * 3),
);
x += 16;
}
while x + 8 <= width {
block_rgb96_to_rgb_u16_8px::<BE>(rgb96.as_ptr().add(x * 3), rgb_out.as_mut_ptr().add(x * 3));
x += 8;
}
if x < width {
scalar::rgb96_to_rgb_u16_row::<BE>(&rgb96[x * 3..], &mut rgb_out[x * 3..], width - x);
}
}
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) unsafe fn avx2_rgb96_to_rgba_u16_row<const BE: bool>(
rgb96: &[u32],
rgba_out: &mut [u16],
width: usize,
) {
debug_assert!(rgb96.len() >= width * 3, "rgb96 row too short");
debug_assert!(rgba_out.len() >= width * 4, "rgba_out row too short");
unsafe {
let mut x = 0usize;
while x + 16 <= width {
block_rgb96_to_rgba_u16_8px::<BE>(
rgb96.as_ptr().add(x * 3),
rgba_out.as_mut_ptr().add(x * 4),
);
block_rgb96_to_rgba_u16_8px::<BE>(
rgb96.as_ptr().add((x + 8) * 3),
rgba_out.as_mut_ptr().add((x + 8) * 4),
);
x += 16;
}
while x + 8 <= width {
block_rgb96_to_rgba_u16_8px::<BE>(
rgb96.as_ptr().add(x * 3),
rgba_out.as_mut_ptr().add(x * 4),
);
x += 8;
}
if x < width {
scalar::rgb96_to_rgba_u16_row::<BE>(&rgb96[x * 3..], &mut rgba_out[x * 4..], width - x);
}
}
}
#[inline(always)]
unsafe fn deinterleave_rgba128_4px(
v0: __m128i,
v1: __m128i,
v2: __m128i,
v3: __m128i,
) -> (__m128i, __m128i, __m128i, __m128i) {
unsafe {
let t0 = _mm_unpacklo_epi32(v0, v1);
let t1 = _mm_unpackhi_epi32(v0, v1);
let t2 = _mm_unpacklo_epi32(v2, v3);
let t3 = _mm_unpackhi_epi32(v2, v3);
let r = _mm_unpacklo_epi64(t0, t2);
let g = _mm_unpackhi_epi64(t0, t2);
let b = _mm_unpacklo_epi64(t1, t3);
let a = _mm_unpackhi_epi64(t1, t3);
(r, g, b, a)
}
}
#[inline(always)]
unsafe fn load_deint_rgba128_4px<const BE: bool>(
ptr: *const u32,
) -> (__m128i, __m128i, __m128i, __m128i) {
unsafe {
let v0 = byteswap32_if_be::<BE>(_mm_loadu_si128(ptr.cast()));
let v1 = byteswap32_if_be::<BE>(_mm_loadu_si128(ptr.add(4).cast()));
let v2 = byteswap32_if_be::<BE>(_mm_loadu_si128(ptr.add(8).cast()));
let v3 = byteswap32_if_be::<BE>(_mm_loadu_si128(ptr.add(12).cast()));
deinterleave_rgba128_4px(v0, v1, v2, v3)
}
}
#[inline(always)]
unsafe fn block_rgba128_to_rgb_8px<const BE: bool>(src_ptr: *const u32, out_ptr: *mut u8) {
unsafe {
let zero = _mm_setzero_si128();
let (r0, g0, b0, _) = load_deint_rgba128_4px::<BE>(src_ptr);
let (r1, g1, b1, _) = load_deint_rgba128_4px::<BE>(src_ptr.add(16));
let r = pack_u32x4_quad_to_u8x16(
_mm_srli_epi32::<24>(r0),
_mm_srli_epi32::<24>(r1),
zero,
zero,
);
let g = pack_u32x4_quad_to_u8x16(
_mm_srli_epi32::<24>(g0),
_mm_srli_epi32::<24>(g1),
zero,
zero,
);
let b = pack_u32x4_quad_to_u8x16(
_mm_srli_epi32::<24>(b0),
_mm_srli_epi32::<24>(b1),
zero,
zero,
);
let mut tmp = [0u8; 48];
write_rgb_16(r, g, b, tmp.as_mut_ptr());
core::ptr::copy_nonoverlapping(tmp.as_ptr(), out_ptr, 24);
}
}
#[inline(always)]
unsafe fn block_rgba128_to_rgba_8px<const BE: bool>(src_ptr: *const u32, out_ptr: *mut u8) {
unsafe {
let zero = _mm_setzero_si128();
let (r0, g0, b0, a0) = load_deint_rgba128_4px::<BE>(src_ptr);
let (r1, g1, b1, a1) = load_deint_rgba128_4px::<BE>(src_ptr.add(16));
let r = pack_u32x4_quad_to_u8x16(
_mm_srli_epi32::<24>(r0),
_mm_srli_epi32::<24>(r1),
zero,
zero,
);
let g = pack_u32x4_quad_to_u8x16(
_mm_srli_epi32::<24>(g0),
_mm_srli_epi32::<24>(g1),
zero,
zero,
);
let b = pack_u32x4_quad_to_u8x16(
_mm_srli_epi32::<24>(b0),
_mm_srli_epi32::<24>(b1),
zero,
zero,
);
let a = pack_u32x4_quad_to_u8x16(
_mm_srli_epi32::<24>(a0),
_mm_srli_epi32::<24>(a1),
zero,
zero,
);
let mut tmp = [0u8; 64];
write_rgba_16(r, g, b, a, tmp.as_mut_ptr());
core::ptr::copy_nonoverlapping(tmp.as_ptr(), out_ptr, 32);
}
}
#[inline(always)]
unsafe fn block_rgba128_to_rgb_u16_8px<const BE: bool>(src_ptr: *const u32, out_ptr: *mut u16) {
unsafe {
let (r0, g0, b0, _) = load_deint_rgba128_4px::<BE>(src_ptr);
let (r1, g1, b1, _) = load_deint_rgba128_4px::<BE>(src_ptr.add(16));
let r = _mm_packus_epi32(_mm_srli_epi32::<16>(r0), _mm_srli_epi32::<16>(r1));
let g = _mm_packus_epi32(_mm_srli_epi32::<16>(g0), _mm_srli_epi32::<16>(g1));
let b = _mm_packus_epi32(_mm_srli_epi32::<16>(b0), _mm_srli_epi32::<16>(b1));
write_rgb_u16_8(r, g, b, out_ptr);
}
}
#[inline(always)]
unsafe fn block_rgba128_to_rgba_u16_8px<const BE: bool>(src_ptr: *const u32, out_ptr: *mut u16) {
unsafe {
let (r0, g0, b0, a0) = load_deint_rgba128_4px::<BE>(src_ptr);
let (r1, g1, b1, a1) = load_deint_rgba128_4px::<BE>(src_ptr.add(16));
let r = _mm_packus_epi32(_mm_srli_epi32::<16>(r0), _mm_srli_epi32::<16>(r1));
let g = _mm_packus_epi32(_mm_srli_epi32::<16>(g0), _mm_srli_epi32::<16>(g1));
let b = _mm_packus_epi32(_mm_srli_epi32::<16>(b0), _mm_srli_epi32::<16>(b1));
let a = _mm_packus_epi32(_mm_srli_epi32::<16>(a0), _mm_srli_epi32::<16>(a1));
write_rgba_u16_8(r, g, b, a, out_ptr);
}
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) unsafe fn avx2_rgba128_to_rgb_row<const BE: bool>(
rgba128: &[u32],
rgb_out: &mut [u8],
width: usize,
) {
debug_assert!(rgba128.len() >= width * 4, "rgba128 row too short");
debug_assert!(rgb_out.len() >= width * 3, "rgb_out row too short");
unsafe {
let mut x = 0usize;
while x + 16 <= width {
block_rgba128_to_rgb_8px::<BE>(rgba128.as_ptr().add(x * 4), rgb_out.as_mut_ptr().add(x * 3));
block_rgba128_to_rgb_8px::<BE>(
rgba128.as_ptr().add((x + 8) * 4),
rgb_out.as_mut_ptr().add((x + 8) * 3),
);
x += 16;
}
while x + 8 <= width {
block_rgba128_to_rgb_8px::<BE>(rgba128.as_ptr().add(x * 4), rgb_out.as_mut_ptr().add(x * 3));
x += 8;
}
if x < width {
scalar::rgba128_to_rgb_row::<BE>(&rgba128[x * 4..], &mut rgb_out[x * 3..], width - x);
}
}
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) unsafe fn avx2_rgba128_to_rgba_row<const BE: bool>(
rgba128: &[u32],
rgba_out: &mut [u8],
width: usize,
) {
debug_assert!(rgba128.len() >= width * 4, "rgba128 row too short");
debug_assert!(rgba_out.len() >= width * 4, "rgba_out row too short");
unsafe {
let mut x = 0usize;
while x + 16 <= width {
block_rgba128_to_rgba_8px::<BE>(
rgba128.as_ptr().add(x * 4),
rgba_out.as_mut_ptr().add(x * 4),
);
block_rgba128_to_rgba_8px::<BE>(
rgba128.as_ptr().add((x + 8) * 4),
rgba_out.as_mut_ptr().add((x + 8) * 4),
);
x += 16;
}
while x + 8 <= width {
block_rgba128_to_rgba_8px::<BE>(
rgba128.as_ptr().add(x * 4),
rgba_out.as_mut_ptr().add(x * 4),
);
x += 8;
}
if x < width {
scalar::rgba128_to_rgba_row::<BE>(&rgba128[x * 4..], &mut rgba_out[x * 4..], width - x);
}
}
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) unsafe fn avx2_rgba128_to_rgb_u16_row<const BE: bool>(
rgba128: &[u32],
rgb_out: &mut [u16],
width: usize,
) {
debug_assert!(rgba128.len() >= width * 4, "rgba128 row too short");
debug_assert!(rgb_out.len() >= width * 3, "rgb_out row too short");
unsafe {
let mut x = 0usize;
while x + 16 <= width {
block_rgba128_to_rgb_u16_8px::<BE>(
rgba128.as_ptr().add(x * 4),
rgb_out.as_mut_ptr().add(x * 3),
);
block_rgba128_to_rgb_u16_8px::<BE>(
rgba128.as_ptr().add((x + 8) * 4),
rgb_out.as_mut_ptr().add((x + 8) * 3),
);
x += 16;
}
while x + 8 <= width {
block_rgba128_to_rgb_u16_8px::<BE>(
rgba128.as_ptr().add(x * 4),
rgb_out.as_mut_ptr().add(x * 3),
);
x += 8;
}
if x < width {
scalar::rgba128_to_rgb_u16_row::<BE>(&rgba128[x * 4..], &mut rgb_out[x * 3..], width - x);
}
}
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) unsafe fn avx2_rgba128_to_rgba_u16_row<const BE: bool>(
rgba128: &[u32],
rgba_out: &mut [u16],
width: usize,
) {
debug_assert!(rgba128.len() >= width * 4, "rgba128 row too short");
debug_assert!(rgba_out.len() >= width * 4, "rgba_out row too short");
unsafe {
let mut x = 0usize;
while x + 16 <= width {
block_rgba128_to_rgba_u16_8px::<BE>(
rgba128.as_ptr().add(x * 4),
rgba_out.as_mut_ptr().add(x * 4),
);
block_rgba128_to_rgba_u16_8px::<BE>(
rgba128.as_ptr().add((x + 8) * 4),
rgba_out.as_mut_ptr().add((x + 8) * 4),
);
x += 16;
}
while x + 8 <= width {
block_rgba128_to_rgba_u16_8px::<BE>(
rgba128.as_ptr().add(x * 4),
rgba_out.as_mut_ptr().add(x * 4),
);
x += 8;
}
if x < width {
scalar::rgba128_to_rgba_u16_row::<BE>(&rgba128[x * 4..], &mut rgba_out[x * 4..], width - x);
}
}
}