#![allow(dead_code)]
#[cfg_attr(miri, allow(unused_imports))]
use core::arch::aarch64::*;
use super::bswap_u32x4_if_be;
use crate::row::scalar;
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn neon_rgb96_to_rgb_row<const BE: bool>(
rgb96: &[u32],
rgb_out: &mut [u8],
width: usize,
) {
debug_assert!(rgb96.len() >= width * 3, "rgb96 row too short");
debug_assert!(rgb_out.len() >= width * 3, "rgb_out row too short");
unsafe {
let mut x = 0usize;
while x + 8 <= width {
let lo: uint32x4x3_t = vld3q_u32(rgb96.as_ptr().add(x * 3));
let hi: uint32x4x3_t = vld3q_u32(rgb96.as_ptr().add((x + 4) * 3));
let r = narrow_u32_pair_to_u8x8::<BE>(lo.0, hi.0);
let g = narrow_u32_pair_to_u8x8::<BE>(lo.1, hi.1);
let b = narrow_u32_pair_to_u8x8::<BE>(lo.2, hi.2);
vst3_u8(rgb_out.as_mut_ptr().add(x * 3), uint8x8x3_t(r, g, b));
x += 8;
}
if x < width {
scalar::rgb96_to_rgb_row::<BE>(&rgb96[x * 3..], &mut rgb_out[x * 3..], width - x);
}
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn neon_rgb96_to_rgba_row<const BE: bool>(
rgb96: &[u32],
rgba_out: &mut [u8],
width: usize,
) {
debug_assert!(rgb96.len() >= width * 3, "rgb96 row too short");
debug_assert!(rgba_out.len() >= width * 4, "rgba_out row too short");
unsafe {
let alpha = vdup_n_u8(0xFF);
let mut x = 0usize;
while x + 8 <= width {
let lo: uint32x4x3_t = vld3q_u32(rgb96.as_ptr().add(x * 3));
let hi: uint32x4x3_t = vld3q_u32(rgb96.as_ptr().add((x + 4) * 3));
let r = narrow_u32_pair_to_u8x8::<BE>(lo.0, hi.0);
let g = narrow_u32_pair_to_u8x8::<BE>(lo.1, hi.1);
let b = narrow_u32_pair_to_u8x8::<BE>(lo.2, hi.2);
vst4_u8(
rgba_out.as_mut_ptr().add(x * 4),
uint8x8x4_t(r, g, b, alpha),
);
x += 8;
}
if x < width {
scalar::rgb96_to_rgba_row::<BE>(&rgb96[x * 3..], &mut rgba_out[x * 4..], width - x);
}
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn neon_rgb96_to_rgb_u16_row<const BE: bool>(
rgb96: &[u32],
rgb_out: &mut [u16],
width: usize,
) {
debug_assert!(rgb96.len() >= width * 3, "rgb96 row too short");
debug_assert!(rgb_out.len() >= width * 3, "rgb_out row too short");
unsafe {
let mut x = 0usize;
while x + 4 <= width {
let px: uint32x4x3_t = vld3q_u32(rgb96.as_ptr().add(x * 3));
vst3_u16(
rgb_out.as_mut_ptr().add(x * 3),
uint16x4x3_t(
vshrn_n_u32::<16>(bswap_u32x4_if_be::<BE>(px.0)),
vshrn_n_u32::<16>(bswap_u32x4_if_be::<BE>(px.1)),
vshrn_n_u32::<16>(bswap_u32x4_if_be::<BE>(px.2)),
),
);
x += 4;
}
if x < width {
scalar::rgb96_to_rgb_u16_row::<BE>(&rgb96[x * 3..], &mut rgb_out[x * 3..], width - x);
}
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn neon_rgb96_to_rgba_u16_row<const BE: bool>(
rgb96: &[u32],
rgba_out: &mut [u16],
width: usize,
) {
debug_assert!(rgb96.len() >= width * 3, "rgb96 row too short");
debug_assert!(rgba_out.len() >= width * 4, "rgba_out row too short");
unsafe {
let alpha = vdup_n_u16(0xFFFF);
let mut x = 0usize;
while x + 4 <= width {
let px: uint32x4x3_t = vld3q_u32(rgb96.as_ptr().add(x * 3));
vst4_u16(
rgba_out.as_mut_ptr().add(x * 4),
uint16x4x4_t(
vshrn_n_u32::<16>(bswap_u32x4_if_be::<BE>(px.0)),
vshrn_n_u32::<16>(bswap_u32x4_if_be::<BE>(px.1)),
vshrn_n_u32::<16>(bswap_u32x4_if_be::<BE>(px.2)),
alpha,
),
);
x += 4;
}
if x < width {
scalar::rgb96_to_rgba_u16_row::<BE>(&rgb96[x * 3..], &mut rgba_out[x * 4..], width - x);
}
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn neon_rgba128_to_rgb_row<const BE: bool>(
rgba128: &[u32],
rgb_out: &mut [u8],
width: usize,
) {
debug_assert!(rgba128.len() >= width * 4, "rgba128 row too short");
debug_assert!(rgb_out.len() >= width * 3, "rgb_out row too short");
unsafe {
let mut x = 0usize;
while x + 8 <= width {
let lo: uint32x4x4_t = vld4q_u32(rgba128.as_ptr().add(x * 4));
let hi: uint32x4x4_t = vld4q_u32(rgba128.as_ptr().add((x + 4) * 4));
let r = narrow_u32_pair_to_u8x8::<BE>(lo.0, hi.0);
let g = narrow_u32_pair_to_u8x8::<BE>(lo.1, hi.1);
let b = narrow_u32_pair_to_u8x8::<BE>(lo.2, hi.2);
vst3_u8(rgb_out.as_mut_ptr().add(x * 3), uint8x8x3_t(r, g, b));
x += 8;
}
if x < width {
scalar::rgba128_to_rgb_row::<BE>(&rgba128[x * 4..], &mut rgb_out[x * 3..], width - x);
}
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn neon_rgba128_to_rgba_row<const BE: bool>(
rgba128: &[u32],
rgba_out: &mut [u8],
width: usize,
) {
debug_assert!(rgba128.len() >= width * 4, "rgba128 row too short");
debug_assert!(rgba_out.len() >= width * 4, "rgba_out row too short");
unsafe {
let mut x = 0usize;
while x + 8 <= width {
let lo: uint32x4x4_t = vld4q_u32(rgba128.as_ptr().add(x * 4));
let hi: uint32x4x4_t = vld4q_u32(rgba128.as_ptr().add((x + 4) * 4));
let r = narrow_u32_pair_to_u8x8::<BE>(lo.0, hi.0);
let g = narrow_u32_pair_to_u8x8::<BE>(lo.1, hi.1);
let b = narrow_u32_pair_to_u8x8::<BE>(lo.2, hi.2);
let a = narrow_u32_pair_to_u8x8::<BE>(lo.3, hi.3);
vst4_u8(rgba_out.as_mut_ptr().add(x * 4), uint8x8x4_t(r, g, b, a));
x += 8;
}
if x < width {
scalar::rgba128_to_rgba_row::<BE>(&rgba128[x * 4..], &mut rgba_out[x * 4..], width - x);
}
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn neon_rgba128_to_rgb_u16_row<const BE: bool>(
rgba128: &[u32],
rgb_out: &mut [u16],
width: usize,
) {
debug_assert!(rgba128.len() >= width * 4, "rgba128 row too short");
debug_assert!(rgb_out.len() >= width * 3, "rgb_out row too short");
unsafe {
let mut x = 0usize;
while x + 4 <= width {
let px: uint32x4x4_t = vld4q_u32(rgba128.as_ptr().add(x * 4));
vst3_u16(
rgb_out.as_mut_ptr().add(x * 3),
uint16x4x3_t(
vshrn_n_u32::<16>(bswap_u32x4_if_be::<BE>(px.0)),
vshrn_n_u32::<16>(bswap_u32x4_if_be::<BE>(px.1)),
vshrn_n_u32::<16>(bswap_u32x4_if_be::<BE>(px.2)),
),
);
x += 4;
}
if x < width {
scalar::rgba128_to_rgb_u16_row::<BE>(&rgba128[x * 4..], &mut rgb_out[x * 3..], width - x);
}
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn neon_rgba128_to_rgba_u16_row<const BE: bool>(
rgba128: &[u32],
rgba_out: &mut [u16],
width: usize,
) {
debug_assert!(rgba128.len() >= width * 4, "rgba128 row too short");
debug_assert!(rgba_out.len() >= width * 4, "rgba_out row too short");
unsafe {
let mut x = 0usize;
while x + 4 <= width {
let px: uint32x4x4_t = vld4q_u32(rgba128.as_ptr().add(x * 4));
vst4_u16(
rgba_out.as_mut_ptr().add(x * 4),
uint16x4x4_t(
vshrn_n_u32::<16>(bswap_u32x4_if_be::<BE>(px.0)),
vshrn_n_u32::<16>(bswap_u32x4_if_be::<BE>(px.1)),
vshrn_n_u32::<16>(bswap_u32x4_if_be::<BE>(px.2)),
vshrn_n_u32::<16>(bswap_u32x4_if_be::<BE>(px.3)),
),
);
x += 4;
}
if x < width {
scalar::rgba128_to_rgba_u16_row::<BE>(&rgba128[x * 4..], &mut rgba_out[x * 4..], width - x);
}
}
}
#[inline(always)]
unsafe fn narrow_u32_pair_to_u8x8<const BE: bool>(lo: uint32x4_t, hi: uint32x4_t) -> uint8x8_t {
unsafe {
let lo16 = vshrn_n_u32::<16>(bswap_u32x4_if_be::<BE>(lo));
let hi16 = vshrn_n_u32::<16>(bswap_u32x4_if_be::<BE>(hi));
vshrn_n_u16::<8>(vcombine_u16(lo16, hi16))
}
}