#![cfg_attr(not(feature = "std"), allow(dead_code))]
#[cfg_attr(miri, allow(unused_imports))]
use core::arch::aarch64::*;
use crate::row::{arch::neon::endian::load_endian_u32x4, scalar::planar_gbr_32bit as scalar};
#[inline(always)]
unsafe fn narrow_u16x4<const BE: bool>(ptr: *const u32) -> uint16x4_t {
unsafe { vmovn_u32(vshrq_n_u32::<16>(load_endian_u32x4::<BE>(ptr.cast()))) }
}
#[inline(always)]
unsafe fn narrow_u8x8<const BE: bool>(ptr: *const u32) -> uint8x8_t {
unsafe {
let lo = narrow_u16x4::<BE>(ptr);
let hi = narrow_u16x4::<BE>(ptr.add(4));
vshrn_n_u16::<8>(vcombine_u16(lo, hi))
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn gbr32_to_rgb_row<const BE: bool>(
g: &[u32],
b: &[u32],
r: &[u32],
rgb_out: &mut [u8],
width: usize,
) {
debug_assert!(g.len() >= width, "g row too short");
debug_assert!(b.len() >= width, "b row too short");
debug_assert!(r.len() >= width, "r row too short");
debug_assert!(rgb_out.len() >= width * 3, "rgb_out row too short");
unsafe {
let mut x = 0usize;
while x + 8 <= width {
let r8 = narrow_u8x8::<BE>(r.as_ptr().add(x));
let g8 = narrow_u8x8::<BE>(g.as_ptr().add(x));
let b8 = narrow_u8x8::<BE>(b.as_ptr().add(x));
vst3_u8(rgb_out.as_mut_ptr().add(x * 3), uint8x8x3_t(r8, g8, b8));
x += 8;
}
if x < width {
scalar::gbr32_to_rgb_row::<BE>(
&g[x..width],
&b[x..width],
&r[x..width],
&mut rgb_out[x * 3..width * 3],
width - x,
);
}
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn gbr32_to_rgb_u16_row<const BE: bool>(
g: &[u32],
b: &[u32],
r: &[u32],
rgb_u16_out: &mut [u16],
width: usize,
) {
debug_assert!(g.len() >= width, "g row too short");
debug_assert!(b.len() >= width, "b row too short");
debug_assert!(r.len() >= width, "r row too short");
debug_assert!(rgb_u16_out.len() >= width * 3, "rgb_u16_out row too short");
unsafe {
let mut x = 0usize;
while x + 4 <= width {
let r16 = narrow_u16x4::<BE>(r.as_ptr().add(x));
let g16 = narrow_u16x4::<BE>(g.as_ptr().add(x));
let b16 = narrow_u16x4::<BE>(b.as_ptr().add(x));
vst3_u16(
rgb_u16_out.as_mut_ptr().add(x * 3),
uint16x4x3_t(r16, g16, b16),
);
x += 4;
}
if x < width {
scalar::gbr32_to_rgb_u16_row::<BE>(
&g[x..width],
&b[x..width],
&r[x..width],
&mut rgb_u16_out[x * 3..width * 3],
width - x,
);
}
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn gbra32_to_rgba_row<const BE: bool>(
g: &[u32],
b: &[u32],
r: &[u32],
a: &[u32],
rgba_out: &mut [u8],
width: usize,
) {
debug_assert!(g.len() >= width, "g row too short");
debug_assert!(b.len() >= width, "b row too short");
debug_assert!(r.len() >= width, "r row too short");
debug_assert!(a.len() >= width, "a row too short");
debug_assert!(rgba_out.len() >= width * 4, "rgba_out row too short");
unsafe {
let mut x = 0usize;
while x + 8 <= width {
let r8 = narrow_u8x8::<BE>(r.as_ptr().add(x));
let g8 = narrow_u8x8::<BE>(g.as_ptr().add(x));
let b8 = narrow_u8x8::<BE>(b.as_ptr().add(x));
let a8 = narrow_u8x8::<BE>(a.as_ptr().add(x));
vst4_u8(
rgba_out.as_mut_ptr().add(x * 4),
uint8x8x4_t(r8, g8, b8, a8),
);
x += 8;
}
if x < width {
scalar::gbra32_to_rgba_row::<BE>(
&g[x..width],
&b[x..width],
&r[x..width],
&a[x..width],
&mut rgba_out[x * 4..width * 4],
width - x,
);
}
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn gbra32_to_rgba_u16_row<const BE: bool>(
g: &[u32],
b: &[u32],
r: &[u32],
a: &[u32],
rgba_u16_out: &mut [u16],
width: usize,
) {
debug_assert!(g.len() >= width, "g row too short");
debug_assert!(b.len() >= width, "b row too short");
debug_assert!(r.len() >= width, "r row too short");
debug_assert!(a.len() >= width, "a row too short");
debug_assert!(
rgba_u16_out.len() >= width * 4,
"rgba_u16_out row too short"
);
unsafe {
let mut x = 0usize;
while x + 4 <= width {
let r16 = narrow_u16x4::<BE>(r.as_ptr().add(x));
let g16 = narrow_u16x4::<BE>(g.as_ptr().add(x));
let b16 = narrow_u16x4::<BE>(b.as_ptr().add(x));
let a16 = narrow_u16x4::<BE>(a.as_ptr().add(x));
vst4_u16(
rgba_u16_out.as_mut_ptr().add(x * 4),
uint16x4x4_t(r16, g16, b16, a16),
);
x += 4;
}
if x < width {
scalar::gbra32_to_rgba_u16_row::<BE>(
&g[x..width],
&b[x..width],
&r[x..width],
&a[x..width],
&mut rgba_u16_out[x * 4..width * 4],
width - x,
);
}
}
}