use super::*;
use crate::{KernelMatrix, row::scalar};
#[inline]
#[target_feature(enable = "neon")]
unsafe fn pick4<const OFF: usize>(q: &uint8x16x4_t) -> uint8x16_t {
match OFF {
0 => q.0,
1 => q.1,
2 => q.2,
_ => q.3,
}
}
#[inline]
#[target_feature(enable = "neon")]
#[allow(clippy::too_many_arguments)]
unsafe fn vuya_compute_rgb16(
v_raw: uint8x16_t,
u_raw: uint8x16_t,
y_raw: uint8x16_t,
y_off_v: int16x8_t,
y_scale_v: int32x4_t,
c_scale_v: int32x4_t,
bias_v: int16x8_t,
rnd_v: int32x4_t,
cru: int32x4_t,
crv: int32x4_t,
cgu: int32x4_t,
cgv: int32x4_t,
cbu: int32x4_t,
cbv: int32x4_t,
) -> (uint8x16_t, uint8x16_t, uint8x16_t) {
unsafe {
let v_lo = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(v_raw)));
let v_hi = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(v_raw)));
let u_lo = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(u_raw)));
let u_hi = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(u_raw)));
let y_lo = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(y_raw)));
let y_hi = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(y_raw)));
let u_sub_lo = vsubq_s16(u_lo, bias_v);
let u_sub_hi = vsubq_s16(u_hi, bias_v);
let v_sub_lo = vsubq_s16(v_lo, bias_v);
let v_sub_hi = vsubq_s16(v_hi, bias_v);
let u_lo_lo_i32 = vmovl_s16(vget_low_s16(u_sub_lo));
let u_lo_hi_i32 = vmovl_s16(vget_high_s16(u_sub_lo));
let v_lo_lo_i32 = vmovl_s16(vget_low_s16(v_sub_lo));
let v_lo_hi_i32 = vmovl_s16(vget_high_s16(v_sub_lo));
let u_d_lo_lo = q15_shift(vaddq_s32(vmulq_s32(u_lo_lo_i32, c_scale_v), rnd_v));
let u_d_lo_hi = q15_shift(vaddq_s32(vmulq_s32(u_lo_hi_i32, c_scale_v), rnd_v));
let v_d_lo_lo = q15_shift(vaddq_s32(vmulq_s32(v_lo_lo_i32, c_scale_v), rnd_v));
let v_d_lo_hi = q15_shift(vaddq_s32(vmulq_s32(v_lo_hi_i32, c_scale_v), rnd_v));
let r_chroma_lo = chroma_i16x8(cru, crv, u_d_lo_lo, v_d_lo_lo, u_d_lo_hi, v_d_lo_hi, rnd_v);
let g_chroma_lo = chroma_i16x8(cgu, cgv, u_d_lo_lo, v_d_lo_lo, u_d_lo_hi, v_d_lo_hi, rnd_v);
let b_chroma_lo = chroma_i16x8(cbu, cbv, u_d_lo_lo, v_d_lo_lo, u_d_lo_hi, v_d_lo_hi, rnd_v);
let u_hi_lo_i32 = vmovl_s16(vget_low_s16(u_sub_hi));
let u_hi_hi_i32 = vmovl_s16(vget_high_s16(u_sub_hi));
let v_hi_lo_i32 = vmovl_s16(vget_low_s16(v_sub_hi));
let v_hi_hi_i32 = vmovl_s16(vget_high_s16(v_sub_hi));
let u_d_hi_lo = q15_shift(vaddq_s32(vmulq_s32(u_hi_lo_i32, c_scale_v), rnd_v));
let u_d_hi_hi = q15_shift(vaddq_s32(vmulq_s32(u_hi_hi_i32, c_scale_v), rnd_v));
let v_d_hi_lo = q15_shift(vaddq_s32(vmulq_s32(v_hi_lo_i32, c_scale_v), rnd_v));
let v_d_hi_hi = q15_shift(vaddq_s32(vmulq_s32(v_hi_hi_i32, c_scale_v), rnd_v));
let r_chroma_hi = chroma_i16x8(cru, crv, u_d_hi_lo, v_d_hi_lo, u_d_hi_hi, v_d_hi_hi, rnd_v);
let g_chroma_hi = chroma_i16x8(cgu, cgv, u_d_hi_lo, v_d_hi_lo, u_d_hi_hi, v_d_hi_hi, rnd_v);
let b_chroma_hi = chroma_i16x8(cbu, cbv, u_d_hi_lo, v_d_hi_lo, u_d_hi_hi, v_d_hi_hi, rnd_v);
let y_scaled_lo = scale_y(y_lo, y_off_v, y_scale_v, rnd_v);
let y_scaled_hi = scale_y(y_hi, y_off_v, y_scale_v, rnd_v);
let r_u8 = vcombine_u8(
vqmovun_s16_compat(vqaddq_s16(y_scaled_lo, r_chroma_lo)),
vqmovun_s16_compat(vqaddq_s16(y_scaled_hi, r_chroma_hi)),
);
let g_u8 = vcombine_u8(
vqmovun_s16_compat(vqaddq_s16(y_scaled_lo, g_chroma_lo)),
vqmovun_s16_compat(vqaddq_s16(y_scaled_hi, g_chroma_hi)),
);
let b_u8 = vcombine_u8(
vqmovun_s16_compat(vqaddq_s16(y_scaled_lo, b_chroma_lo)),
vqmovun_s16_compat(vqaddq_s16(y_scaled_hi, b_chroma_hi)),
);
(r_u8, g_u8, b_u8)
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn packed444_to_rgb_or_rgba_row<
const ALPHA: bool,
const ALPHA_SRC: bool,
const V_OFF: usize,
const U_OFF: usize,
const Y_OFF: usize,
const A_OFF: usize,
>(
packed: &[u8],
out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
const { assert!(!ALPHA_SRC || ALPHA) };
debug_assert!(packed.len() >= width * 4, "packed row too short");
let bpp: usize = if ALPHA { 4 } else { 3 };
debug_assert!(out.len() >= width * bpp, "out row too short");
let coeffs = scalar::Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = scalar::range_params_n::<8, 8>(full_range);
let bias = scalar::chroma_bias::<8>();
const RND: i32 = 1 << 14;
unsafe {
let rnd_v = vdupq_n_s32(RND);
let y_off_v = vdupq_n_s16(y_off as i16);
let y_scale_v = vdupq_n_s32(y_scale);
let c_scale_v = vdupq_n_s32(c_scale);
let bias_v = vdupq_n_s16(bias as i16);
let cru = vdupq_n_s32(coeffs.r_u());
let crv = vdupq_n_s32(coeffs.r_v());
let cgu = vdupq_n_s32(coeffs.g_u());
let cgv = vdupq_n_s32(coeffs.g_v());
let cbu = vdupq_n_s32(coeffs.b_u());
let cbv = vdupq_n_s32(coeffs.b_v());
let mut x = 0usize;
while x + 16 <= width {
let q = vld4q_u8(packed.as_ptr().add(x * 4));
let v_raw = pick4::<V_OFF>(&q); let u_raw = pick4::<U_OFF>(&q); let y_raw = pick4::<Y_OFF>(&q); let a_raw = pick4::<A_OFF>(&q);
let (r_u8, g_u8, b_u8) = vuya_compute_rgb16(
v_raw, u_raw, y_raw, y_off_v, y_scale_v, c_scale_v, bias_v, rnd_v, cru, crv, cgu, cgv, cbu,
cbv,
);
let off = x * bpp;
if ALPHA {
let a_vec: uint8x16_t = if ALPHA_SRC {
a_raw } else {
vdupq_n_u8(0xFFu8) };
vst4q_u8(
out.as_mut_ptr().add(off),
uint8x16x4_t(r_u8, g_u8, b_u8, a_vec),
);
} else {
vst3q_u8(out.as_mut_ptr().add(off), uint8x16x3_t(r_u8, g_u8, b_u8));
}
x += 16;
}
let processed = x;
if processed < width {
let tail_packed = &packed[processed * 4..];
let tail_out = &mut out[processed * bpp..];
scalar::packed444_to_rgb_or_rgba_row::<ALPHA, ALPHA_SRC, V_OFF, U_OFF, Y_OFF, A_OFF>(
tail_packed,
tail_out,
width - processed,
matrix,
full_range,
);
}
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn vuya_to_rgb_or_rgba_row<const ALPHA: bool, const ALPHA_SRC: bool>(
packed: &[u8],
out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
packed444_to_rgb_or_rgba_row::<ALPHA, ALPHA_SRC, 0, 1, 2, 3>(
packed, out, width, matrix, full_range,
);
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn vuya_to_rgb_row(
packed: &[u8],
rgb_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
vuya_to_rgb_or_rgba_row::<false, false>(packed, rgb_out, width, matrix, full_range);
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn vuya_to_rgba_row(
packed: &[u8],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
vuya_to_rgb_or_rgba_row::<true, true>(packed, rgba_out, width, matrix, full_range);
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn vuyx_to_rgba_row(
packed: &[u8],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
vuya_to_rgb_or_rgba_row::<true, false>(packed, rgba_out, width, matrix, full_range);
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn packed444_to_luma_row<const Y_OFF: usize>(
packed: &[u8],
luma_out: &mut [u8],
width: usize,
) {
debug_assert!(packed.len() >= width * 4, "packed row too short");
debug_assert!(luma_out.len() >= width, "luma row too short");
unsafe {
let mut x = 0usize;
while x + 16 <= width {
let q = vld4q_u8(packed.as_ptr().add(x * 4));
vst1q_u8(luma_out.as_mut_ptr().add(x), pick4::<Y_OFF>(&q));
x += 16;
}
if x < width {
scalar::packed444_to_luma_row::<Y_OFF>(&packed[x * 4..], &mut luma_out[x..], width - x);
}
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn vuya_to_luma_row(packed: &[u8], luma_out: &mut [u8], width: usize) {
unsafe {
packed444_to_luma_row::<2>(packed, luma_out, width);
}
}
#[cfg_attr(not(any(feature = "std", feature = "alloc")), allow(dead_code))]
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn packed444_to_luma_u16_row<const Y_OFF: usize>(
packed: &[u8],
out: &mut [u16],
width: usize,
) {
debug_assert!(packed.len() >= width * 4, "packed row too short");
debug_assert!(out.len() >= width, "out too short");
unsafe {
let mut x = 0usize;
while x + 16 <= width {
let q = vld4q_u8(packed.as_ptr().add(x * 4));
let y = pick4::<Y_OFF>(&q);
let y_lo = vmovl_u8(vget_low_u8(y)); let y_hi = vmovl_u8(vget_high_u8(y)); vst1q_u16(out.as_mut_ptr().add(x), y_lo);
vst1q_u16(out.as_mut_ptr().add(x + 8), y_hi);
x += 16;
}
if x < width {
scalar::packed444_to_luma_u16_row::<Y_OFF>(&packed[x * 4..], &mut out[x..], width - x);
}
}
}
#[cfg_attr(not(any(feature = "std", feature = "alloc")), allow(dead_code))]
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn vuya_to_luma_u16_row(packed: &[u8], out: &mut [u16], width: usize) {
unsafe {
packed444_to_luma_u16_row::<2>(packed, out, width);
}
}
#[allow(dead_code)]
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn vuyx_to_luma_u16_row(packed: &[u8], out: &mut [u16], width: usize) {
unsafe {
vuya_to_luma_u16_row(packed, out, width);
}
}
const HSV_CHUNK: usize = 64;
#[inline]
unsafe fn vuya_hsv_via_rgb_chunks(
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
mut fill_rgb: impl FnMut(usize, usize, &mut [u8]),
) {
let mut scratch = [0u8; HSV_CHUNK * 3];
let mut offset = 0;
while offset < width {
let n = (width - offset).min(HSV_CHUNK);
fill_rgb(offset, n, &mut scratch[..n * 3]);
unsafe {
rgb_to_hsv_row(
&scratch[..n * 3],
&mut h_out[offset..offset + n],
&mut s_out[offset..offset + n],
&mut v_out[offset..offset + n],
n,
);
}
offset += n;
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn packed444_to_hsv_row<
const V_OFF: usize,
const U_OFF: usize,
const Y_OFF: usize,
>(
packed: &[u8],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
debug_assert!(packed.len() >= width * 4, "packed row too short");
debug_assert!(h_out.len() >= width, "h_out row too short");
debug_assert!(s_out.len() >= width, "s_out row too short");
debug_assert!(v_out.len() >= width, "v_out row too short");
unsafe {
vuya_hsv_via_rgb_chunks(h_out, s_out, v_out, width, |offset, n, rgb| {
packed444_to_rgb_or_rgba_row::<false, false, V_OFF, U_OFF, Y_OFF, Y_OFF>(
&packed[offset * 4..],
rgb,
n,
matrix,
full_range,
);
});
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn vuya_to_hsv_row(
packed: &[u8],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
packed444_to_hsv_row::<0, 1, 2>(packed, h_out, s_out, v_out, width, matrix, full_range);
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn ayuv_to_rgb_row(
packed: &[u8],
rgb_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
packed444_to_rgb_or_rgba_row::<false, false, 3, 2, 1, 0>(
packed, rgb_out, width, matrix, full_range,
);
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn ayuv_to_rgba_row(
packed: &[u8],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
packed444_to_rgb_or_rgba_row::<true, true, 3, 2, 1, 0>(
packed, rgba_out, width, matrix, full_range,
);
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn ayuv_to_hsv_row(
packed: &[u8],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
packed444_to_hsv_row::<3, 2, 1>(packed, h_out, s_out, v_out, width, matrix, full_range);
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn ayuv_to_luma_row(packed: &[u8], luma_out: &mut [u8], width: usize) {
unsafe {
packed444_to_luma_row::<1>(packed, luma_out, width);
}
}
#[cfg_attr(not(any(feature = "std", feature = "alloc")), allow(dead_code))]
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn ayuv_to_luma_u16_row(packed: &[u8], out: &mut [u16], width: usize) {
unsafe {
packed444_to_luma_u16_row::<1>(packed, out, width);
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn uyva_to_rgb_row(
packed: &[u8],
rgb_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
packed444_to_rgb_or_rgba_row::<false, false, 2, 0, 1, 3>(
packed, rgb_out, width, matrix, full_range,
);
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn uyva_to_rgba_row(
packed: &[u8],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
packed444_to_rgb_or_rgba_row::<true, true, 2, 0, 1, 3>(
packed, rgba_out, width, matrix, full_range,
);
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn uyva_to_hsv_row(
packed: &[u8],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
packed444_to_hsv_row::<2, 0, 1>(packed, h_out, s_out, v_out, width, matrix, full_range);
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn uyva_to_luma_row(packed: &[u8], luma_out: &mut [u8], width: usize) {
unsafe {
packed444_to_luma_row::<1>(packed, luma_out, width);
}
}
#[cfg_attr(not(any(feature = "std", feature = "alloc")), allow(dead_code))]
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn uyva_to_luma_u16_row(packed: &[u8], out: &mut [u16], width: usize) {
unsafe {
packed444_to_luma_u16_row::<1>(packed, out, width);
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn vyu444_to_rgb_or_rgba_row<const ALPHA: bool>(
packed: &[u8],
out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
debug_assert!(packed.len() >= width * 3, "packed row too short");
let bpp: usize = if ALPHA { 4 } else { 3 };
debug_assert!(out.len() >= width * bpp, "out row too short");
let coeffs = scalar::Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = scalar::range_params_n::<8, 8>(full_range);
let bias = scalar::chroma_bias::<8>();
const RND: i32 = 1 << 14;
unsafe {
let rnd_v = vdupq_n_s32(RND);
let y_off_v = vdupq_n_s16(y_off as i16);
let y_scale_v = vdupq_n_s32(y_scale);
let c_scale_v = vdupq_n_s32(c_scale);
let bias_v = vdupq_n_s16(bias as i16);
let cru = vdupq_n_s32(coeffs.r_u());
let crv = vdupq_n_s32(coeffs.r_v());
let cgu = vdupq_n_s32(coeffs.g_u());
let cgv = vdupq_n_s32(coeffs.g_v());
let cbu = vdupq_n_s32(coeffs.b_u());
let cbv = vdupq_n_s32(coeffs.b_v());
let mut x = 0usize;
while x + 16 <= width {
let q = vld3q_u8(packed.as_ptr().add(x * 3));
let v_raw = q.0;
let y_raw = q.1;
let u_raw = q.2;
let (r_u8, g_u8, b_u8) = vuya_compute_rgb16(
v_raw, u_raw, y_raw, y_off_v, y_scale_v, c_scale_v, bias_v, rnd_v, cru, crv, cgu, cgv, cbu,
cbv,
);
let off = x * bpp;
if ALPHA {
vst4q_u8(
out.as_mut_ptr().add(off),
uint8x16x4_t(r_u8, g_u8, b_u8, vdupq_n_u8(0xFFu8)),
);
} else {
vst3q_u8(out.as_mut_ptr().add(off), uint8x16x3_t(r_u8, g_u8, b_u8));
}
x += 16;
}
let processed = x;
if processed < width {
let tail_packed = &packed[processed * 3..];
let tail_out = &mut out[processed * bpp..];
scalar::vyu444_to_rgb_or_rgba_row::<ALPHA>(
tail_packed,
tail_out,
width - processed,
matrix,
full_range,
);
}
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn vyu444_to_rgb_row(
packed: &[u8],
rgb_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
vyu444_to_rgb_or_rgba_row::<false>(packed, rgb_out, width, matrix, full_range);
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn vyu444_to_rgba_row(
packed: &[u8],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
vyu444_to_rgb_or_rgba_row::<true>(packed, rgba_out, width, matrix, full_range);
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn vyu444_to_luma_row(packed: &[u8], luma_out: &mut [u8], width: usize) {
debug_assert!(packed.len() >= width * 3, "packed row too short");
debug_assert!(luma_out.len() >= width, "luma row too short");
unsafe {
let mut x = 0usize;
while x + 16 <= width {
let q = vld3q_u8(packed.as_ptr().add(x * 3));
vst1q_u8(luma_out.as_mut_ptr().add(x), q.1);
x += 16;
}
if x < width {
scalar::vyu444_to_luma_row(&packed[x * 3..], &mut luma_out[x..], width - x);
}
}
}
#[cfg_attr(not(any(feature = "std", feature = "alloc")), allow(dead_code))]
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn vyu444_to_luma_u16_row(packed: &[u8], out: &mut [u16], width: usize) {
debug_assert!(packed.len() >= width * 3, "packed row too short");
debug_assert!(out.len() >= width, "out too short");
unsafe {
let mut x = 0usize;
while x + 16 <= width {
let q = vld3q_u8(packed.as_ptr().add(x * 3));
let y_lo = vmovl_u8(vget_low_u8(q.1));
let y_hi = vmovl_u8(vget_high_u8(q.1));
vst1q_u16(out.as_mut_ptr().add(x), y_lo);
vst1q_u16(out.as_mut_ptr().add(x + 8), y_hi);
x += 16;
}
if x < width {
scalar::vyu444_to_luma_u16_row(&packed[x * 3..], &mut out[x..], width - x);
}
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn vyu444_to_hsv_row(
packed: &[u8],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
debug_assert!(packed.len() >= width * 3, "packed row too short");
debug_assert!(h_out.len() >= width, "h_out row too short");
debug_assert!(s_out.len() >= width, "s_out row too short");
debug_assert!(v_out.len() >= width, "v_out row too short");
unsafe {
vuya_hsv_via_rgb_chunks(h_out, s_out, v_out, width, |offset, n, rgb| {
vyu444_to_rgb_row(&packed[offset * 3..], rgb, n, matrix, full_range);
});
}
}