use super::*;
use crate::{KernelMatrix, row::scalar};
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn xv48_to_rgb_or_rgba_row<const ALPHA: bool, const BE: bool>(
packed: &[u16],
out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
debug_assert!(packed.len() >= width * 4, "packed row too short");
let bpp: usize = if ALPHA { 4 } else { 3 };
debug_assert!(out.len() >= width * bpp, "out row too short");
let coeffs = scalar::Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = scalar::range_params_n::<16, 8>(full_range);
let bias = scalar::chroma_bias::<16>();
const RND: i32 = 1 << 14;
unsafe {
let rnd_v = vdupq_n_s32(RND);
let y_off_v = vdupq_n_s32(y_off);
let y_scale_v = vdupq_n_s32(y_scale);
let c_scale_v = vdupq_n_s32(c_scale);
let bias_v = vdupq_n_s16(bias as i16);
let cru = vdupq_n_s32(coeffs.r_u());
let crv = vdupq_n_s32(coeffs.r_v());
let cgu = vdupq_n_s32(coeffs.g_u());
let cgv = vdupq_n_s32(coeffs.g_v());
let cbu = vdupq_n_s32(coeffs.b_u());
let cbv = vdupq_n_s32(coeffs.b_v());
let mut x = 0usize;
while x + 8 <= width {
let q = vld4q_u16(packed.as_ptr().add(x * 4));
let u_u16 = bswap_u16x8_if_be::<BE>(q.0);
let y_u16 = bswap_u16x8_if_be::<BE>(q.1);
let v_u16 = bswap_u16x8_if_be::<BE>(q.2);
let u_i16 = vreinterpretq_s16_u16(u_u16);
let v_i16 = vreinterpretq_s16_u16(v_u16);
let u_sub = vsubq_s16(u_i16, bias_v);
let v_sub = vsubq_s16(v_i16, bias_v);
let u_lo_i32 = vmovl_s16(vget_low_s16(u_sub));
let u_hi_i32 = vmovl_s16(vget_high_s16(u_sub));
let v_lo_i32 = vmovl_s16(vget_low_s16(v_sub));
let v_hi_i32 = vmovl_s16(vget_high_s16(v_sub));
let u_d_lo = q15_shift(vaddq_s32(vmulq_s32(u_lo_i32, c_scale_v), rnd_v));
let u_d_hi = q15_shift(vaddq_s32(vmulq_s32(u_hi_i32, c_scale_v), rnd_v));
let v_d_lo = q15_shift(vaddq_s32(vmulq_s32(v_lo_i32, c_scale_v), rnd_v));
let v_d_hi = q15_shift(vaddq_s32(vmulq_s32(v_hi_i32, c_scale_v), rnd_v));
let r_chroma = chroma_i16x8(cru, crv, u_d_lo, v_d_lo, u_d_hi, v_d_hi, rnd_v);
let g_chroma = chroma_i16x8(cgu, cgv, u_d_lo, v_d_lo, u_d_hi, v_d_hi, rnd_v);
let b_chroma = chroma_i16x8(cbu, cbv, u_d_lo, v_d_lo, u_d_hi, v_d_hi, rnd_v);
let y_scaled = scale_y_u16_to_i16(y_u16, y_off_v, y_scale_v, rnd_v);
let r_u8 = vqmovun_s16_compat(vqaddq_s16(y_scaled, r_chroma));
let g_u8 = vqmovun_s16_compat(vqaddq_s16(y_scaled, g_chroma));
let b_u8 = vqmovun_s16_compat(vqaddq_s16(y_scaled, b_chroma));
let off = x * bpp;
if ALPHA {
let alpha = vdup_n_u8(0xFF);
vst4_u8(
out.as_mut_ptr().add(off),
uint8x8x4_t(r_u8, g_u8, b_u8, alpha),
);
} else {
vst3_u8(out.as_mut_ptr().add(off), uint8x8x3_t(r_u8, g_u8, b_u8));
}
x += 8;
}
if x < width {
let tail_packed = &packed[x * 4..width * 4];
let tail_out = &mut out[x * bpp..width * bpp];
let tail_w = width - x;
scalar::xv48_to_rgb_or_rgba_row::<ALPHA, BE>(
tail_packed,
tail_out,
tail_w,
matrix,
full_range,
);
}
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn xv48_to_rgb_u16_or_rgba_u16_row<const ALPHA: bool, const BE: bool>(
packed: &[u16],
out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
debug_assert!(packed.len() >= width * 4, "packed row too short");
let bpp: usize = if ALPHA { 4 } else { 3 };
debug_assert!(out.len() >= width * bpp, "out row too short");
let coeffs = scalar::Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = scalar::range_params_n::<16, 16>(full_range);
let bias = scalar::chroma_bias::<16>();
const RND: i32 = 1 << 14;
unsafe {
let alpha_u16 = vdupq_n_u16(0xFFFF);
let rnd_v = vdupq_n_s32(RND);
let rnd64 = vdupq_n_s64(RND as i64);
let y_off_v = vdupq_n_s32(y_off);
let y_scale_d = vdup_n_s32(y_scale); let c_scale_v = vdupq_n_s32(c_scale);
let bias_v = vdupq_n_s32(bias);
let cru = vdupq_n_s32(coeffs.r_u());
let crv = vdupq_n_s32(coeffs.r_v());
let cgu = vdupq_n_s32(coeffs.g_u());
let cgv = vdupq_n_s32(coeffs.g_v());
let cbu = vdupq_n_s32(coeffs.b_u());
let cbv = vdupq_n_s32(coeffs.b_v());
let mut x = 0usize;
while x + 16 <= width {
let q_lo = vld4q_u16(packed.as_ptr().add(x * 4));
let q_hi = vld4q_u16(packed.as_ptr().add(x * 4 + 32));
let y_lo_u16 = bswap_u16x8_if_be::<BE>(q_lo.1);
let u_lo_u16 = bswap_u16x8_if_be::<BE>(q_lo.0);
let v_lo_u16 = bswap_u16x8_if_be::<BE>(q_lo.2);
let y_hi_u16 = bswap_u16x8_if_be::<BE>(q_hi.1);
let u_hi_u16 = bswap_u16x8_if_be::<BE>(q_hi.0);
let v_hi_u16 = bswap_u16x8_if_be::<BE>(q_hi.2);
let u_lo0_i32 = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(u_lo_u16)));
let u_lo1_i32 = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(u_lo_u16)));
let v_lo0_i32 = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(v_lo_u16)));
let v_lo1_i32 = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(v_lo_u16)));
let u_d_lo0 = q15_shift(vaddq_s32(
vmulq_s32(vsubq_s32(u_lo0_i32, bias_v), c_scale_v),
rnd_v,
));
let u_d_lo1 = q15_shift(vaddq_s32(
vmulq_s32(vsubq_s32(u_lo1_i32, bias_v), c_scale_v),
rnd_v,
));
let v_d_lo0 = q15_shift(vaddq_s32(
vmulq_s32(vsubq_s32(v_lo0_i32, bias_v), c_scale_v),
rnd_v,
));
let v_d_lo1 = q15_shift(vaddq_s32(
vmulq_s32(vsubq_s32(v_lo1_i32, bias_v), c_scale_v),
rnd_v,
));
let u_hi0_i32 = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(u_hi_u16)));
let u_hi1_i32 = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(u_hi_u16)));
let v_hi0_i32 = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(v_hi_u16)));
let v_hi1_i32 = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(v_hi_u16)));
let u_d_hi0 = q15_shift(vaddq_s32(
vmulq_s32(vsubq_s32(u_hi0_i32, bias_v), c_scale_v),
rnd_v,
));
let u_d_hi1 = q15_shift(vaddq_s32(
vmulq_s32(vsubq_s32(u_hi1_i32, bias_v), c_scale_v),
rnd_v,
));
let v_d_hi0 = q15_shift(vaddq_s32(
vmulq_s32(vsubq_s32(v_hi0_i32, bias_v), c_scale_v),
rnd_v,
));
let v_d_hi1 = q15_shift(vaddq_s32(
vmulq_s32(vsubq_s32(v_hi1_i32, bias_v), c_scale_v),
rnd_v,
));
let r_ch_lo0 = chroma_i64x4(cru, crv, u_d_lo0, v_d_lo0, rnd64);
let r_ch_lo1 = chroma_i64x4(cru, crv, u_d_lo1, v_d_lo1, rnd64);
let g_ch_lo0 = chroma_i64x4(cgu, cgv, u_d_lo0, v_d_lo0, rnd64);
let g_ch_lo1 = chroma_i64x4(cgu, cgv, u_d_lo1, v_d_lo1, rnd64);
let b_ch_lo0 = chroma_i64x4(cbu, cbv, u_d_lo0, v_d_lo0, rnd64);
let b_ch_lo1 = chroma_i64x4(cbu, cbv, u_d_lo1, v_d_lo1, rnd64);
let r_ch_hi0 = chroma_i64x4(cru, crv, u_d_hi0, v_d_hi0, rnd64);
let r_ch_hi1 = chroma_i64x4(cru, crv, u_d_hi1, v_d_hi1, rnd64);
let g_ch_hi0 = chroma_i64x4(cgu, cgv, u_d_hi0, v_d_hi0, rnd64);
let g_ch_hi1 = chroma_i64x4(cgu, cgv, u_d_hi1, v_d_hi1, rnd64);
let b_ch_hi0 = chroma_i64x4(cbu, cbv, u_d_hi0, v_d_hi0, rnd64);
let b_ch_hi1 = chroma_i64x4(cbu, cbv, u_d_hi1, v_d_hi1, rnd64);
let y_lo_0 = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(y_lo_u16)));
let y_lo_1 = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(y_lo_u16)));
let y_hi_0 = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(y_hi_u16)));
let y_hi_1 = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(y_hi_u16)));
let ys_lo_0 = scale_y_u16_i64(y_lo_0, y_off_v, y_scale_d, rnd64);
let ys_lo_1 = scale_y_u16_i64(y_lo_1, y_off_v, y_scale_d, rnd64);
let ys_hi_0 = scale_y_u16_i64(y_hi_0, y_off_v, y_scale_d, rnd64);
let ys_hi_1 = scale_y_u16_i64(y_hi_1, y_off_v, y_scale_d, rnd64);
let r_lo_u16 = vcombine_u16(
vqmovun_s32_compat(vaddq_s32(ys_lo_0, r_ch_lo0)),
vqmovun_s32_compat(vaddq_s32(ys_lo_1, r_ch_lo1)),
);
let g_lo_u16 = vcombine_u16(
vqmovun_s32_compat(vaddq_s32(ys_lo_0, g_ch_lo0)),
vqmovun_s32_compat(vaddq_s32(ys_lo_1, g_ch_lo1)),
);
let b_lo_u16 = vcombine_u16(
vqmovun_s32_compat(vaddq_s32(ys_lo_0, b_ch_lo0)),
vqmovun_s32_compat(vaddq_s32(ys_lo_1, b_ch_lo1)),
);
let r_hi_u16 = vcombine_u16(
vqmovun_s32_compat(vaddq_s32(ys_hi_0, r_ch_hi0)),
vqmovun_s32_compat(vaddq_s32(ys_hi_1, r_ch_hi1)),
);
let g_hi_u16 = vcombine_u16(
vqmovun_s32_compat(vaddq_s32(ys_hi_0, g_ch_hi0)),
vqmovun_s32_compat(vaddq_s32(ys_hi_1, g_ch_hi1)),
);
let b_hi_u16 = vcombine_u16(
vqmovun_s32_compat(vaddq_s32(ys_hi_0, b_ch_hi0)),
vqmovun_s32_compat(vaddq_s32(ys_hi_1, b_ch_hi1)),
);
if ALPHA {
vst4q_u16(
out.as_mut_ptr().add(x * 4),
uint16x8x4_t(r_lo_u16, g_lo_u16, b_lo_u16, alpha_u16),
);
vst4q_u16(
out.as_mut_ptr().add(x * 4 + 32),
uint16x8x4_t(r_hi_u16, g_hi_u16, b_hi_u16, alpha_u16),
);
} else {
vst3q_u16(
out.as_mut_ptr().add(x * 3),
uint16x8x3_t(r_lo_u16, g_lo_u16, b_lo_u16),
);
vst3q_u16(
out.as_mut_ptr().add(x * 3 + 24),
uint16x8x3_t(r_hi_u16, g_hi_u16, b_hi_u16),
);
}
x += 16;
}
if x < width {
let tail_packed = &packed[x * 4..width * 4];
let tail_out = &mut out[x * bpp..width * bpp];
let tail_w = width - x;
scalar::xv48_to_rgb_u16_or_rgba_u16_row::<ALPHA, BE>(
tail_packed,
tail_out,
tail_w,
matrix,
full_range,
);
}
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn xv48_to_luma_row<const BE: bool>(
packed: &[u16],
out: &mut [u8],
width: usize,
) {
debug_assert!(packed.len() >= width * 4);
debug_assert!(out.len() >= width);
unsafe {
let mut x = 0usize;
while x + 8 <= width {
let q = vld4q_u16(packed.as_ptr().add(x * 4));
let y_raw = bswap_u16x8_if_be::<BE>(q.1);
let y_u8 = vshrn_n_u16::<8>(y_raw);
vst1_u8(out.as_mut_ptr().add(x), y_u8);
x += 8;
}
if x < width {
scalar::xv48_to_luma_row::<BE>(&packed[x * 4..width * 4], &mut out[x..width], width - x);
}
}
}
#[inline]
#[target_feature(enable = "neon")]
pub(crate) unsafe fn xv48_to_luma_u16_row<const BE: bool>(
packed: &[u16],
out: &mut [u16],
width: usize,
) {
debug_assert!(packed.len() >= width * 4);
debug_assert!(out.len() >= width);
unsafe {
let mut x = 0usize;
while x + 8 <= width {
let q = vld4q_u16(packed.as_ptr().add(x * 4));
let y_u16 = bswap_u16x8_if_be::<BE>(q.1);
vst1q_u16(out.as_mut_ptr().add(x), y_u16);
x += 8;
}
if x < width {
scalar::xv48_to_luma_u16_row::<BE>(&packed[x * 4..width * 4], &mut out[x..width], width - x);
}
}
}
const HSV_CHUNK: usize = 64;
#[inline]
unsafe fn xv48_hsv_via_rgb_chunks(
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
mut fill_rgb: impl FnMut(usize, usize, &mut [u8]),
) {
let mut scratch = [0u8; HSV_CHUNK * 3];
let mut offset = 0;
while offset < width {
let n = (width - offset).min(HSV_CHUNK);
fill_rgb(offset, n, &mut scratch[..n * 3]);
unsafe {
rgb_to_hsv_row(
&scratch[..n * 3],
&mut h_out[offset..offset + n],
&mut s_out[offset..offset + n],
&mut v_out[offset..offset + n],
n,
);
}
offset += n;
}
}
#[inline]
#[target_feature(enable = "neon")]
#[allow(clippy::too_many_arguments)]
pub(crate) unsafe fn xv48_to_hsv_row<const BE: bool>(
packed: &[u16],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
debug_assert!(packed.len() >= width * 4, "packed row too short");
debug_assert!(h_out.len() >= width, "h_out row too short");
debug_assert!(s_out.len() >= width, "s_out row too short");
debug_assert!(v_out.len() >= width, "v_out row too short");
unsafe {
xv48_hsv_via_rgb_chunks(h_out, s_out, v_out, width, |offset, n, rgb| {
xv48_to_rgb_or_rgba_row::<false, BE>(&packed[offset * 4..], rgb, n, matrix, full_range);
});
}
}