use core::arch::wasm32::*;
use super::*;
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn yuv_420p_n_to_rgb_row<const BITS: u32, const BE: bool>(
y: &[u16],
u_half: &[u16],
v_half: &[u16],
rgb_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
yuv_420p_n_to_rgb_or_rgba_row::<BITS, false, false, BE>(
y, u_half, v_half, None, rgb_out, width, matrix, full_range,
);
}
}
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn yuv_420p_n_to_rgba_row<const BITS: u32, const BE: bool>(
y: &[u16],
u_half: &[u16],
v_half: &[u16],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
yuv_420p_n_to_rgb_or_rgba_row::<BITS, true, false, BE>(
y, u_half, v_half, None, rgba_out, width, matrix, full_range,
);
}
}
#[cfg(feature = "yuva")]
#[inline]
#[target_feature(enable = "simd128")]
#[allow(clippy::too_many_arguments)]
pub(crate) unsafe fn yuv_420p_n_to_rgba_with_alpha_src_row<const BITS: u32, const BE: bool>(
y: &[u16],
u_half: &[u16],
v_half: &[u16],
a_src: &[u16],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
yuv_420p_n_to_rgb_or_rgba_row::<BITS, true, true, BE>(
y,
u_half,
v_half,
Some(a_src),
rgba_out,
width,
matrix,
full_range,
);
}
}
#[inline]
#[target_feature(enable = "simd128")]
#[allow(clippy::too_many_arguments)]
pub(crate) unsafe fn yuv_420p_n_to_rgb_or_rgba_row<
const BITS: u32,
const ALPHA: bool,
const ALPHA_SRC: bool,
const BE: bool,
>(
y: &[u16],
u_half: &[u16],
v_half: &[u16],
a_src: Option<&[u16]>,
out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
const { assert!(BITS == 9 || BITS == 10 || BITS == 12 || BITS == 14) };
const { assert!(!ALPHA_SRC || ALPHA) };
let bpp: usize = if ALPHA { 4 } else { 3 };
debug_assert_eq!(width & 1, 0);
debug_assert!(y.len() >= width);
debug_assert!(u_half.len() >= width / 2);
debug_assert!(v_half.len() >= width / 2);
debug_assert!(out.len() >= width * bpp);
if ALPHA_SRC {
debug_assert!(a_src.as_ref().is_some_and(|s| s.len() >= width));
}
let coeffs = scalar::Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = scalar::range_params_n::<BITS, 8>(full_range);
let bias = scalar::chroma_bias::<BITS>();
const RND: i32 = 1 << 14;
unsafe {
let rnd_v = i32x4_splat(RND);
let y_off_v = i16x8_splat(y_off as i16);
let y_scale_v = i32x4_splat(y_scale);
let c_scale_v = i32x4_splat(c_scale);
let bias_v = i16x8_splat(bias as i16);
let mask_v = u16x8_splat(scalar::bits_mask::<BITS>());
let cru = i32x4_splat(coeffs.r_u());
let crv = i32x4_splat(coeffs.r_v());
let cgu = i32x4_splat(coeffs.g_u());
let cgv = i32x4_splat(coeffs.g_v());
let cbu = i32x4_splat(coeffs.b_u());
let cbv = i32x4_splat(coeffs.b_v());
let alpha_u8 = u8x16_splat(0xFF);
let mut x = 0usize;
while x + 16 <= width {
let y_low_i16 = v128_and(
endian::load_endian_u16x8::<BE>(y.as_ptr().add(x) as *const u8),
mask_v,
);
let y_high_i16 = v128_and(
endian::load_endian_u16x8::<BE>(y.as_ptr().add(x + 8) as *const u8),
mask_v,
);
let u_vec = v128_and(
endian::load_endian_u16x8::<BE>(u_half.as_ptr().add(x / 2) as *const u8),
mask_v,
);
let v_vec = v128_and(
endian::load_endian_u16x8::<BE>(v_half.as_ptr().add(x / 2) as *const u8),
mask_v,
);
let u_i16 = i16x8_sub(u_vec, bias_v);
let v_i16 = i16x8_sub(v_vec, bias_v);
let u_lo_i32 = i32x4_extend_low_i16x8(u_i16);
let u_hi_i32 = i32x4_extend_high_i16x8(u_i16);
let v_lo_i32 = i32x4_extend_low_i16x8(v_i16);
let v_hi_i32 = i32x4_extend_high_i16x8(v_i16);
let u_d_lo = q15_shift(i32x4_add(i32x4_mul(u_lo_i32, c_scale_v), rnd_v));
let u_d_hi = q15_shift(i32x4_add(i32x4_mul(u_hi_i32, c_scale_v), rnd_v));
let v_d_lo = q15_shift(i32x4_add(i32x4_mul(v_lo_i32, c_scale_v), rnd_v));
let v_d_hi = q15_shift(i32x4_add(i32x4_mul(v_hi_i32, c_scale_v), rnd_v));
let r_chroma = chroma_i16x8(cru, crv, u_d_lo, v_d_lo, u_d_hi, v_d_hi, rnd_v);
let g_chroma = chroma_i16x8(cgu, cgv, u_d_lo, v_d_lo, u_d_hi, v_d_hi, rnd_v);
let b_chroma = chroma_i16x8(cbu, cbv, u_d_lo, v_d_lo, u_d_hi, v_d_hi, rnd_v);
let r_dup_lo = dup_lo(r_chroma);
let r_dup_hi = dup_hi(r_chroma);
let g_dup_lo = dup_lo(g_chroma);
let g_dup_hi = dup_hi(g_chroma);
let b_dup_lo = dup_lo(b_chroma);
let b_dup_hi = dup_hi(b_chroma);
let y_scaled_lo = scale_y(y_low_i16, y_off_v, y_scale_v, rnd_v);
let y_scaled_hi = scale_y(y_high_i16, y_off_v, y_scale_v, rnd_v);
let b_lo = i16x8_add_sat(y_scaled_lo, b_dup_lo);
let b_hi = i16x8_add_sat(y_scaled_hi, b_dup_hi);
let g_lo = i16x8_add_sat(y_scaled_lo, g_dup_lo);
let g_hi = i16x8_add_sat(y_scaled_hi, g_dup_hi);
let r_lo = i16x8_add_sat(y_scaled_lo, r_dup_lo);
let r_hi = i16x8_add_sat(y_scaled_hi, r_dup_hi);
let b_u8 = u8x16_narrow_i16x8(b_lo, b_hi);
let g_u8 = u8x16_narrow_i16x8(g_lo, g_hi);
let r_u8 = u8x16_narrow_i16x8(r_lo, r_hi);
if ALPHA {
let a_u8 = if ALPHA_SRC {
let a_ptr = a_src.as_ref().unwrap_unchecked().as_ptr();
let a_lo = v128_and(
endian::load_endian_u16x8::<BE>(a_ptr.add(x) as *const u8),
mask_v,
);
let a_hi = v128_and(
endian::load_endian_u16x8::<BE>(a_ptr.add(x + 8) as *const u8),
mask_v,
);
let a_lo_shifted = u16x8_shr(a_lo, BITS - 8);
let a_hi_shifted = u16x8_shr(a_hi, BITS - 8);
u8x16_narrow_i16x8(a_lo_shifted, a_hi_shifted)
} else {
alpha_u8
};
write_rgba_16(r_u8, g_u8, b_u8, a_u8, out.as_mut_ptr().add(x * 4));
} else {
write_rgb_16(r_u8, g_u8, b_u8, out.as_mut_ptr().add(x * 3));
}
x += 16;
}
if x < width {
let tail_y = &y[x..width];
let tail_u = &u_half[x / 2..width / 2];
let tail_v = &v_half[x / 2..width / 2];
let tail_out = &mut out[x * bpp..width * bpp];
let tail_w = width - x;
if ALPHA_SRC {
let tail_a = &a_src.as_ref().unwrap_unchecked()[x..width];
scalar::yuv_420p_n_to_rgba_with_alpha_src_row::<BITS, BE>(
tail_y, tail_u, tail_v, tail_a, tail_out, tail_w, matrix, full_range,
);
} else if ALPHA {
scalar::yuv_420p_n_to_rgba_row::<BITS, BE>(
tail_y, tail_u, tail_v, tail_out, tail_w, matrix, full_range,
);
} else {
scalar::yuv_420p_n_to_rgb_row::<BITS, BE>(
tail_y, tail_u, tail_v, tail_out, tail_w, matrix, full_range,
);
}
}
}
}
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn yuv_420p_n_to_rgb_u16_row<const BITS: u32, const BE: bool>(
y: &[u16],
u_half: &[u16],
v_half: &[u16],
rgb_out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
yuv_420p_n_to_rgb_or_rgba_u16_row::<BITS, false, false, BE>(
y, u_half, v_half, None, rgb_out, width, matrix, full_range,
);
}
}
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn yuv_420p_n_to_rgba_u16_row<const BITS: u32, const BE: bool>(
y: &[u16],
u_half: &[u16],
v_half: &[u16],
rgba_out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
yuv_420p_n_to_rgb_or_rgba_u16_row::<BITS, true, false, BE>(
y, u_half, v_half, None, rgba_out, width, matrix, full_range,
);
}
}
#[cfg(feature = "yuva")]
#[inline]
#[target_feature(enable = "simd128")]
#[allow(clippy::too_many_arguments)]
pub(crate) unsafe fn yuv_420p_n_to_rgba_u16_with_alpha_src_row<const BITS: u32, const BE: bool>(
y: &[u16],
u_half: &[u16],
v_half: &[u16],
a_src: &[u16],
rgba_out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
yuv_420p_n_to_rgb_or_rgba_u16_row::<BITS, true, true, BE>(
y,
u_half,
v_half,
Some(a_src),
rgba_out,
width,
matrix,
full_range,
);
}
}
#[inline]
#[target_feature(enable = "simd128")]
#[allow(clippy::too_many_arguments)]
pub(crate) unsafe fn yuv_420p_n_to_rgb_or_rgba_u16_row<
const BITS: u32,
const ALPHA: bool,
const ALPHA_SRC: bool,
const BE: bool,
>(
y: &[u16],
u_half: &[u16],
v_half: &[u16],
a_src: Option<&[u16]>,
out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
const { assert!(BITS == 9 || BITS == 10 || BITS == 12 || BITS == 14) };
const { assert!(!ALPHA_SRC || ALPHA) };
let bpp: usize = if ALPHA { 4 } else { 3 };
debug_assert_eq!(width & 1, 0);
debug_assert!(y.len() >= width);
debug_assert!(u_half.len() >= width / 2);
debug_assert!(v_half.len() >= width / 2);
debug_assert!(out.len() >= width * bpp);
if ALPHA_SRC {
debug_assert!(a_src.as_ref().is_some_and(|s| s.len() >= width));
}
let coeffs = scalar::Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = scalar::range_params_n::<BITS, BITS>(full_range);
let bias = scalar::chroma_bias::<BITS>();
const RND: i32 = 1 << 14;
let out_max: i16 = ((1i32 << BITS) - 1) as i16;
unsafe {
let rnd_v = i32x4_splat(RND);
let y_off_v = i16x8_splat(y_off as i16);
let y_scale_v = i32x4_splat(y_scale);
let c_scale_v = i32x4_splat(c_scale);
let bias_v = i16x8_splat(bias as i16);
let mask_v = u16x8_splat(scalar::bits_mask::<BITS>());
let max_v = i16x8_splat(out_max);
let zero_v = i16x8_splat(0);
let cru = i32x4_splat(coeffs.r_u());
let crv = i32x4_splat(coeffs.r_v());
let cgu = i32x4_splat(coeffs.g_u());
let cgv = i32x4_splat(coeffs.g_v());
let cbu = i32x4_splat(coeffs.b_u());
let cbv = i32x4_splat(coeffs.b_v());
let alpha_u16 = u16x8_splat(out_max as u16);
let mut x = 0usize;
while x + 16 <= width {
let y_low_i16 = v128_and(
endian::load_endian_u16x8::<BE>(y.as_ptr().add(x) as *const u8),
mask_v,
);
let y_high_i16 = v128_and(
endian::load_endian_u16x8::<BE>(y.as_ptr().add(x + 8) as *const u8),
mask_v,
);
let u_vec = v128_and(
endian::load_endian_u16x8::<BE>(u_half.as_ptr().add(x / 2) as *const u8),
mask_v,
);
let v_vec = v128_and(
endian::load_endian_u16x8::<BE>(v_half.as_ptr().add(x / 2) as *const u8),
mask_v,
);
let u_i16 = i16x8_sub(u_vec, bias_v);
let v_i16 = i16x8_sub(v_vec, bias_v);
let u_lo_i32 = i32x4_extend_low_i16x8(u_i16);
let u_hi_i32 = i32x4_extend_high_i16x8(u_i16);
let v_lo_i32 = i32x4_extend_low_i16x8(v_i16);
let v_hi_i32 = i32x4_extend_high_i16x8(v_i16);
let u_d_lo = q15_shift(i32x4_add(i32x4_mul(u_lo_i32, c_scale_v), rnd_v));
let u_d_hi = q15_shift(i32x4_add(i32x4_mul(u_hi_i32, c_scale_v), rnd_v));
let v_d_lo = q15_shift(i32x4_add(i32x4_mul(v_lo_i32, c_scale_v), rnd_v));
let v_d_hi = q15_shift(i32x4_add(i32x4_mul(v_hi_i32, c_scale_v), rnd_v));
let r_chroma = chroma_i16x8(cru, crv, u_d_lo, v_d_lo, u_d_hi, v_d_hi, rnd_v);
let g_chroma = chroma_i16x8(cgu, cgv, u_d_lo, v_d_lo, u_d_hi, v_d_hi, rnd_v);
let b_chroma = chroma_i16x8(cbu, cbv, u_d_lo, v_d_lo, u_d_hi, v_d_hi, rnd_v);
let r_dup_lo = dup_lo(r_chroma);
let r_dup_hi = dup_hi(r_chroma);
let g_dup_lo = dup_lo(g_chroma);
let g_dup_hi = dup_hi(g_chroma);
let b_dup_lo = dup_lo(b_chroma);
let b_dup_hi = dup_hi(b_chroma);
let y_scaled_lo = scale_y(y_low_i16, y_off_v, y_scale_v, rnd_v);
let y_scaled_hi = scale_y(y_high_i16, y_off_v, y_scale_v, rnd_v);
let r_lo = clamp_u16_max_wasm(i16x8_add_sat(y_scaled_lo, r_dup_lo), zero_v, max_v);
let r_hi = clamp_u16_max_wasm(i16x8_add_sat(y_scaled_hi, r_dup_hi), zero_v, max_v);
let g_lo = clamp_u16_max_wasm(i16x8_add_sat(y_scaled_lo, g_dup_lo), zero_v, max_v);
let g_hi = clamp_u16_max_wasm(i16x8_add_sat(y_scaled_hi, g_dup_hi), zero_v, max_v);
let b_lo = clamp_u16_max_wasm(i16x8_add_sat(y_scaled_lo, b_dup_lo), zero_v, max_v);
let b_hi = clamp_u16_max_wasm(i16x8_add_sat(y_scaled_hi, b_dup_hi), zero_v, max_v);
if ALPHA {
let (a_lo_v, a_hi_v) = if ALPHA_SRC {
let a_ptr = a_src.as_ref().unwrap_unchecked().as_ptr();
let lo = v128_and(
endian::load_endian_u16x8::<BE>(a_ptr.add(x) as *const u8),
mask_v,
);
let hi = v128_and(
endian::load_endian_u16x8::<BE>(a_ptr.add(x + 8) as *const u8),
mask_v,
);
(lo, hi)
} else {
(alpha_u16, alpha_u16)
};
let dst = out.as_mut_ptr().add(x * 4);
write_rgba_u16_8(r_lo, g_lo, b_lo, a_lo_v, dst);
write_rgba_u16_8(r_hi, g_hi, b_hi, a_hi_v, dst.add(32));
} else {
let dst = out.as_mut_ptr().add(x * 3);
write_rgb_u16_8(r_lo, g_lo, b_lo, dst);
write_rgb_u16_8(r_hi, g_hi, b_hi, dst.add(24));
}
x += 16;
}
if x < width {
let tail_y = &y[x..width];
let tail_u = &u_half[x / 2..width / 2];
let tail_v = &v_half[x / 2..width / 2];
let tail_out = &mut out[x * bpp..width * bpp];
let tail_w = width - x;
if ALPHA_SRC {
let tail_a = &a_src.as_ref().unwrap_unchecked()[x..width];
scalar::yuv_420p_n_to_rgba_u16_with_alpha_src_row::<BITS, BE>(
tail_y, tail_u, tail_v, tail_a, tail_out, tail_w, matrix, full_range,
);
} else if ALPHA {
scalar::yuv_420p_n_to_rgba_u16_row::<BITS, BE>(
tail_y, tail_u, tail_v, tail_out, tail_w, matrix, full_range,
);
} else {
scalar::yuv_420p_n_to_rgb_u16_row::<BITS, BE>(
tail_y, tail_u, tail_v, tail_out, tail_w, matrix, full_range,
);
}
}
}
}
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn yuv_444p_n_to_rgb_row<const BITS: u32, const BE: bool>(
y: &[u16],
u: &[u16],
v: &[u16],
rgb_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
yuv_444p_n_to_rgb_or_rgba_row::<BITS, false, false, BE, false>(
y, u, v, rgb_out, width, matrix, full_range, None,
);
}
}
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn yuv_444p_n_to_rgba_row<const BITS: u32, const BE: bool>(
y: &[u16],
u: &[u16],
v: &[u16],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
yuv_444p_n_to_rgb_or_rgba_row::<BITS, true, false, BE, false>(
y, u, v, rgba_out, width, matrix, full_range, None,
);
}
}
#[cfg(feature = "yuva")]
#[inline]
#[target_feature(enable = "simd128")]
#[allow(clippy::too_many_arguments)]
pub(crate) unsafe fn yuv_444p_n_to_rgba_with_alpha_src_row<const BITS: u32, const BE: bool>(
y: &[u16],
u: &[u16],
v: &[u16],
a_src: &[u16],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
yuv_444p_n_to_rgb_or_rgba_row::<BITS, true, true, BE, false>(
y,
u,
v,
rgba_out,
width,
matrix,
full_range,
Some(a_src),
);
}
}
#[inline]
#[target_feature(enable = "simd128")]
#[allow(clippy::too_many_arguments)]
pub(crate) unsafe fn yuv_444p_n_to_rgb_or_rgba_row<
const BITS: u32,
const ALPHA: bool,
const ALPHA_SRC: bool,
const BE: bool,
const MSB: bool,
>(
y: &[u16],
u: &[u16],
v: &[u16],
out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
a_src: Option<&[u16]>,
) {
const { assert!(BITS == 9 || BITS == 10 || BITS == 12 || BITS == 14) };
const { assert!(!ALPHA_SRC || ALPHA) };
const { assert!(!MSB || !ALPHA_SRC) };
let bpp: usize = if ALPHA { 4 } else { 3 };
debug_assert!(y.len() >= width);
debug_assert!(u.len() >= width);
debug_assert!(v.len() >= width);
debug_assert!(out.len() >= width * bpp);
if ALPHA_SRC {
debug_assert!(a_src.as_ref().is_some_and(|s| s.len() >= width));
}
let coeffs = scalar::Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = scalar::range_params_n::<BITS, 8>(full_range);
let bias = scalar::chroma_bias::<BITS>();
const RND: i32 = 1 << 14;
unsafe {
let rnd_v = i32x4_splat(RND);
let y_off_v = i16x8_splat(y_off as i16);
let y_scale_v = i32x4_splat(y_scale);
let c_scale_v = i32x4_splat(c_scale);
let bias_v = i16x8_splat(bias as i16);
let mask_v = u16x8_splat(scalar::bits_mask::<BITS>());
let cru = i32x4_splat(coeffs.r_u());
let crv = i32x4_splat(coeffs.r_v());
let cgu = i32x4_splat(coeffs.g_u());
let cgv = i32x4_splat(coeffs.g_v());
let cbu = i32x4_splat(coeffs.b_u());
let cbv = i32x4_splat(coeffs.b_v());
let alpha_u8 = u8x16_splat(0xFF);
let mut x = 0usize;
while x + 16 <= width {
let y_low_i16 = extract_hb_simd::<BITS, MSB>(
endian::load_endian_u16x8::<BE>(y.as_ptr().add(x) as *const u8),
mask_v,
);
let y_high_i16 = extract_hb_simd::<BITS, MSB>(
endian::load_endian_u16x8::<BE>(y.as_ptr().add(x + 8) as *const u8),
mask_v,
);
let u_lo_vec = extract_hb_simd::<BITS, MSB>(
endian::load_endian_u16x8::<BE>(u.as_ptr().add(x) as *const u8),
mask_v,
);
let u_hi_vec = extract_hb_simd::<BITS, MSB>(
endian::load_endian_u16x8::<BE>(u.as_ptr().add(x + 8) as *const u8),
mask_v,
);
let v_lo_vec = extract_hb_simd::<BITS, MSB>(
endian::load_endian_u16x8::<BE>(v.as_ptr().add(x) as *const u8),
mask_v,
);
let v_hi_vec = extract_hb_simd::<BITS, MSB>(
endian::load_endian_u16x8::<BE>(v.as_ptr().add(x + 8) as *const u8),
mask_v,
);
let u_lo_i16 = i16x8_sub(u_lo_vec, bias_v);
let u_hi_i16 = i16x8_sub(u_hi_vec, bias_v);
let v_lo_i16 = i16x8_sub(v_lo_vec, bias_v);
let v_hi_i16 = i16x8_sub(v_hi_vec, bias_v);
let u_lo_a = i32x4_extend_low_i16x8(u_lo_i16);
let u_lo_b = i32x4_extend_high_i16x8(u_lo_i16);
let u_hi_a = i32x4_extend_low_i16x8(u_hi_i16);
let u_hi_b = i32x4_extend_high_i16x8(u_hi_i16);
let v_lo_a = i32x4_extend_low_i16x8(v_lo_i16);
let v_lo_b = i32x4_extend_high_i16x8(v_lo_i16);
let v_hi_a = i32x4_extend_low_i16x8(v_hi_i16);
let v_hi_b = i32x4_extend_high_i16x8(v_hi_i16);
let u_d_lo_a = q15_shift(i32x4_add(i32x4_mul(u_lo_a, c_scale_v), rnd_v));
let u_d_lo_b = q15_shift(i32x4_add(i32x4_mul(u_lo_b, c_scale_v), rnd_v));
let u_d_hi_a = q15_shift(i32x4_add(i32x4_mul(u_hi_a, c_scale_v), rnd_v));
let u_d_hi_b = q15_shift(i32x4_add(i32x4_mul(u_hi_b, c_scale_v), rnd_v));
let v_d_lo_a = q15_shift(i32x4_add(i32x4_mul(v_lo_a, c_scale_v), rnd_v));
let v_d_lo_b = q15_shift(i32x4_add(i32x4_mul(v_lo_b, c_scale_v), rnd_v));
let v_d_hi_a = q15_shift(i32x4_add(i32x4_mul(v_hi_a, c_scale_v), rnd_v));
let v_d_hi_b = q15_shift(i32x4_add(i32x4_mul(v_hi_b, c_scale_v), rnd_v));
let r_chroma_lo = chroma_i16x8(cru, crv, u_d_lo_a, v_d_lo_a, u_d_lo_b, v_d_lo_b, rnd_v);
let r_chroma_hi = chroma_i16x8(cru, crv, u_d_hi_a, v_d_hi_a, u_d_hi_b, v_d_hi_b, rnd_v);
let g_chroma_lo = chroma_i16x8(cgu, cgv, u_d_lo_a, v_d_lo_a, u_d_lo_b, v_d_lo_b, rnd_v);
let g_chroma_hi = chroma_i16x8(cgu, cgv, u_d_hi_a, v_d_hi_a, u_d_hi_b, v_d_hi_b, rnd_v);
let b_chroma_lo = chroma_i16x8(cbu, cbv, u_d_lo_a, v_d_lo_a, u_d_lo_b, v_d_lo_b, rnd_v);
let b_chroma_hi = chroma_i16x8(cbu, cbv, u_d_hi_a, v_d_hi_a, u_d_hi_b, v_d_hi_b, rnd_v);
let y_scaled_lo = scale_y(y_low_i16, y_off_v, y_scale_v, rnd_v);
let y_scaled_hi = scale_y(y_high_i16, y_off_v, y_scale_v, rnd_v);
let r_lo = i16x8_add_sat(y_scaled_lo, r_chroma_lo);
let r_hi = i16x8_add_sat(y_scaled_hi, r_chroma_hi);
let g_lo = i16x8_add_sat(y_scaled_lo, g_chroma_lo);
let g_hi = i16x8_add_sat(y_scaled_hi, g_chroma_hi);
let b_lo = i16x8_add_sat(y_scaled_lo, b_chroma_lo);
let b_hi = i16x8_add_sat(y_scaled_hi, b_chroma_hi);
let b_u8 = u8x16_narrow_i16x8(b_lo, b_hi);
let g_u8 = u8x16_narrow_i16x8(g_lo, g_hi);
let r_u8 = u8x16_narrow_i16x8(r_lo, r_hi);
if ALPHA {
let a_u8 = if ALPHA_SRC {
let a_ptr = a_src.as_ref().unwrap_unchecked().as_ptr();
let a_lo = v128_and(
endian::load_endian_u16x8::<BE>(a_ptr.add(x) as *const u8),
mask_v,
);
let a_hi = v128_and(
endian::load_endian_u16x8::<BE>(a_ptr.add(x + 8) as *const u8),
mask_v,
);
let a_lo_shifted = u16x8_shr(a_lo, BITS - 8);
let a_hi_shifted = u16x8_shr(a_hi, BITS - 8);
u8x16_narrow_i16x8(a_lo_shifted, a_hi_shifted)
} else {
alpha_u8
};
write_rgba_16(r_u8, g_u8, b_u8, a_u8, out.as_mut_ptr().add(x * 4));
} else {
write_rgb_16(r_u8, g_u8, b_u8, out.as_mut_ptr().add(x * 3));
}
x += 16;
}
if x < width {
let tail_y = &y[x..width];
let tail_u = &u[x..width];
let tail_v = &v[x..width];
let tail_out = &mut out[x * bpp..width * bpp];
let tail_w = width - x;
if ALPHA_SRC {
let tail_a = &a_src.as_ref().unwrap_unchecked()[x..width];
scalar::yuv_444p_n_to_rgba_with_alpha_src_row::<BITS, BE>(
tail_y, tail_u, tail_v, tail_a, tail_out, tail_w, matrix, full_range,
);
} else if ALPHA {
if MSB {
scalar::yuv_444p_n_msb_to_rgba_row::<BITS, BE>(
tail_y, tail_u, tail_v, tail_out, tail_w, matrix, full_range,
);
} else {
scalar::yuv_444p_n_to_rgba_row::<BITS, BE>(
tail_y, tail_u, tail_v, tail_out, tail_w, matrix, full_range,
);
}
} else if MSB {
scalar::yuv_444p_n_msb_to_rgb_row::<BITS, BE>(
tail_y, tail_u, tail_v, tail_out, tail_w, matrix, full_range,
);
} else {
scalar::yuv_444p_n_to_rgb_row::<BITS, BE>(
tail_y, tail_u, tail_v, tail_out, tail_w, matrix, full_range,
);
}
}
}
}
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn yuv_444p_n_to_rgb_u16_row<const BITS: u32, const BE: bool>(
y: &[u16],
u: &[u16],
v: &[u16],
rgb_out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
yuv_444p_n_to_rgb_or_rgba_u16_row::<BITS, false, false, BE, false>(
y, u, v, rgb_out, width, matrix, full_range, None,
);
}
}
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn yuv_444p_n_to_rgba_u16_row<const BITS: u32, const BE: bool>(
y: &[u16],
u: &[u16],
v: &[u16],
rgba_out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
yuv_444p_n_to_rgb_or_rgba_u16_row::<BITS, true, false, BE, false>(
y, u, v, rgba_out, width, matrix, full_range, None,
);
}
}
#[cfg(feature = "yuva")]
#[inline]
#[target_feature(enable = "simd128")]
#[allow(clippy::too_many_arguments)]
pub(crate) unsafe fn yuv_444p_n_to_rgba_u16_with_alpha_src_row<const BITS: u32, const BE: bool>(
y: &[u16],
u: &[u16],
v: &[u16],
a_src: &[u16],
rgba_out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
yuv_444p_n_to_rgb_or_rgba_u16_row::<BITS, true, true, BE, false>(
y,
u,
v,
rgba_out,
width,
matrix,
full_range,
Some(a_src),
);
}
}
#[inline]
#[target_feature(enable = "simd128")]
#[allow(clippy::too_many_arguments)]
pub(crate) unsafe fn yuv_444p_n_to_rgb_or_rgba_u16_row<
const BITS: u32,
const ALPHA: bool,
const ALPHA_SRC: bool,
const BE: bool,
const MSB: bool,
>(
y: &[u16],
u: &[u16],
v: &[u16],
out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
a_src: Option<&[u16]>,
) {
const { assert!(BITS == 9 || BITS == 10 || BITS == 12 || BITS == 14) };
const { assert!(!ALPHA_SRC || ALPHA) };
const { assert!(!MSB || !ALPHA_SRC) };
let bpp: usize = if ALPHA { 4 } else { 3 };
debug_assert!(y.len() >= width);
debug_assert!(u.len() >= width);
debug_assert!(v.len() >= width);
debug_assert!(out.len() >= width * bpp);
if ALPHA_SRC {
debug_assert!(a_src.as_ref().is_some_and(|s| s.len() >= width));
}
let coeffs = scalar::Coefficients::for_matrix(matrix);
let (y_off, y_scale, c_scale) = scalar::range_params_n::<BITS, BITS>(full_range);
let bias = scalar::chroma_bias::<BITS>();
const RND: i32 = 1 << 14;
let out_max: i16 = ((1i32 << BITS) - 1) as i16;
unsafe {
let rnd_v = i32x4_splat(RND);
let y_off_v = i16x8_splat(y_off as i16);
let y_scale_v = i32x4_splat(y_scale);
let c_scale_v = i32x4_splat(c_scale);
let bias_v = i16x8_splat(bias as i16);
let mask_v = u16x8_splat(scalar::bits_mask::<BITS>());
let max_v = i16x8_splat(out_max);
let zero_v = i16x8_splat(0);
let cru = i32x4_splat(coeffs.r_u());
let crv = i32x4_splat(coeffs.r_v());
let cgu = i32x4_splat(coeffs.g_u());
let cgv = i32x4_splat(coeffs.g_v());
let cbu = i32x4_splat(coeffs.b_u());
let cbv = i32x4_splat(coeffs.b_v());
let alpha_u16 = u16x8_splat(out_max as u16);
let mut x = 0usize;
while x + 16 <= width {
let y_low_i16 = extract_hb_simd::<BITS, MSB>(
endian::load_endian_u16x8::<BE>(y.as_ptr().add(x) as *const u8),
mask_v,
);
let y_high_i16 = extract_hb_simd::<BITS, MSB>(
endian::load_endian_u16x8::<BE>(y.as_ptr().add(x + 8) as *const u8),
mask_v,
);
let u_lo_vec = extract_hb_simd::<BITS, MSB>(
endian::load_endian_u16x8::<BE>(u.as_ptr().add(x) as *const u8),
mask_v,
);
let u_hi_vec = extract_hb_simd::<BITS, MSB>(
endian::load_endian_u16x8::<BE>(u.as_ptr().add(x + 8) as *const u8),
mask_v,
);
let v_lo_vec = extract_hb_simd::<BITS, MSB>(
endian::load_endian_u16x8::<BE>(v.as_ptr().add(x) as *const u8),
mask_v,
);
let v_hi_vec = extract_hb_simd::<BITS, MSB>(
endian::load_endian_u16x8::<BE>(v.as_ptr().add(x + 8) as *const u8),
mask_v,
);
let u_lo_i16 = i16x8_sub(u_lo_vec, bias_v);
let u_hi_i16 = i16x8_sub(u_hi_vec, bias_v);
let v_lo_i16 = i16x8_sub(v_lo_vec, bias_v);
let v_hi_i16 = i16x8_sub(v_hi_vec, bias_v);
let u_lo_a = i32x4_extend_low_i16x8(u_lo_i16);
let u_lo_b = i32x4_extend_high_i16x8(u_lo_i16);
let u_hi_a = i32x4_extend_low_i16x8(u_hi_i16);
let u_hi_b = i32x4_extend_high_i16x8(u_hi_i16);
let v_lo_a = i32x4_extend_low_i16x8(v_lo_i16);
let v_lo_b = i32x4_extend_high_i16x8(v_lo_i16);
let v_hi_a = i32x4_extend_low_i16x8(v_hi_i16);
let v_hi_b = i32x4_extend_high_i16x8(v_hi_i16);
let u_d_lo_a = q15_shift(i32x4_add(i32x4_mul(u_lo_a, c_scale_v), rnd_v));
let u_d_lo_b = q15_shift(i32x4_add(i32x4_mul(u_lo_b, c_scale_v), rnd_v));
let u_d_hi_a = q15_shift(i32x4_add(i32x4_mul(u_hi_a, c_scale_v), rnd_v));
let u_d_hi_b = q15_shift(i32x4_add(i32x4_mul(u_hi_b, c_scale_v), rnd_v));
let v_d_lo_a = q15_shift(i32x4_add(i32x4_mul(v_lo_a, c_scale_v), rnd_v));
let v_d_lo_b = q15_shift(i32x4_add(i32x4_mul(v_lo_b, c_scale_v), rnd_v));
let v_d_hi_a = q15_shift(i32x4_add(i32x4_mul(v_hi_a, c_scale_v), rnd_v));
let v_d_hi_b = q15_shift(i32x4_add(i32x4_mul(v_hi_b, c_scale_v), rnd_v));
let r_chroma_lo = chroma_i16x8(cru, crv, u_d_lo_a, v_d_lo_a, u_d_lo_b, v_d_lo_b, rnd_v);
let r_chroma_hi = chroma_i16x8(cru, crv, u_d_hi_a, v_d_hi_a, u_d_hi_b, v_d_hi_b, rnd_v);
let g_chroma_lo = chroma_i16x8(cgu, cgv, u_d_lo_a, v_d_lo_a, u_d_lo_b, v_d_lo_b, rnd_v);
let g_chroma_hi = chroma_i16x8(cgu, cgv, u_d_hi_a, v_d_hi_a, u_d_hi_b, v_d_hi_b, rnd_v);
let b_chroma_lo = chroma_i16x8(cbu, cbv, u_d_lo_a, v_d_lo_a, u_d_lo_b, v_d_lo_b, rnd_v);
let b_chroma_hi = chroma_i16x8(cbu, cbv, u_d_hi_a, v_d_hi_a, u_d_hi_b, v_d_hi_b, rnd_v);
let y_scaled_lo = scale_y(y_low_i16, y_off_v, y_scale_v, rnd_v);
let y_scaled_hi = scale_y(y_high_i16, y_off_v, y_scale_v, rnd_v);
let r_lo = clamp_u16_max_wasm(i16x8_add_sat(y_scaled_lo, r_chroma_lo), zero_v, max_v);
let r_hi = clamp_u16_max_wasm(i16x8_add_sat(y_scaled_hi, r_chroma_hi), zero_v, max_v);
let g_lo = clamp_u16_max_wasm(i16x8_add_sat(y_scaled_lo, g_chroma_lo), zero_v, max_v);
let g_hi = clamp_u16_max_wasm(i16x8_add_sat(y_scaled_hi, g_chroma_hi), zero_v, max_v);
let b_lo = clamp_u16_max_wasm(i16x8_add_sat(y_scaled_lo, b_chroma_lo), zero_v, max_v);
let b_hi = clamp_u16_max_wasm(i16x8_add_sat(y_scaled_hi, b_chroma_hi), zero_v, max_v);
if ALPHA {
let (a_lo_v, a_hi_v) = if ALPHA_SRC {
let a_ptr = a_src.as_ref().unwrap_unchecked().as_ptr();
let lo = v128_and(
endian::load_endian_u16x8::<BE>(a_ptr.add(x) as *const u8),
mask_v,
);
let hi = v128_and(
endian::load_endian_u16x8::<BE>(a_ptr.add(x + 8) as *const u8),
mask_v,
);
(lo, hi)
} else {
(alpha_u16, alpha_u16)
};
let dst = out.as_mut_ptr().add(x * 4);
write_rgba_u16_8(r_lo, g_lo, b_lo, a_lo_v, dst);
write_rgba_u16_8(r_hi, g_hi, b_hi, a_hi_v, dst.add(32));
} else {
let dst = out.as_mut_ptr().add(x * 3);
write_rgb_u16_8(r_lo, g_lo, b_lo, dst);
write_rgb_u16_8(r_hi, g_hi, b_hi, dst.add(24));
}
x += 16;
}
if x < width {
let tail_y = &y[x..width];
let tail_u = &u[x..width];
let tail_v = &v[x..width];
let tail_out = &mut out[x * bpp..width * bpp];
let tail_w = width - x;
if ALPHA_SRC {
let tail_a = &a_src.as_ref().unwrap_unchecked()[x..width];
scalar::yuv_444p_n_to_rgba_u16_with_alpha_src_row::<BITS, BE>(
tail_y, tail_u, tail_v, tail_a, tail_out, tail_w, matrix, full_range,
);
} else if ALPHA {
if MSB {
scalar::yuv_444p_n_msb_to_rgba_u16_row::<BITS, BE>(
tail_y, tail_u, tail_v, tail_out, tail_w, matrix, full_range,
);
} else {
scalar::yuv_444p_n_to_rgba_u16_row::<BITS, BE>(
tail_y, tail_u, tail_v, tail_out, tail_w, matrix, full_range,
);
}
} else if MSB {
scalar::yuv_444p_n_msb_to_rgb_u16_row::<BITS, BE>(
tail_y, tail_u, tail_v, tail_out, tail_w, matrix, full_range,
);
} else {
scalar::yuv_444p_n_to_rgb_u16_row::<BITS, BE>(
tail_y, tail_u, tail_v, tail_out, tail_w, matrix, full_range,
);
}
}
}
}
#[inline]
#[target_feature(enable = "simd128")]
#[allow(clippy::too_many_arguments)]
pub(crate) unsafe fn yuv_420p_n_to_hsv_row<const BITS: u32, const BE: bool>(
y: &[u16],
u_half: &[u16],
v_half: &[u16],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
debug_assert_eq!(width & 1, 0, "YUV 4:2:0 requires even width");
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(u_half.len() >= width / 2, "u_half row too short");
debug_assert!(v_half.len() >= width / 2, "v_half row too short");
debug_assert!(h_out.len() >= width, "h_out row too short");
debug_assert!(s_out.len() >= width, "s_out row too short");
debug_assert!(v_out.len() >= width, "v_out row too short");
unsafe {
yuv_to_hsv_via_rgb_chunks(h_out, s_out, v_out, width, |offset, n, rgb| {
yuv_420p_n_to_rgb_row::<BITS, BE>(
&y[offset..],
&u_half[offset / 2..],
&v_half[offset / 2..],
rgb,
n,
matrix,
full_range,
);
});
}
}
#[inline]
#[target_feature(enable = "simd128")]
#[allow(clippy::too_many_arguments)]
pub(crate) unsafe fn yuv_444p_n_to_hsv_row<const BITS: u32, const BE: bool>(
y: &[u16],
u: &[u16],
v: &[u16],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(u.len() >= width, "u row too short");
debug_assert!(v.len() >= width, "v row too short");
debug_assert!(h_out.len() >= width, "h_out row too short");
debug_assert!(s_out.len() >= width, "s_out row too short");
debug_assert!(v_out.len() >= width, "v_out row too short");
unsafe {
yuv_to_hsv_via_rgb_chunks(h_out, s_out, v_out, width, |offset, n, rgb| {
yuv_444p_n_to_rgb_row::<BITS, BE>(
&y[offset..],
&u[offset..],
&v[offset..],
rgb,
n,
matrix,
full_range,
);
});
}
}
#[inline]
#[target_feature(enable = "simd128")]
unsafe fn extract_hb_simd<const BITS: u32, const MSB: bool>(raw: v128, mask_v: v128) -> v128 {
if MSB {
u16x8_shr(raw, (16 - BITS) as u32)
} else {
v128_and(raw, mask_v)
}
}
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn yuv_444p_n_msb_to_rgb_row<const BITS: u32, const BE: bool>(
y: &[u16],
u: &[u16],
v: &[u16],
rgb_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
yuv_444p_n_to_rgb_or_rgba_row::<BITS, false, false, BE, true>(
y, u, v, rgb_out, width, matrix, full_range, None,
);
}
}
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn yuv_444p_n_msb_to_rgba_row<const BITS: u32, const BE: bool>(
y: &[u16],
u: &[u16],
v: &[u16],
rgba_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
yuv_444p_n_to_rgb_or_rgba_row::<BITS, true, false, BE, true>(
y, u, v, rgba_out, width, matrix, full_range, None,
);
}
}
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn yuv_444p_n_msb_to_rgb_u16_row<const BITS: u32, const BE: bool>(
y: &[u16],
u: &[u16],
v: &[u16],
rgb_out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
yuv_444p_n_to_rgb_or_rgba_u16_row::<BITS, false, false, BE, true>(
y, u, v, rgb_out, width, matrix, full_range, None,
);
}
}
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn yuv_444p_n_msb_to_rgba_u16_row<const BITS: u32, const BE: bool>(
y: &[u16],
u: &[u16],
v: &[u16],
rgba_out: &mut [u16],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
unsafe {
yuv_444p_n_to_rgb_or_rgba_u16_row::<BITS, true, false, BE, true>(
y, u, v, rgba_out, width, matrix, full_range, None,
);
}
}
#[inline]
#[target_feature(enable = "simd128")]
#[allow(clippy::too_many_arguments)]
pub(crate) unsafe fn yuv_444p_n_msb_to_hsv_row<const BITS: u32, const BE: bool>(
y: &[u16],
u: &[u16],
v: &[u16],
h_out: &mut [u8],
s_out: &mut [u8],
v_out: &mut [u8],
width: usize,
matrix: KernelMatrix,
full_range: bool,
) {
debug_assert!(y.len() >= width, "y row too short");
debug_assert!(u.len() >= width, "u row too short");
debug_assert!(v.len() >= width, "v row too short");
debug_assert!(h_out.len() >= width, "h_out row too short");
debug_assert!(s_out.len() >= width, "s_out row too short");
debug_assert!(v_out.len() >= width, "v_out row too short");
unsafe {
yuv_to_hsv_via_rgb_chunks(h_out, s_out, v_out, width, |offset, n, rgb| {
yuv_444p_n_msb_to_rgb_row::<BITS, BE>(
&y[offset..],
&u[offset..],
&v[offset..],
rgb,
n,
matrix,
full_range,
);
});
}
}