#![cfg_attr(
not(all(
any(feature = "std", feature = "alloc"),
feature = "yuv-planar",
feature = "yuv-semi-planar"
)),
allow(dead_code)
)]
use core::arch::wasm32::*;
use crate::row::scalar;
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[inline]
#[target_feature(enable = "simd128")]
fn bswap_u16(v: v128) -> v128 {
i8x16_shuffle::<1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14>(v, v)
}
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[inline]
#[target_feature(enable = "simd128")]
fn blend_u16(left: v128, mid: v128, right: v128) -> (v128, v128) {
let two = i32x4_splat(2);
let three = i32x4_splat(3);
let mid_lo = u32x4_extend_low_u16x8(mid);
let mid_hi = u32x4_extend_high_u16x8(mid);
let left_lo = u32x4_extend_low_u16x8(left);
let left_hi = u32x4_extend_high_u16x8(left);
let right_lo = u32x4_extend_low_u16x8(right);
let right_hi = u32x4_extend_high_u16x8(right);
let tm_lo = i32x4_mul(mid_lo, three);
let tm_hi = i32x4_mul(mid_hi, three);
let even_lo = u32x4_shr(i32x4_add(i32x4_add(left_lo, tm_lo), two), 2);
let even_hi = u32x4_shr(i32x4_add(i32x4_add(left_hi, tm_hi), two), 2);
let odd_lo = u32x4_shr(i32x4_add(i32x4_add(tm_lo, right_lo), two), 2);
let odd_hi = u32x4_shr(i32x4_add(i32x4_add(tm_hi, right_hi), two), 2);
(
u16x8_narrow_i32x4(even_lo, even_hi),
u16x8_narrow_i32x4(odd_lo, odd_hi),
)
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn chroma_upsample_2to1_center_h_row(
c_half: &[u8],
c_full: &mut [u8],
width: usize,
) {
debug_assert_eq!(width & 1, 0, "2:1 horizontal chroma requires even width");
debug_assert!(c_half.len() >= width / 2, "c_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let half = width / 2;
if half == 0 {
return;
}
scalar::chroma_upsample_2to1_center_h_pair(c_half, c_full, 0, half);
if half == 1 {
return;
}
let mut j = 1usize;
let two = i16x8_splat(2);
let three = i16x8_splat(3);
unsafe {
while j + 16 < half {
let mid = v128_load(c_half.as_ptr().add(j).cast());
let left = v128_load(c_half.as_ptr().add(j - 1).cast());
let right = v128_load(c_half.as_ptr().add(j + 1).cast());
let mid_lo = u16x8_extend_low_u8x16(mid);
let mid_hi = u16x8_extend_high_u8x16(mid);
let left_lo = u16x8_extend_low_u8x16(left);
let left_hi = u16x8_extend_high_u8x16(left);
let right_lo = u16x8_extend_low_u8x16(right);
let right_hi = u16x8_extend_high_u8x16(right);
let tm_lo = i16x8_mul(mid_lo, three);
let tm_hi = i16x8_mul(mid_hi, three);
let even_lo = u16x8_shr(i16x8_add(i16x8_add(left_lo, tm_lo), two), 2);
let even_hi = u16x8_shr(i16x8_add(i16x8_add(left_hi, tm_hi), two), 2);
let odd_lo = u16x8_shr(i16x8_add(i16x8_add(tm_lo, right_lo), two), 2);
let odd_hi = u16x8_shr(i16x8_add(i16x8_add(tm_hi, right_hi), two), 2);
let even = u8x16_narrow_i16x8(even_lo, even_hi);
let odd = u8x16_narrow_i16x8(odd_lo, odd_hi);
let out_lo =
i8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(even, odd);
let out_hi =
i8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(even, odd);
v128_store(c_full.as_mut_ptr().add(2 * j).cast(), out_lo);
v128_store(c_full.as_mut_ptr().add(2 * j + 16).cast(), out_hi);
j += 16;
}
}
while j < half {
scalar::chroma_upsample_2to1_center_h_pair(c_half, c_full, j, half);
j += 1;
}
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn chroma_upsample_2to1_center_h_u16_row<const BITS: u32>(
c_half: &[u16],
c_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(width & 1, 0, "2:1 horizontal chroma requires even width");
debug_assert!(c_half.len() >= width / 2, "c_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let half = width / 2;
if half == 0 {
return;
}
scalar::chroma_upsample_2to1_center_h_u16_pair::<BITS>(c_half, c_full, 0, half, big_endian);
if half == 1 {
return;
}
let swap = big_endian != cfg!(target_endian = "big");
let mask = u16x8_splat(((1u32 << BITS) - 1) as u16);
let mut j = 1usize;
unsafe {
while j + 8 < half {
let norm = |ptr_off: usize| {
let raw = v128_load(c_half.as_ptr().add(ptr_off).cast());
let host = if swap { bswap_u16(raw) } else { raw };
v128_and(host, mask)
};
let mid = norm(j);
let left = norm(j - 1);
let right = norm(j + 1);
let (even, odd) = blend_u16(left, mid, right);
let even = if swap { bswap_u16(even) } else { even };
let odd = if swap { bswap_u16(odd) } else { odd };
let out_lo =
i8x16_shuffle::<0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23>(even, odd);
let out_hi =
i8x16_shuffle::<8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31>(even, odd);
v128_store(c_full.as_mut_ptr().add(2 * j).cast(), out_lo);
v128_store(c_full.as_mut_ptr().add(2 * j + 8).cast(), out_hi);
j += 8;
}
}
while j < half {
scalar::chroma_upsample_2to1_center_h_u16_pair::<BITS>(c_half, c_full, j, half, big_endian);
j += 1;
}
}
#[cfg(all(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn chroma_upsample_2to1_center_h_p0xx_row<
const BITS: u32,
const LOW_PACKED: bool,
>(
uv_half: &[u16],
uv_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(width & 1, 0, "P-format 4:2:0 requires even width");
debug_assert!(uv_half.len() >= width, "uv_half row too short");
debug_assert!(uv_full.len() >= 2 * width, "uv_full row too short");
let half = width / 2;
if half == 0 {
return;
}
scalar::chroma_upsample_2to1_center_h_p0xx_pair::<BITS, LOW_PACKED>(
uv_half, uv_full, 0, half, big_endian,
);
if half == 1 {
return;
}
let swap = big_endian != cfg!(target_endian = "big");
let shift = 16 - BITS;
let mask = u32x4_splat((1u32 << BITS) - 1);
let mut j = 1usize;
unsafe {
while j + 4 < half {
let load_uv = |off: usize| -> (v128, v128) {
let raw = v128_load(uv_half.as_ptr().add(2 * off).cast());
let host = if swap { bswap_u16(raw) } else { raw };
let u = i8x16_shuffle::<0, 1, 4, 5, 8, 9, 12, 13, 0, 1, 4, 5, 8, 9, 12, 13>(host, host);
let v = i8x16_shuffle::<2, 3, 6, 7, 10, 11, 14, 15, 2, 3, 6, 7, 10, 11, 14, 15>(host, host);
(u32x4_extend_low_u16x8(u), u32x4_extend_low_u16x8(v))
};
let depack = |v: v128| {
if LOW_PACKED {
v128_and(v, mask)
} else {
u32x4_shr(v, shift)
}
};
let (ul_r, vl_r) = load_uv(j - 1);
let (um_r, vm_r) = load_uv(j);
let (ur_r, vr_r) = load_uv(j + 1);
let ul = depack(ul_r);
let um = depack(um_r);
let ur = depack(ur_r);
let vl = depack(vl_r);
let vm = depack(vm_r);
let vr = depack(vr_r);
let two = i32x4_splat(2);
let three = i32x4_splat(3);
let blend_even =
|a: v128, b: v128| u32x4_shr(i32x4_add(i32x4_add(a, i32x4_mul(b, three)), two), 2);
let blend_odd =
|b: v128, c: v128| u32x4_shr(i32x4_add(i32x4_add(i32x4_mul(b, three), c), two), 2);
let repack = |v: v128| {
let n = u16x8_narrow_i32x4(v, v);
let aligned = if LOW_PACKED { n } else { u16x8_shl(n, shift) };
if swap { bswap_u16(aligned) } else { aligned }
};
let u_even = repack(blend_even(ul, um));
let u_odd = repack(blend_odd(um, ur));
let v_even = repack(blend_even(vl, vm));
let v_odd = repack(blend_odd(vm, vr));
let uv_even =
i8x16_shuffle::<0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23>(u_even, v_even);
let uv_odd =
i8x16_shuffle::<0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23>(u_odd, v_odd);
let out_lo =
i8x16_shuffle::<0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23>(uv_even, uv_odd);
let out_hi = i8x16_shuffle::<8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31>(
uv_even, uv_odd,
);
v128_store(uv_full.as_mut_ptr().add(4 * j).cast(), out_lo);
v128_store(uv_full.as_mut_ptr().add(4 * j + 8).cast(), out_hi);
j += 4;
}
}
while j < half {
scalar::chroma_upsample_2to1_center_h_p0xx_pair::<BITS, LOW_PACKED>(
uv_half, uv_full, j, half, big_endian,
);
j += 1;
}
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn chroma_upsample_420_bottom_even_h_row(
prev_half: &[u8],
cur_half: &[u8],
c_full: &mut [u8],
width: usize,
) {
debug_assert_eq!(width & 1, 0, "YUV 4:2:0 requires even width");
debug_assert!(prev_half.len() >= width / 2, "prev_half row too short");
debug_assert!(cur_half.len() >= width / 2, "cur_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let half = width / 2;
if half == 0 {
return;
}
scalar::chroma_upsample_420_bottom_even_h_pair(prev_half, cur_half, c_full, 0, half);
if half == 1 {
return;
}
let mut j = 1usize;
let two = i16x8_splat(2);
let three = i16x8_splat(3);
unsafe {
while j + 16 < half {
let avg = |off: usize| {
u8x16_avgr(
v128_load(prev_half.as_ptr().add(off).cast()),
v128_load(cur_half.as_ptr().add(off).cast()),
)
};
let e_left = avg(j - 1);
let e_mid = avg(j);
let e_right = avg(j + 1);
let mid_lo = u16x8_extend_low_u8x16(e_mid);
let mid_hi = u16x8_extend_high_u8x16(e_mid);
let left_lo = u16x8_extend_low_u8x16(e_left);
let left_hi = u16x8_extend_high_u8x16(e_left);
let right_lo = u16x8_extend_low_u8x16(e_right);
let right_hi = u16x8_extend_high_u8x16(e_right);
let tm_lo = i16x8_mul(mid_lo, three);
let tm_hi = i16x8_mul(mid_hi, three);
let even_lo = u16x8_shr(i16x8_add(i16x8_add(left_lo, tm_lo), two), 2);
let even_hi = u16x8_shr(i16x8_add(i16x8_add(left_hi, tm_hi), two), 2);
let odd_lo = u16x8_shr(i16x8_add(i16x8_add(tm_lo, right_lo), two), 2);
let odd_hi = u16x8_shr(i16x8_add(i16x8_add(tm_hi, right_hi), two), 2);
let even = u8x16_narrow_i16x8(even_lo, even_hi);
let odd = u8x16_narrow_i16x8(odd_lo, odd_hi);
let out_lo =
i8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(even, odd);
let out_hi =
i8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(even, odd);
v128_store(c_full.as_mut_ptr().add(2 * j).cast(), out_lo);
v128_store(c_full.as_mut_ptr().add(2 * j + 16).cast(), out_hi);
j += 16;
}
}
while j < half {
scalar::chroma_upsample_420_bottom_even_h_pair(prev_half, cur_half, c_full, j, half);
j += 1;
}
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn chroma_upsample_420_bottomleft_even_h_row(
prev_half: &[u8],
cur_half: &[u8],
c_full: &mut [u8],
width: usize,
) {
debug_assert_eq!(width & 1, 0, "YUV 4:2:0 requires even width");
debug_assert!(prev_half.len() >= width / 2, "prev_half row too short");
debug_assert!(cur_half.len() >= width / 2, "cur_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let half = width / 2;
let mut j = 0usize;
unsafe {
while j + 16 <= half {
let e = u8x16_avgr(
v128_load(prev_half.as_ptr().add(j).cast()),
v128_load(cur_half.as_ptr().add(j).cast()),
);
let out_lo = i8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(e, e);
let out_hi =
i8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(e, e);
v128_store(c_full.as_mut_ptr().add(2 * j).cast(), out_lo);
v128_store(c_full.as_mut_ptr().add(2 * j + 16).cast(), out_hi);
j += 16;
}
}
while j < half {
scalar::chroma_upsample_420_bottomleft_even_h_pair(prev_half, cur_half, c_full, j);
j += 1;
}
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn chroma_upsample_420_bottom_even_h_u16_row<const BITS: u32>(
prev_half: &[u16],
cur_half: &[u16],
c_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(width & 1, 0, "YUV 4:2:0 requires even width");
debug_assert!(prev_half.len() >= width / 2, "prev_half row too short");
debug_assert!(cur_half.len() >= width / 2, "cur_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let half = width / 2;
if half == 0 {
return;
}
scalar::chroma_upsample_420_bottom_even_h_u16_pair::<BITS>(
prev_half, cur_half, c_full, 0, half, big_endian,
);
if half == 1 {
return;
}
let swap = big_endian != cfg!(target_endian = "big");
let mask = u16x8_splat(((1u32 << BITS) - 1) as u16);
let mut j = 1usize;
unsafe {
while j + 8 < half {
let norm = |row: &[u16], off: usize| {
let raw = v128_load(row.as_ptr().add(off).cast());
let host = if swap { bswap_u16(raw) } else { raw };
v128_and(host, mask)
};
let vavg = |off: usize| u16x8_avgr(norm(prev_half, off), norm(cur_half, off));
let e_left = vavg(j - 1);
let e_mid = vavg(j);
let e_right = vavg(j + 1);
let (even, odd) = blend_u16(e_left, e_mid, e_right);
let even = if swap { bswap_u16(even) } else { even };
let odd = if swap { bswap_u16(odd) } else { odd };
let out_lo =
i8x16_shuffle::<0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23>(even, odd);
let out_hi =
i8x16_shuffle::<8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31>(even, odd);
v128_store(c_full.as_mut_ptr().add(2 * j).cast(), out_lo);
v128_store(c_full.as_mut_ptr().add(2 * j + 8).cast(), out_hi);
j += 8;
}
}
while j < half {
scalar::chroma_upsample_420_bottom_even_h_u16_pair::<BITS>(
prev_half, cur_half, c_full, j, half, big_endian,
);
j += 1;
}
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn chroma_upsample_420_bottomleft_even_h_u16_row<const BITS: u32>(
prev_half: &[u16],
cur_half: &[u16],
c_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(width & 1, 0, "YUV 4:2:0 requires even width");
debug_assert!(prev_half.len() >= width / 2, "prev_half row too short");
debug_assert!(cur_half.len() >= width / 2, "cur_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let half = width / 2;
let swap = big_endian != cfg!(target_endian = "big");
let mask = u16x8_splat(((1u32 << BITS) - 1) as u16);
let mut j = 0usize;
unsafe {
while j + 8 <= half {
let norm = |row: &[u16], off: usize| {
let raw = v128_load(row.as_ptr().add(off).cast());
let host = if swap { bswap_u16(raw) } else { raw };
v128_and(host, mask)
};
let e = u16x8_avgr(norm(prev_half, j), norm(cur_half, j));
let e = if swap { bswap_u16(e) } else { e };
let out_lo = i8x16_shuffle::<0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23>(e, e);
let out_hi =
i8x16_shuffle::<8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31>(e, e);
v128_store(c_full.as_mut_ptr().add(2 * j).cast(), out_lo);
v128_store(c_full.as_mut_ptr().add(2 * j + 8).cast(), out_hi);
j += 8;
}
}
while j < half {
scalar::chroma_upsample_420_bottomleft_even_h_u16_pair::<BITS>(
prev_half, cur_half, c_full, j, big_endian,
);
j += 1;
}
}
#[cfg(all(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn chroma_upsample_420_bottom_even_h_p0xx_row<const BITS: u32>(
prev_uv_half: &[u16],
cur_uv_half: &[u16],
uv_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(width & 1, 0, "P-format 4:2:0 requires even width");
debug_assert!(prev_uv_half.len() >= width, "prev_uv_half row too short");
debug_assert!(cur_uv_half.len() >= width, "cur_uv_half row too short");
debug_assert!(uv_full.len() >= 2 * width, "uv_full row too short");
let half = width / 2;
if half == 0 {
return;
}
scalar::chroma_upsample_420_bottom_even_h_p0xx_pair::<BITS>(
prev_uv_half,
cur_uv_half,
uv_full,
0,
half,
big_endian,
);
if half == 1 {
return;
}
let swap = big_endian != cfg!(target_endian = "big");
let shift = 16 - BITS;
let one = u32x4_splat(1);
let mut j = 1usize;
unsafe {
while j + 4 < half {
let dp = |row: &[u16], off: usize| -> (v128, v128) {
let raw = v128_load(row.as_ptr().add(2 * off).cast());
let host = if swap { bswap_u16(raw) } else { raw };
let u = i8x16_shuffle::<0, 1, 4, 5, 8, 9, 12, 13, 0, 1, 4, 5, 8, 9, 12, 13>(host, host);
let v = i8x16_shuffle::<2, 3, 6, 7, 10, 11, 14, 15, 2, 3, 6, 7, 10, 11, 14, 15>(host, host);
(
u32x4_shr(u32x4_extend_low_u16x8(u), shift),
u32x4_shr(u32x4_extend_low_u16x8(v), shift),
)
};
let vavg = |a: v128, b: v128| u32x4_shr(u32x4_add(u32x4_add(a, b), one), 1);
let (pul, pvl) = dp(prev_uv_half, j - 1);
let (cul, cvl) = dp(cur_uv_half, j - 1);
let (pum, pvm) = dp(prev_uv_half, j);
let (cum, cvm) = dp(cur_uv_half, j);
let (pur, pvr) = dp(prev_uv_half, j + 1);
let (cur_, cvr) = dp(cur_uv_half, j + 1);
let ul = vavg(pul, cul);
let um = vavg(pum, cum);
let ur = vavg(pur, cur_);
let vl = vavg(pvl, cvl);
let vm = vavg(pvm, cvm);
let vr = vavg(pvr, cvr);
let two = i32x4_splat(2);
let three = i32x4_splat(3);
let blend_even =
|a: v128, b: v128| u32x4_shr(i32x4_add(i32x4_add(a, i32x4_mul(b, three)), two), 2);
let blend_odd =
|b: v128, c: v128| u32x4_shr(i32x4_add(i32x4_add(i32x4_mul(b, three), c), two), 2);
let repack = |v: v128| {
let n = u16x8_narrow_i32x4(v, v);
let aligned = u16x8_shl(n, shift);
if swap { bswap_u16(aligned) } else { aligned }
};
let u_even = repack(blend_even(ul, um));
let u_odd = repack(blend_odd(um, ur));
let v_even = repack(blend_even(vl, vm));
let v_odd = repack(blend_odd(vm, vr));
let uv_even =
i8x16_shuffle::<0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23>(u_even, v_even);
let uv_odd =
i8x16_shuffle::<0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23>(u_odd, v_odd);
let out_lo =
i8x16_shuffle::<0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23>(uv_even, uv_odd);
let out_hi = i8x16_shuffle::<8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31>(
uv_even, uv_odd,
);
v128_store(uv_full.as_mut_ptr().add(4 * j).cast(), out_lo);
v128_store(uv_full.as_mut_ptr().add(4 * j + 8).cast(), out_hi);
j += 4;
}
}
while j < half {
scalar::chroma_upsample_420_bottom_even_h_p0xx_pair::<BITS>(
prev_uv_half,
cur_uv_half,
uv_full,
j,
half,
big_endian,
);
j += 1;
}
}
#[cfg(all(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn chroma_upsample_420_bottomleft_even_h_p0xx_row<const BITS: u32>(
prev_uv_half: &[u16],
cur_uv_half: &[u16],
uv_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(width & 1, 0, "P-format 4:2:0 requires even width");
debug_assert!(prev_uv_half.len() >= width, "prev_uv_half row too short");
debug_assert!(cur_uv_half.len() >= width, "cur_uv_half row too short");
debug_assert!(uv_full.len() >= 2 * width, "uv_full row too short");
let half = width / 2;
let swap = big_endian != cfg!(target_endian = "big");
let shift = 16 - BITS;
let one = u32x4_splat(1);
let mut j = 0usize;
unsafe {
while j + 4 <= half {
let dp = |row: &[u16], off: usize| -> (v128, v128) {
let raw = v128_load(row.as_ptr().add(2 * off).cast());
let host = if swap { bswap_u16(raw) } else { raw };
let u = i8x16_shuffle::<0, 1, 4, 5, 8, 9, 12, 13, 0, 1, 4, 5, 8, 9, 12, 13>(host, host);
let v = i8x16_shuffle::<2, 3, 6, 7, 10, 11, 14, 15, 2, 3, 6, 7, 10, 11, 14, 15>(host, host);
(
u32x4_shr(u32x4_extend_low_u16x8(u), shift),
u32x4_shr(u32x4_extend_low_u16x8(v), shift),
)
};
let vavg = |a: v128, b: v128| u32x4_shr(u32x4_add(u32x4_add(a, b), one), 1);
let repack = |v: v128| {
let n = u16x8_narrow_i32x4(v, v);
let aligned = u16x8_shl(n, shift);
if swap { bswap_u16(aligned) } else { aligned }
};
let (pu, pv) = dp(prev_uv_half, j);
let (cu, cv) = dp(cur_uv_half, j);
let u = repack(vavg(pu, cu));
let v = repack(vavg(pv, cv));
let uv = i8x16_shuffle::<0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23>(u, v);
let out_lo = i8x16_shuffle::<0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23>(uv, uv);
let out_hi =
i8x16_shuffle::<8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31>(uv, uv);
v128_store(uv_full.as_mut_ptr().add(4 * j).cast(), out_lo);
v128_store(uv_full.as_mut_ptr().add(4 * j + 8).cast(), out_hi);
j += 4;
}
}
while j < half {
scalar::chroma_upsample_420_bottomleft_even_h_p0xx_pair::<BITS>(
prev_uv_half,
cur_uv_half,
uv_full,
j,
big_endian,
);
j += 1;
}
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn chroma_upsample_440_bottom_v_row(
prev: &[u8],
cur: &[u8],
out: &mut [u8],
width: usize,
) {
debug_assert!(prev.len() >= width, "prev row too short");
debug_assert!(cur.len() >= width, "cur row too short");
debug_assert!(out.len() >= width, "out row too short");
let mut j = 0;
unsafe {
while j + 16 <= width {
let p = v128_load(prev.as_ptr().add(j).cast());
let c = v128_load(cur.as_ptr().add(j).cast());
v128_store(out.as_mut_ptr().add(j).cast(), u8x16_avgr(p, c));
j += 16;
}
}
while j < width {
out[j] = (((prev[j] as u16) + (cur[j] as u16) + 1) >> 1) as u8;
j += 1;
}
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn chroma_upsample_440_bottom_v_u16_row<const BITS: u32>(
prev: &[u16],
cur: &[u16],
out: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert!(prev.len() >= width, "prev row too short");
debug_assert!(cur.len() >= width, "cur row too short");
debug_assert!(out.len() >= width, "out row too short");
let swap = big_endian != cfg!(target_endian = "big");
let mask = u16x8_splat(((1u32 << BITS) - 1) as u16);
let mut j = 0;
unsafe {
while j + 8 <= width {
let p_raw = v128_load(prev.as_ptr().add(j).cast());
let p = v128_and(if swap { bswap_u16(p_raw) } else { p_raw }, mask);
let c_raw = v128_load(cur.as_ptr().add(j).cast());
let c = v128_and(if swap { bswap_u16(c_raw) } else { c_raw }, mask);
let avg = u16x8_avgr(p, c);
let enc = if swap { bswap_u16(avg) } else { avg };
v128_store(out.as_mut_ptr().add(j).cast(), enc);
j += 8;
}
}
let mask = ((1u32 << BITS) - 1) as u16;
let load = |raw: u16| -> u32 {
let logical = if big_endian {
u16::from_be(raw)
} else {
u16::from_le(raw)
};
u32::from(logical & mask)
};
while j < width {
let blended = ((load(prev[j]) + load(cur[j]) + 1) >> 1) as u16;
out[j] = if big_endian {
blended.to_be()
} else {
blended.to_le()
};
j += 1;
}
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "simd128")]
fn blend4_u8x16(left: v128, mid: v128, right: v128) -> (v128, v128, v128, v128) {
let four = i16x8_splat(4);
let ll = u16x8_extend_low_u8x16(left);
let lh = u16x8_extend_high_u8x16(left);
let ml = u16x8_extend_low_u8x16(mid);
let mh = u16x8_extend_high_u8x16(mid);
let rl = u16x8_extend_low_u8x16(right);
let rh = u16x8_extend_high_u8x16(right);
let phase = |l: v128, wl: i16, m: v128, wm: i16, r: v128, wr: i16| {
let acc = i16x8_add(
i16x8_add(
i16x8_add(i16x8_mul(l, i16x8_splat(wl)), i16x8_mul(m, i16x8_splat(wm))),
i16x8_mul(r, i16x8_splat(wr)),
),
four,
);
u16x8_shr(acc, 3)
};
let a = u8x16_narrow_i16x8(phase(ll, 3, ml, 5, rl, 0), phase(lh, 3, mh, 5, rh, 0));
let b = u8x16_narrow_i16x8(phase(ll, 1, ml, 7, rl, 0), phase(lh, 1, mh, 7, rh, 0));
let c = u8x16_narrow_i16x8(phase(ll, 0, ml, 7, rl, 1), phase(lh, 0, mh, 7, rh, 1));
let d = u8x16_narrow_i16x8(phase(ll, 0, ml, 5, rl, 3), phase(lh, 0, mh, 5, rh, 3));
(a, b, c, d)
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "simd128")]
pub(crate) unsafe fn chroma_upsample_4to1_center_h_row(
c_quarter: &[u8],
c_full: &mut [u8],
width: usize,
) {
debug_assert!(
c_quarter.len() >= width.div_ceil(4),
"c_quarter row too short"
);
debug_assert!(c_full.len() >= width, "c_full row too short");
let quarter = width.div_ceil(4);
if quarter == 0 {
return;
}
scalar::chroma_upsample_4to1_center_h_group(c_quarter, c_full, 0, quarter, width);
if quarter == 1 {
return;
}
let mut j = 1usize;
unsafe {
while j + 16 < quarter {
let left = v128_load(c_quarter.as_ptr().add(j - 1).cast());
let mid = v128_load(c_quarter.as_ptr().add(j).cast());
let right = v128_load(c_quarter.as_ptr().add(j + 1).cast());
let (a, b, c, d) = blend4_u8x16(left, mid, right);
let ab0 = i8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(a, b);
let ab1 = i8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(a, b);
let cd0 = i8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(c, d);
let cd1 = i8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(c, d);
let o0 = i8x16_shuffle::<0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23>(ab0, cd0);
let o1 =
i8x16_shuffle::<8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31>(ab0, cd0);
let o2 = i8x16_shuffle::<0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23>(ab1, cd1);
let o3 =
i8x16_shuffle::<8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31>(ab1, cd1);
let base = 4 * j;
v128_store(c_full.as_mut_ptr().add(base).cast(), o0);
v128_store(c_full.as_mut_ptr().add(base + 16).cast(), o1);
v128_store(c_full.as_mut_ptr().add(base + 32).cast(), o2);
v128_store(c_full.as_mut_ptr().add(base + 48).cast(), o3);
j += 16;
}
}
while j < quarter {
scalar::chroma_upsample_4to1_center_h_group(c_quarter, c_full, j, quarter, width);
j += 1;
}
}
#[cfg(all(
test,
feature = "std",
feature = "yuv-planar",
feature = "yuv-semi-planar"
))]
mod tests {
use crate::row::scalar;
fn pseudo_random_u16(out: &mut [u16], seed: u32) {
let mut state = seed;
for v in out.iter_mut() {
state = state.wrapping_mul(1664525).wrapping_add(1013904223);
*v = (state >> 8) as u16;
}
}
const WIDTHS: &[usize] = &[2, 4, 6, 8, 16, 18, 30, 32, 34, 62, 64, 66, 128, 130];
fn check_p0xx<const BITS: u32, const LOW_PACKED: bool>(big_endian: bool) {
for &w in WIDTHS {
let mut uv_half = std::vec![0u16; w];
pseudo_random_u16(
&mut uv_half,
0x9E37 ^ BITS ^ ((LOW_PACKED as u32) << 8) ^ (big_endian as u32),
);
let mut out_simd = std::vec![0u16; 2 * w];
let mut out_scalar = std::vec![0u16; 2 * w];
unsafe {
super::chroma_upsample_2to1_center_h_p0xx_row::<BITS, LOW_PACKED>(
&uv_half,
&mut out_simd,
w,
big_endian,
)
};
scalar::chroma_upsample_2to1_center_h_p0xx::<BITS, LOW_PACKED>(
&uv_half,
&mut out_scalar,
w,
big_endian,
);
assert_eq!(
out_simd, out_scalar,
"p0xx BITS={BITS} low_packed={LOW_PACKED} be={big_endian} width={w}"
);
}
}
#[test]
#[cfg_attr(
miri,
ignore = "SIMD-dispatched row kernels use intrinsics unsupported by Miri"
)]
fn wasm_p0xx_matches_scalar_widths() {
check_p0xx::<10, false>(false);
check_p0xx::<10, false>(true);
check_p0xx::<10, true>(false);
check_p0xx::<10, true>(true);
check_p0xx::<12, false>(false);
check_p0xx::<16, false>(false);
check_p0xx::<16, false>(true);
}
fn check_p0xx_vertical<const BITS: u32>(big_endian: bool) {
for &w in WIDTHS {
let mut prev = std::vec![0u16; w];
let mut cur = std::vec![0u16; w];
pseudo_random_u16(&mut prev, 0x7E57 ^ BITS ^ (big_endian as u32));
pseudo_random_u16(&mut cur, 0xABCD ^ BITS ^ (big_endian as u32));
for (prev_row, tag) in [(prev.as_slice(), "interior"), (cur.as_slice(), "topedge")] {
let mut bot_simd = std::vec![0u16; 2 * w];
let mut bot_scalar = std::vec![0u16; 2 * w];
let mut bl_simd = std::vec![0u16; 2 * w];
let mut bl_scalar = std::vec![0u16; 2 * w];
unsafe {
super::chroma_upsample_420_bottom_even_h_p0xx_row::<BITS>(
prev_row,
&cur,
&mut bot_simd,
w,
big_endian,
);
super::chroma_upsample_420_bottomleft_even_h_p0xx_row::<BITS>(
prev_row,
&cur,
&mut bl_simd,
w,
big_endian,
);
}
scalar::chroma_upsample_420_bottom_even_h_p0xx::<BITS>(
prev_row,
&cur,
&mut bot_scalar,
w,
big_endian,
);
scalar::chroma_upsample_420_bottomleft_even_h_p0xx::<BITS>(
prev_row,
&cur,
&mut bl_scalar,
w,
big_endian,
);
assert_eq!(
bot_simd, bot_scalar,
"p0xx bottom BITS={BITS} be={big_endian} {tag} width={w}"
);
assert_eq!(
bl_simd, bl_scalar,
"p0xx bottomleft BITS={BITS} be={big_endian} {tag} width={w}"
);
}
}
}
#[test]
#[cfg_attr(
miri,
ignore = "SIMD-dispatched row kernels use intrinsics unsupported by Miri"
)]
fn wasm_p0xx_vertical_matches_scalar_widths() {
check_p0xx_vertical::<10>(false);
check_p0xx_vertical::<10>(true);
check_p0xx_vertical::<12>(false);
check_p0xx_vertical::<12>(true);
check_p0xx_vertical::<16>(false);
check_p0xx_vertical::<16>(true);
}
}
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
mod tests_planar {
use crate::row::scalar;
fn pseudo_random_u8(out: &mut [u8], seed: u32) {
let mut state = seed;
for v in out.iter_mut() {
state = state.wrapping_mul(1664525).wrapping_add(1013904223);
*v = (state >> 16) as u8;
}
}
fn pseudo_random_u16(out: &mut [u16], seed: u32) {
let mut state = seed;
for v in out.iter_mut() {
state = state.wrapping_mul(1664525).wrapping_add(1013904223);
*v = (state >> 8) as u16;
}
}
const WIDTHS_440: &[usize] = &[
1, 2, 3, 4, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, 128, 129, 1920,
];
const WIDTHS_4TO1: &[usize] = &[
1, 2, 3, 4, 5, 6, 7, 8, 16, 20, 63, 64, 65, 66, 67, 68, 69, 72, 73, 128, 129, 130, 131, 260,
];
const WIDTHS: &[usize] = &[2, 4, 6, 8, 16, 18, 30, 32, 34, 62, 64, 66, 128, 130];
#[test]
#[cfg_attr(
miri,
ignore = "SIMD-dispatched row kernels use intrinsics unsupported by Miri"
)]
fn wasm_u8_matches_scalar_widths() {
for &w in WIDTHS {
let half = w / 2;
let mut c_half = std::vec![0u8; half];
let mut state = 0xC0FFEEu32;
for v in c_half.iter_mut() {
state = state.wrapping_mul(1664525).wrapping_add(1013904223);
*v = (state >> 16) as u8;
}
let mut out_simd = std::vec![0u8; w];
let mut out_scalar = std::vec![0u8; w];
unsafe { super::chroma_upsample_2to1_center_h_row(&c_half, &mut out_simd, w) };
scalar::chroma_upsample_2to1_center_h(&c_half, &mut out_scalar, w);
assert_eq!(out_simd, out_scalar, "u8 width={w}");
}
}
fn check_u16<const BITS: u32>(big_endian: bool) {
for &w in WIDTHS {
let half = w / 2;
let mut c_half = std::vec![0u16; half];
pseudo_random_u16(&mut c_half, 0x1234 ^ BITS ^ (big_endian as u32));
let mut out_simd = std::vec![0u16; w];
let mut out_scalar = std::vec![0u16; w];
unsafe {
super::chroma_upsample_2to1_center_h_u16_row::<BITS>(&c_half, &mut out_simd, w, big_endian)
};
scalar::chroma_upsample_2to1_center_h_u16::<BITS>(&c_half, &mut out_scalar, w, big_endian);
assert_eq!(
out_simd, out_scalar,
"u16 BITS={BITS} be={big_endian} width={w}"
);
}
}
#[test]
#[cfg_attr(
miri,
ignore = "SIMD-dispatched row kernels use intrinsics unsupported by Miri"
)]
fn wasm_u16_matches_scalar_widths() {
check_u16::<10>(false);
check_u16::<10>(true);
check_u16::<12>(false);
check_u16::<12>(true);
check_u16::<16>(false);
check_u16::<16>(true);
}
#[test]
#[cfg_attr(
miri,
ignore = "SIMD-dispatched row kernels use intrinsics unsupported by Miri"
)]
fn wasm_u8_vertical_matches_scalar_widths() {
for &w in WIDTHS {
let half = w / 2;
let mut prev = std::vec![0u8; half];
let mut cur = std::vec![0u8; half];
pseudo_random_u8(&mut prev, 0xBEEF);
pseudo_random_u8(&mut cur, 0xFACE);
for (prev_row, tag) in [(prev.as_slice(), "interior"), (cur.as_slice(), "topedge")] {
let mut bot_simd = std::vec![0u8; w];
let mut bot_scalar = std::vec![0u8; w];
let mut bl_simd = std::vec![0u8; w];
let mut bl_scalar = std::vec![0u8; w];
unsafe {
super::chroma_upsample_420_bottom_even_h_row(prev_row, &cur, &mut bot_simd, w);
super::chroma_upsample_420_bottomleft_even_h_row(prev_row, &cur, &mut bl_simd, w);
}
scalar::chroma_upsample_420_bottom_even_h(prev_row, &cur, &mut bot_scalar, w);
scalar::chroma_upsample_420_bottomleft_even_h(prev_row, &cur, &mut bl_scalar, w);
assert_eq!(bot_simd, bot_scalar, "u8 bottom {tag} width={w}");
assert_eq!(bl_simd, bl_scalar, "u8 bottomleft {tag} width={w}");
}
}
}
fn check_u16_vertical<const BITS: u32>(big_endian: bool) {
for &w in WIDTHS {
let half = w / 2;
let mut prev = std::vec![0u16; half];
let mut cur = std::vec![0u16; half];
pseudo_random_u16(&mut prev, 0x51A5 ^ BITS ^ (big_endian as u32));
pseudo_random_u16(&mut cur, 0xC0DE ^ BITS ^ (big_endian as u32));
for (prev_row, tag) in [(prev.as_slice(), "interior"), (cur.as_slice(), "topedge")] {
let mut bot_simd = std::vec![0u16; w];
let mut bot_scalar = std::vec![0u16; w];
let mut bl_simd = std::vec![0u16; w];
let mut bl_scalar = std::vec![0u16; w];
unsafe {
super::chroma_upsample_420_bottom_even_h_u16_row::<BITS>(
prev_row,
&cur,
&mut bot_simd,
w,
big_endian,
);
super::chroma_upsample_420_bottomleft_even_h_u16_row::<BITS>(
prev_row,
&cur,
&mut bl_simd,
w,
big_endian,
);
}
scalar::chroma_upsample_420_bottom_even_h_u16::<BITS>(
prev_row,
&cur,
&mut bot_scalar,
w,
big_endian,
);
scalar::chroma_upsample_420_bottomleft_even_h_u16::<BITS>(
prev_row,
&cur,
&mut bl_scalar,
w,
big_endian,
);
assert_eq!(
bot_simd, bot_scalar,
"u16 bottom BITS={BITS} be={big_endian} {tag} width={w}"
);
assert_eq!(
bl_simd, bl_scalar,
"u16 bottomleft BITS={BITS} be={big_endian} {tag} width={w}"
);
}
}
}
#[test]
#[cfg_attr(
miri,
ignore = "SIMD-dispatched row kernels use intrinsics unsupported by Miri"
)]
fn wasm_u16_vertical_matches_scalar_widths() {
check_u16_vertical::<10>(false);
check_u16_vertical::<10>(true);
check_u16_vertical::<12>(false);
check_u16_vertical::<12>(true);
check_u16_vertical::<16>(false);
check_u16_vertical::<16>(true);
}
#[test]
#[cfg_attr(
miri,
ignore = "SIMD-dispatched row kernels use intrinsics unsupported by Miri"
)]
fn wasm_440_bottom_v_matches_scalar_widths() {
for &w in WIDTHS_440 {
let mut prev = std::vec![0u8; w];
let mut cur = std::vec![0u8; w];
pseudo_random_u8(&mut prev, 0x440B);
pseudo_random_u8(&mut cur, 0x440C);
for (prev_row, tag) in [(prev.as_slice(), "interior"), (cur.as_slice(), "topedge")] {
let mut simd = std::vec![0u8; w];
let mut sc = std::vec![0u8; w];
unsafe { super::chroma_upsample_440_bottom_v_row(prev_row, &cur, &mut simd, w) };
scalar::chroma_upsample_440_bottom_v(prev_row, &cur, &mut sc, w);
assert_eq!(simd, sc, "u8 440 {tag} width={w}");
}
}
}
fn check_440_u16<const BITS: u32>(big_endian: bool) {
for &w in WIDTHS_440 {
let mut prev = std::vec![0u16; w];
let mut cur = std::vec![0u16; w];
pseudo_random_u16(&mut prev, 0x4416 ^ BITS ^ (big_endian as u32));
pseudo_random_u16(&mut cur, 0x4417 ^ BITS ^ (big_endian as u32));
for (prev_row, tag) in [(prev.as_slice(), "interior"), (cur.as_slice(), "topedge")] {
let mut simd = std::vec![0u16; w];
let mut sc = std::vec![0u16; w];
unsafe {
super::chroma_upsample_440_bottom_v_u16_row::<BITS>(
prev_row, &cur, &mut simd, w, big_endian,
)
};
scalar::chroma_upsample_440_bottom_v_u16_wire::<BITS>(
prev_row, &cur, &mut sc, w, big_endian,
);
assert_eq!(
simd, sc,
"u16 440 BITS={BITS} be={big_endian} {tag} width={w}"
);
}
}
}
#[test]
#[cfg_attr(
miri,
ignore = "SIMD-dispatched row kernels use intrinsics unsupported by Miri"
)]
fn wasm_440_bottom_v_u16_matches_scalar_widths() {
check_440_u16::<10>(false);
check_440_u16::<10>(true);
check_440_u16::<12>(false);
check_440_u16::<12>(true);
check_440_u16::<16>(false);
check_440_u16::<16>(true);
}
#[test]
#[cfg_attr(
miri,
ignore = "SIMD-dispatched row kernels use intrinsics unsupported by Miri"
)]
fn wasm_4to1_center_h_matches_scalar_widths() {
for &w in WIDTHS_4TO1 {
let quarter = w.div_ceil(4);
let mut cq = std::vec![0u8; quarter];
pseudo_random_u8(&mut cq, 0x4701 ^ w as u32);
let mut simd = std::vec![0u8; w];
let mut sc = std::vec![0u8; w];
unsafe { super::chroma_upsample_4to1_center_h_row(&cq, &mut simd, w) };
scalar::chroma_upsample_4to1_center_h(&cq, &mut sc, w);
assert_eq!(simd, sc, "u8 4to1 width={w}");
}
}
}