#![cfg_attr(
not(all(
any(feature = "std", feature = "alloc"),
feature = "yuv-planar",
feature = "yuv-semi-planar"
)),
allow(dead_code)
)]
#[cfg(target_arch = "x86_64")]
#[cfg_attr(miri, allow(unused_imports))]
use core::arch::x86_64::*;
use crate::row::scalar;
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[inline]
#[target_feature(enable = "sse4.1")]
fn bswap16_mask() -> __m128i {
_mm_set_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1)
}
#[cfg(any(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[inline]
#[target_feature(enable = "sse4.1")]
fn blend_u16x8(left: __m128i, mid: __m128i, right: __m128i) -> (__m128i, __m128i) {
let two = _mm_set1_epi32(2);
let three = _mm_set1_epi32(3);
let widen_lo = |v: __m128i| _mm_cvtepu16_epi32(v);
let widen_hi = |v: __m128i| _mm_cvtepu16_epi32(_mm_srli_si128::<8>(v));
let mid_lo = widen_lo(mid);
let mid_hi = widen_hi(mid);
let left_lo = widen_lo(left);
let left_hi = widen_hi(left);
let right_lo = widen_lo(right);
let right_hi = widen_hi(right);
let tm_lo = _mm_mullo_epi32(mid_lo, three);
let tm_hi = _mm_mullo_epi32(mid_hi, three);
let even_lo = _mm_srli_epi32::<2>(_mm_add_epi32(_mm_add_epi32(left_lo, tm_lo), two));
let even_hi = _mm_srli_epi32::<2>(_mm_add_epi32(_mm_add_epi32(left_hi, tm_hi), two));
let odd_lo = _mm_srli_epi32::<2>(_mm_add_epi32(_mm_add_epi32(tm_lo, right_lo), two));
let odd_hi = _mm_srli_epi32::<2>(_mm_add_epi32(_mm_add_epi32(tm_hi, right_hi), two));
(
_mm_packus_epi32(even_lo, even_hi),
_mm_packus_epi32(odd_lo, odd_hi),
)
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn chroma_upsample_2to1_center_h_row(
c_half: &[u8],
c_full: &mut [u8],
width: usize,
) {
debug_assert_eq!(width & 1, 0, "2:1 horizontal chroma requires even width");
debug_assert!(c_half.len() >= width / 2, "c_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let half = width / 2;
if half == 0 {
return;
}
scalar::chroma_upsample_2to1_center_h_pair(c_half, c_full, 0, half);
if half == 1 {
return;
}
let mut j = 1usize;
unsafe {
let two = _mm_set1_epi16(2);
let three = _mm_set1_epi16(3);
while j + 16 < half {
let mid = _mm_loadu_si128(c_half.as_ptr().add(j).cast());
let left = _mm_loadu_si128(c_half.as_ptr().add(j - 1).cast());
let right = _mm_loadu_si128(c_half.as_ptr().add(j + 1).cast());
let widen_lo = |v: __m128i| _mm_cvtepu8_epi16(v);
let widen_hi = |v: __m128i| _mm_cvtepu8_epi16(_mm_srli_si128::<8>(v));
let mid_lo = widen_lo(mid);
let mid_hi = widen_hi(mid);
let left_lo = widen_lo(left);
let left_hi = widen_hi(left);
let right_lo = widen_lo(right);
let right_hi = widen_hi(right);
let tm_lo = _mm_mullo_epi16(mid_lo, three);
let tm_hi = _mm_mullo_epi16(mid_hi, three);
let even_lo = _mm_srli_epi16::<2>(_mm_add_epi16(_mm_add_epi16(left_lo, tm_lo), two));
let even_hi = _mm_srli_epi16::<2>(_mm_add_epi16(_mm_add_epi16(left_hi, tm_hi), two));
let odd_lo = _mm_srli_epi16::<2>(_mm_add_epi16(_mm_add_epi16(tm_lo, right_lo), two));
let odd_hi = _mm_srli_epi16::<2>(_mm_add_epi16(_mm_add_epi16(tm_hi, right_hi), two));
let even = _mm_packus_epi16(even_lo, even_hi);
let odd = _mm_packus_epi16(odd_lo, odd_hi);
let out_lo = _mm_unpacklo_epi8(even, odd);
let out_hi = _mm_unpackhi_epi8(even, odd);
_mm_storeu_si128(c_full.as_mut_ptr().add(2 * j).cast(), out_lo);
_mm_storeu_si128(c_full.as_mut_ptr().add(2 * j + 16).cast(), out_hi);
j += 16;
}
}
while j < half {
scalar::chroma_upsample_2to1_center_h_pair(c_half, c_full, j, half);
j += 1;
}
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn chroma_upsample_2to1_center_h_u16_row<const BITS: u32>(
c_half: &[u16],
c_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(width & 1, 0, "2:1 horizontal chroma requires even width");
debug_assert!(c_half.len() >= width / 2, "c_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let half = width / 2;
if half == 0 {
return;
}
scalar::chroma_upsample_2to1_center_h_u16_pair::<BITS>(c_half, c_full, 0, half, big_endian);
if half == 1 {
return;
}
let swap = big_endian != cfg!(target_endian = "big");
let mut j = 1usize;
unsafe {
let bmask = bswap16_mask();
let mask = _mm_set1_epi16(((1u32 << BITS) - 1) as i16);
let norm = |v: __m128i| {
let host = if swap { _mm_shuffle_epi8(v, bmask) } else { v };
_mm_and_si128(host, mask)
};
while j + 8 < half {
let mid = norm(_mm_loadu_si128(c_half.as_ptr().add(j).cast()));
let left = norm(_mm_loadu_si128(c_half.as_ptr().add(j - 1).cast()));
let right = norm(_mm_loadu_si128(c_half.as_ptr().add(j + 1).cast()));
let (even, odd) = blend_u16x8(left, mid, right);
let even = if swap {
_mm_shuffle_epi8(even, bmask)
} else {
even
};
let odd = if swap {
_mm_shuffle_epi8(odd, bmask)
} else {
odd
};
let out_lo = _mm_unpacklo_epi16(even, odd);
let out_hi = _mm_unpackhi_epi16(even, odd);
_mm_storeu_si128(c_full.as_mut_ptr().add(2 * j).cast(), out_lo);
_mm_storeu_si128(c_full.as_mut_ptr().add(2 * j + 8).cast(), out_hi);
j += 8;
}
}
while j < half {
scalar::chroma_upsample_2to1_center_h_u16_pair::<BITS>(c_half, c_full, j, half, big_endian);
j += 1;
}
}
#[cfg(all(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn chroma_upsample_2to1_center_h_p0xx_row<
const BITS: u32,
const LOW_PACKED: bool,
>(
uv_half: &[u16],
uv_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(width & 1, 0, "P-format 4:2:0 requires even width");
debug_assert!(uv_half.len() >= width, "uv_half row too short");
debug_assert!(uv_full.len() >= 2 * width, "uv_full row too short");
let half = width / 2;
if half == 0 {
return;
}
scalar::chroma_upsample_2to1_center_h_p0xx_pair::<BITS, LOW_PACKED>(
uv_half, uv_full, 0, half, big_endian,
);
if half == 1 {
return;
}
let swap = big_endian != cfg!(target_endian = "big");
let mut j = 1usize;
unsafe {
let bmask = bswap16_mask();
let mask32 = _mm_set1_epi32(((1u32 << BITS) - 1) as i32);
let mask16 = _mm_set1_epi16(((1u32 << BITS) - 1) as i16);
let shift_ct = _mm_cvtsi32_si128((16 - BITS) as i32);
let u_mask = _mm_set_epi8(
-128, -128, -128, -128, -128, -128, -128, -128, 13, 12, 9, 8, 5, 4, 1, 0,
);
let v_mask = _mm_set_epi8(
-128, -128, -128, -128, -128, -128, -128, -128, 15, 14, 11, 10, 7, 6, 3, 2,
);
while j + 4 < half {
let load = |off: usize| _mm_loadu_si128(uv_half.as_ptr().add(2 * off).cast());
let ll = load(j - 1);
let mm = load(j);
let rr = load(j + 1);
let deint = |v: __m128i, chan: __m128i| {
let host = if swap { _mm_shuffle_epi8(v, bmask) } else { v };
_mm_cvtepu16_epi32(_mm_shuffle_epi8(host, chan))
};
let depack = |v: __m128i| {
if LOW_PACKED {
_mm_and_si128(v, mask32)
} else {
_mm_srl_epi32(v, shift_ct)
}
};
let ul = depack(deint(ll, u_mask));
let um = depack(deint(mm, u_mask));
let ur = depack(deint(rr, u_mask));
let vl = depack(deint(ll, v_mask));
let vm = depack(deint(mm, v_mask));
let vr = depack(deint(rr, v_mask));
let two = _mm_set1_epi32(2);
let three = _mm_set1_epi32(3);
let blend_even = |a: __m128i, b: __m128i| {
_mm_srli_epi32::<2>(_mm_add_epi32(
_mm_add_epi32(a, _mm_mullo_epi32(b, three)),
two,
))
};
let blend_odd = |b: __m128i, c: __m128i| {
_mm_srli_epi32::<2>(_mm_add_epi32(
_mm_add_epi32(_mm_mullo_epi32(b, three), c),
two,
))
};
let repack = |v: __m128i| {
let packed16 = _mm_packus_epi32(v, _mm_setzero_si128());
let aligned = if LOW_PACKED {
packed16
} else {
_mm_sll_epi16(_mm_and_si128(packed16, mask16), shift_ct)
};
if swap {
_mm_shuffle_epi8(aligned, bmask)
} else {
aligned
}
};
let u_even = repack(blend_even(ul, um));
let u_odd = repack(blend_odd(um, ur));
let v_even = repack(blend_even(vl, vm));
let v_odd = repack(blend_odd(vm, vr));
let uv_even = _mm_unpacklo_epi16(u_even, v_even);
let uv_odd = _mm_unpacklo_epi16(u_odd, v_odd);
let out_lo = _mm_unpacklo_epi32(uv_even, uv_odd);
let out_hi = _mm_unpackhi_epi32(uv_even, uv_odd);
_mm_storeu_si128(uv_full.as_mut_ptr().add(4 * j).cast(), out_lo);
_mm_storeu_si128(uv_full.as_mut_ptr().add(4 * j + 8).cast(), out_hi);
j += 4;
}
}
while j < half {
scalar::chroma_upsample_2to1_center_h_p0xx_pair::<BITS, LOW_PACKED>(
uv_half, uv_full, j, half, big_endian,
);
j += 1;
}
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn chroma_upsample_420_bottom_even_h_row(
prev_half: &[u8],
cur_half: &[u8],
c_full: &mut [u8],
width: usize,
) {
debug_assert_eq!(width & 1, 0, "YUV 4:2:0 requires even width");
debug_assert!(prev_half.len() >= width / 2, "prev_half row too short");
debug_assert!(cur_half.len() >= width / 2, "cur_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let half = width / 2;
if half == 0 {
return;
}
scalar::chroma_upsample_420_bottom_even_h_pair(prev_half, cur_half, c_full, 0, half);
if half == 1 {
return;
}
let mut j = 1usize;
unsafe {
let two = _mm_set1_epi16(2);
let three = _mm_set1_epi16(3);
let avg = |off: usize| {
_mm_avg_epu8(
_mm_loadu_si128(prev_half.as_ptr().add(off).cast()),
_mm_loadu_si128(cur_half.as_ptr().add(off).cast()),
)
};
let widen_lo = |v: __m128i| _mm_cvtepu8_epi16(v);
let widen_hi = |v: __m128i| _mm_cvtepu8_epi16(_mm_srli_si128::<8>(v));
while j + 16 < half {
let e_left = avg(j - 1);
let e_mid = avg(j);
let e_right = avg(j + 1);
let mid_lo = widen_lo(e_mid);
let mid_hi = widen_hi(e_mid);
let left_lo = widen_lo(e_left);
let left_hi = widen_hi(e_left);
let right_lo = widen_lo(e_right);
let right_hi = widen_hi(e_right);
let tm_lo = _mm_mullo_epi16(mid_lo, three);
let tm_hi = _mm_mullo_epi16(mid_hi, three);
let even_lo = _mm_srli_epi16::<2>(_mm_add_epi16(_mm_add_epi16(left_lo, tm_lo), two));
let even_hi = _mm_srli_epi16::<2>(_mm_add_epi16(_mm_add_epi16(left_hi, tm_hi), two));
let odd_lo = _mm_srli_epi16::<2>(_mm_add_epi16(_mm_add_epi16(tm_lo, right_lo), two));
let odd_hi = _mm_srli_epi16::<2>(_mm_add_epi16(_mm_add_epi16(tm_hi, right_hi), two));
let even = _mm_packus_epi16(even_lo, even_hi);
let odd = _mm_packus_epi16(odd_lo, odd_hi);
let out_lo = _mm_unpacklo_epi8(even, odd);
let out_hi = _mm_unpackhi_epi8(even, odd);
_mm_storeu_si128(c_full.as_mut_ptr().add(2 * j).cast(), out_lo);
_mm_storeu_si128(c_full.as_mut_ptr().add(2 * j + 16).cast(), out_hi);
j += 16;
}
}
while j < half {
scalar::chroma_upsample_420_bottom_even_h_pair(prev_half, cur_half, c_full, j, half);
j += 1;
}
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn chroma_upsample_420_bottomleft_even_h_row(
prev_half: &[u8],
cur_half: &[u8],
c_full: &mut [u8],
width: usize,
) {
debug_assert_eq!(width & 1, 0, "YUV 4:2:0 requires even width");
debug_assert!(prev_half.len() >= width / 2, "prev_half row too short");
debug_assert!(cur_half.len() >= width / 2, "cur_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let half = width / 2;
let mut j = 0usize;
unsafe {
while j + 16 <= half {
let e = _mm_avg_epu8(
_mm_loadu_si128(prev_half.as_ptr().add(j).cast()),
_mm_loadu_si128(cur_half.as_ptr().add(j).cast()),
);
let out_lo = _mm_unpacklo_epi8(e, e);
let out_hi = _mm_unpackhi_epi8(e, e);
_mm_storeu_si128(c_full.as_mut_ptr().add(2 * j).cast(), out_lo);
_mm_storeu_si128(c_full.as_mut_ptr().add(2 * j + 16).cast(), out_hi);
j += 16;
}
}
while j < half {
scalar::chroma_upsample_420_bottomleft_even_h_pair(prev_half, cur_half, c_full, j);
j += 1;
}
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn chroma_upsample_420_bottom_even_h_u16_row<const BITS: u32>(
prev_half: &[u16],
cur_half: &[u16],
c_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(width & 1, 0, "YUV 4:2:0 requires even width");
debug_assert!(prev_half.len() >= width / 2, "prev_half row too short");
debug_assert!(cur_half.len() >= width / 2, "cur_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let half = width / 2;
if half == 0 {
return;
}
scalar::chroma_upsample_420_bottom_even_h_u16_pair::<BITS>(
prev_half, cur_half, c_full, 0, half, big_endian,
);
if half == 1 {
return;
}
let swap = big_endian != cfg!(target_endian = "big");
let mut j = 1usize;
unsafe {
let bmask = bswap16_mask();
let mask = _mm_set1_epi16(((1u32 << BITS) - 1) as i16);
let norm = |row: &[u16], off: usize| {
let raw = _mm_loadu_si128(row.as_ptr().add(off).cast());
let host = if swap {
_mm_shuffle_epi8(raw, bmask)
} else {
raw
};
_mm_and_si128(host, mask)
};
let vavg = |off: usize| _mm_avg_epu16(norm(prev_half, off), norm(cur_half, off));
while j + 8 < half {
let e_left = vavg(j - 1);
let e_mid = vavg(j);
let e_right = vavg(j + 1);
let (even, odd) = blend_u16x8(e_left, e_mid, e_right);
let even = if swap {
_mm_shuffle_epi8(even, bmask)
} else {
even
};
let odd = if swap {
_mm_shuffle_epi8(odd, bmask)
} else {
odd
};
let out_lo = _mm_unpacklo_epi16(even, odd);
let out_hi = _mm_unpackhi_epi16(even, odd);
_mm_storeu_si128(c_full.as_mut_ptr().add(2 * j).cast(), out_lo);
_mm_storeu_si128(c_full.as_mut_ptr().add(2 * j + 8).cast(), out_hi);
j += 8;
}
}
while j < half {
scalar::chroma_upsample_420_bottom_even_h_u16_pair::<BITS>(
prev_half, cur_half, c_full, j, half, big_endian,
);
j += 1;
}
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn chroma_upsample_420_bottomleft_even_h_u16_row<const BITS: u32>(
prev_half: &[u16],
cur_half: &[u16],
c_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(width & 1, 0, "YUV 4:2:0 requires even width");
debug_assert!(prev_half.len() >= width / 2, "prev_half row too short");
debug_assert!(cur_half.len() >= width / 2, "cur_half row too short");
debug_assert!(c_full.len() >= width, "c_full row too short");
let half = width / 2;
let swap = big_endian != cfg!(target_endian = "big");
let mut j = 0usize;
unsafe {
let bmask = bswap16_mask();
let mask = _mm_set1_epi16(((1u32 << BITS) - 1) as i16);
let norm = |row: &[u16], off: usize| {
let raw = _mm_loadu_si128(row.as_ptr().add(off).cast());
let host = if swap {
_mm_shuffle_epi8(raw, bmask)
} else {
raw
};
_mm_and_si128(host, mask)
};
while j + 8 <= half {
let e = _mm_avg_epu16(norm(prev_half, j), norm(cur_half, j));
let e = if swap { _mm_shuffle_epi8(e, bmask) } else { e };
let out_lo = _mm_unpacklo_epi16(e, e);
let out_hi = _mm_unpackhi_epi16(e, e);
_mm_storeu_si128(c_full.as_mut_ptr().add(2 * j).cast(), out_lo);
_mm_storeu_si128(c_full.as_mut_ptr().add(2 * j + 8).cast(), out_hi);
j += 8;
}
}
while j < half {
scalar::chroma_upsample_420_bottomleft_even_h_u16_pair::<BITS>(
prev_half, cur_half, c_full, j, big_endian,
);
j += 1;
}
}
#[cfg(all(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn chroma_upsample_420_bottom_even_h_p0xx_row<const BITS: u32>(
prev_uv_half: &[u16],
cur_uv_half: &[u16],
uv_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(width & 1, 0, "P-format 4:2:0 requires even width");
debug_assert!(prev_uv_half.len() >= width, "prev_uv_half row too short");
debug_assert!(cur_uv_half.len() >= width, "cur_uv_half row too short");
debug_assert!(uv_full.len() >= 2 * width, "uv_full row too short");
let half = width / 2;
if half == 0 {
return;
}
scalar::chroma_upsample_420_bottom_even_h_p0xx_pair::<BITS>(
prev_uv_half,
cur_uv_half,
uv_full,
0,
half,
big_endian,
);
if half == 1 {
return;
}
let swap = big_endian != cfg!(target_endian = "big");
let mut j = 1usize;
unsafe {
let bmask = bswap16_mask();
let mask16 = _mm_set1_epi16(((1u32 << BITS) - 1) as i16);
let shift_ct = _mm_cvtsi32_si128((16 - BITS) as i32);
let one = _mm_set1_epi32(1);
let two = _mm_set1_epi32(2);
let three = _mm_set1_epi32(3);
let u_mask = _mm_set_epi8(
-128, -128, -128, -128, -128, -128, -128, -128, 13, 12, 9, 8, 5, 4, 1, 0,
);
let v_mask = _mm_set_epi8(
-128, -128, -128, -128, -128, -128, -128, -128, 15, 14, 11, 10, 7, 6, 3, 2,
);
let dp = |row: &[u16], off: usize, chan: __m128i| {
let raw = _mm_loadu_si128(row.as_ptr().add(2 * off).cast());
let host = if swap {
_mm_shuffle_epi8(raw, bmask)
} else {
raw
};
_mm_srl_epi32(_mm_cvtepu16_epi32(_mm_shuffle_epi8(host, chan)), shift_ct)
};
let vavg =
|a: __m128i, b: __m128i| _mm_srli_epi32::<1>(_mm_add_epi32(_mm_add_epi32(a, b), one));
let blend_even = |a: __m128i, b: __m128i| {
_mm_srli_epi32::<2>(_mm_add_epi32(
_mm_add_epi32(a, _mm_mullo_epi32(b, three)),
two,
))
};
let blend_odd = |b: __m128i, c: __m128i| {
_mm_srli_epi32::<2>(_mm_add_epi32(
_mm_add_epi32(_mm_mullo_epi32(b, three), c),
two,
))
};
let repack = |v: __m128i| {
let packed16 = _mm_packus_epi32(v, _mm_setzero_si128());
let aligned = _mm_sll_epi16(_mm_and_si128(packed16, mask16), shift_ct);
if swap {
_mm_shuffle_epi8(aligned, bmask)
} else {
aligned
}
};
while j + 4 < half {
let ul = vavg(
dp(prev_uv_half, j - 1, u_mask),
dp(cur_uv_half, j - 1, u_mask),
);
let um = vavg(dp(prev_uv_half, j, u_mask), dp(cur_uv_half, j, u_mask));
let ur = vavg(
dp(prev_uv_half, j + 1, u_mask),
dp(cur_uv_half, j + 1, u_mask),
);
let vl = vavg(
dp(prev_uv_half, j - 1, v_mask),
dp(cur_uv_half, j - 1, v_mask),
);
let vm = vavg(dp(prev_uv_half, j, v_mask), dp(cur_uv_half, j, v_mask));
let vr = vavg(
dp(prev_uv_half, j + 1, v_mask),
dp(cur_uv_half, j + 1, v_mask),
);
let u_even = repack(blend_even(ul, um));
let u_odd = repack(blend_odd(um, ur));
let v_even = repack(blend_even(vl, vm));
let v_odd = repack(blend_odd(vm, vr));
let uv_even = _mm_unpacklo_epi16(u_even, v_even);
let uv_odd = _mm_unpacklo_epi16(u_odd, v_odd);
let out_lo = _mm_unpacklo_epi32(uv_even, uv_odd);
let out_hi = _mm_unpackhi_epi32(uv_even, uv_odd);
_mm_storeu_si128(uv_full.as_mut_ptr().add(4 * j).cast(), out_lo);
_mm_storeu_si128(uv_full.as_mut_ptr().add(4 * j + 8).cast(), out_hi);
j += 4;
}
}
while j < half {
scalar::chroma_upsample_420_bottom_even_h_p0xx_pair::<BITS>(
prev_uv_half,
cur_uv_half,
uv_full,
j,
half,
big_endian,
);
j += 1;
}
}
#[cfg(all(feature = "yuv-planar", feature = "yuv-semi-planar"))]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn chroma_upsample_420_bottomleft_even_h_p0xx_row<const BITS: u32>(
prev_uv_half: &[u16],
cur_uv_half: &[u16],
uv_full: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert_eq!(width & 1, 0, "P-format 4:2:0 requires even width");
debug_assert!(prev_uv_half.len() >= width, "prev_uv_half row too short");
debug_assert!(cur_uv_half.len() >= width, "cur_uv_half row too short");
debug_assert!(uv_full.len() >= 2 * width, "uv_full row too short");
let half = width / 2;
let swap = big_endian != cfg!(target_endian = "big");
let mut j = 0usize;
unsafe {
let bmask = bswap16_mask();
let mask16 = _mm_set1_epi16(((1u32 << BITS) - 1) as i16);
let shift_ct = _mm_cvtsi32_si128((16 - BITS) as i32);
let one = _mm_set1_epi32(1);
let u_mask = _mm_set_epi8(
-128, -128, -128, -128, -128, -128, -128, -128, 13, 12, 9, 8, 5, 4, 1, 0,
);
let v_mask = _mm_set_epi8(
-128, -128, -128, -128, -128, -128, -128, -128, 15, 14, 11, 10, 7, 6, 3, 2,
);
let dp = |row: &[u16], off: usize, chan: __m128i| {
let raw = _mm_loadu_si128(row.as_ptr().add(2 * off).cast());
let host = if swap {
_mm_shuffle_epi8(raw, bmask)
} else {
raw
};
_mm_srl_epi32(_mm_cvtepu16_epi32(_mm_shuffle_epi8(host, chan)), shift_ct)
};
let vavg =
|a: __m128i, b: __m128i| _mm_srli_epi32::<1>(_mm_add_epi32(_mm_add_epi32(a, b), one));
let repack = |v: __m128i| {
let packed16 = _mm_packus_epi32(v, _mm_setzero_si128());
let aligned = _mm_sll_epi16(_mm_and_si128(packed16, mask16), shift_ct);
if swap {
_mm_shuffle_epi8(aligned, bmask)
} else {
aligned
}
};
while j + 4 <= half {
let u = repack(vavg(
dp(prev_uv_half, j, u_mask),
dp(cur_uv_half, j, u_mask),
));
let v = repack(vavg(
dp(prev_uv_half, j, v_mask),
dp(cur_uv_half, j, v_mask),
));
let uv = _mm_unpacklo_epi16(u, v);
let out_lo = _mm_unpacklo_epi32(uv, uv);
let out_hi = _mm_unpackhi_epi32(uv, uv);
_mm_storeu_si128(uv_full.as_mut_ptr().add(4 * j).cast(), out_lo);
_mm_storeu_si128(uv_full.as_mut_ptr().add(4 * j + 8).cast(), out_hi);
j += 4;
}
}
while j < half {
scalar::chroma_upsample_420_bottomleft_even_h_p0xx_pair::<BITS>(
prev_uv_half,
cur_uv_half,
uv_full,
j,
big_endian,
);
j += 1;
}
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn chroma_upsample_440_bottom_v_row(
prev: &[u8],
cur: &[u8],
out: &mut [u8],
width: usize,
) {
debug_assert!(prev.len() >= width, "prev row too short");
debug_assert!(cur.len() >= width, "cur row too short");
debug_assert!(out.len() >= width, "out row too short");
let mut j = 0;
unsafe {
while j + 16 <= width {
let p = _mm_loadu_si128(prev.as_ptr().add(j).cast());
let c = _mm_loadu_si128(cur.as_ptr().add(j).cast());
_mm_storeu_si128(out.as_mut_ptr().add(j).cast(), _mm_avg_epu8(p, c));
j += 16;
}
}
while j < width {
out[j] = (((prev[j] as u16) + (cur[j] as u16) + 1) >> 1) as u8;
j += 1;
}
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn chroma_upsample_440_bottom_v_u16_row<const BITS: u32>(
prev: &[u16],
cur: &[u16],
out: &mut [u16],
width: usize,
big_endian: bool,
) {
debug_assert!(prev.len() >= width, "prev row too short");
debug_assert!(cur.len() >= width, "cur row too short");
debug_assert!(out.len() >= width, "out row too short");
let swap = big_endian != cfg!(target_endian = "big");
let mut j = 0;
unsafe {
let bmask = bswap16_mask();
let mask = _mm_set1_epi16(((1u32 << BITS) - 1) as i16);
let norm = |raw: __m128i| {
let host = if swap {
_mm_shuffle_epi8(raw, bmask)
} else {
raw
};
_mm_and_si128(host, mask)
};
while j + 8 <= width {
let p = norm(_mm_loadu_si128(prev.as_ptr().add(j).cast()));
let c = norm(_mm_loadu_si128(cur.as_ptr().add(j).cast()));
let avg = _mm_avg_epu16(p, c);
let enc = if swap {
_mm_shuffle_epi8(avg, bmask)
} else {
avg
};
_mm_storeu_si128(out.as_mut_ptr().add(j).cast(), enc);
j += 8;
}
}
let mask = ((1u32 << BITS) - 1) as u16;
let load = |raw: u16| -> u32 {
let logical = if big_endian {
u16::from_be(raw)
} else {
u16::from_le(raw)
};
u32::from(logical & mask)
};
while j < width {
let blended = ((load(prev[j]) + load(cur[j]) + 1) >> 1) as u16;
out[j] = if big_endian {
blended.to_be()
} else {
blended.to_le()
};
j += 1;
}
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "sse4.1")]
fn blend4_u8x16(
left: __m128i,
mid: __m128i,
right: __m128i,
) -> (__m128i, __m128i, __m128i, __m128i) {
let four = _mm_set1_epi16(4);
let widen_lo = |v: __m128i| _mm_cvtepu8_epi16(v);
let widen_hi = |v: __m128i| _mm_cvtepu8_epi16(_mm_srli_si128::<8>(v));
let ll = widen_lo(left);
let lh = widen_hi(left);
let ml = widen_lo(mid);
let mh = widen_hi(mid);
let rl = widen_lo(right);
let rh = widen_hi(right);
let phase = |l: __m128i, wl: i16, m: __m128i, wm: i16, r: __m128i, wr: i16| {
let acc = _mm_add_epi16(
_mm_add_epi16(
_mm_add_epi16(
_mm_mullo_epi16(l, _mm_set1_epi16(wl)),
_mm_mullo_epi16(m, _mm_set1_epi16(wm)),
),
_mm_mullo_epi16(r, _mm_set1_epi16(wr)),
),
four,
);
_mm_srli_epi16::<3>(acc)
};
let a = _mm_packus_epi16(phase(ll, 3, ml, 5, rl, 0), phase(lh, 3, mh, 5, rh, 0));
let b = _mm_packus_epi16(phase(ll, 1, ml, 7, rl, 0), phase(lh, 1, mh, 7, rh, 0));
let c = _mm_packus_epi16(phase(ll, 0, ml, 7, rl, 1), phase(lh, 0, mh, 7, rh, 1));
let d = _mm_packus_epi16(phase(ll, 0, ml, 5, rl, 3), phase(lh, 0, mh, 5, rh, 3));
(a, b, c, d)
}
#[cfg(feature = "yuv-planar")]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn chroma_upsample_4to1_center_h_row(
c_quarter: &[u8],
c_full: &mut [u8],
width: usize,
) {
debug_assert!(
c_quarter.len() >= width.div_ceil(4),
"c_quarter row too short"
);
debug_assert!(c_full.len() >= width, "c_full row too short");
let quarter = width.div_ceil(4);
if quarter == 0 {
return;
}
scalar::chroma_upsample_4to1_center_h_group(c_quarter, c_full, 0, quarter, width);
if quarter == 1 {
return;
}
let mut j = 1usize;
unsafe {
while j + 16 < quarter {
let left = _mm_loadu_si128(c_quarter.as_ptr().add(j - 1).cast());
let mid = _mm_loadu_si128(c_quarter.as_ptr().add(j).cast());
let right = _mm_loadu_si128(c_quarter.as_ptr().add(j + 1).cast());
let (a, b, c, d) = blend4_u8x16(left, mid, right);
let ab0 = _mm_unpacklo_epi8(a, b);
let ab1 = _mm_unpackhi_epi8(a, b);
let cd0 = _mm_unpacklo_epi8(c, d);
let cd1 = _mm_unpackhi_epi8(c, d);
let base = 4 * j;
_mm_storeu_si128(
c_full.as_mut_ptr().add(base).cast(),
_mm_unpacklo_epi16(ab0, cd0),
);
_mm_storeu_si128(
c_full.as_mut_ptr().add(base + 16).cast(),
_mm_unpackhi_epi16(ab0, cd0),
);
_mm_storeu_si128(
c_full.as_mut_ptr().add(base + 32).cast(),
_mm_unpacklo_epi16(ab1, cd1),
);
_mm_storeu_si128(
c_full.as_mut_ptr().add(base + 48).cast(),
_mm_unpackhi_epi16(ab1, cd1),
);
j += 16;
}
}
while j < quarter {
scalar::chroma_upsample_4to1_center_h_group(c_quarter, c_full, j, quarter, width);
j += 1;
}
}
#[cfg(all(
test,
feature = "std",
feature = "yuv-planar",
feature = "yuv-semi-planar"
))]
mod tests {
use crate::row::scalar;
fn pseudo_random_u16(out: &mut [u16], seed: u32) {
let mut state = seed;
for v in out.iter_mut() {
state = state.wrapping_mul(1664525).wrapping_add(1013904223);
*v = (state >> 8) as u16;
}
}
const WIDTHS: &[usize] = &[2, 4, 6, 8, 16, 18, 30, 32, 34, 62, 64, 66, 128, 130];
fn check_p0xx<const BITS: u32, const LOW_PACKED: bool>(big_endian: bool) {
for &w in WIDTHS {
let mut uv_half = std::vec![0u16; w];
pseudo_random_u16(
&mut uv_half,
0x9E37 ^ BITS ^ ((LOW_PACKED as u32) << 8) ^ (big_endian as u32),
);
let mut out_simd = std::vec![0u16; 2 * w];
let mut out_scalar = std::vec![0u16; 2 * w];
unsafe {
super::chroma_upsample_2to1_center_h_p0xx_row::<BITS, LOW_PACKED>(
&uv_half,
&mut out_simd,
w,
big_endian,
)
};
scalar::chroma_upsample_2to1_center_h_p0xx::<BITS, LOW_PACKED>(
&uv_half,
&mut out_scalar,
w,
big_endian,
);
assert_eq!(
out_simd, out_scalar,
"p0xx BITS={BITS} low_packed={LOW_PACKED} be={big_endian} width={w}"
);
}
}
#[test]
#[cfg_attr(
miri,
ignore = "SIMD-dispatched row kernels use intrinsics unsupported by Miri"
)]
fn sse41_p0xx_matches_scalar_widths() {
if !std::arch::is_x86_feature_detected!("sse4.1") {
return;
}
check_p0xx::<10, false>(false);
check_p0xx::<10, false>(true);
check_p0xx::<10, true>(false);
check_p0xx::<10, true>(true);
check_p0xx::<12, false>(false);
check_p0xx::<16, false>(false);
check_p0xx::<16, false>(true);
}
fn check_p0xx_vertical<const BITS: u32>(big_endian: bool) {
for &w in WIDTHS {
let mut prev = std::vec![0u16; w];
let mut cur = std::vec![0u16; w];
pseudo_random_u16(&mut prev, 0x7E57 ^ BITS ^ (big_endian as u32));
pseudo_random_u16(&mut cur, 0xABCD ^ BITS ^ (big_endian as u32));
for (prev_row, tag) in [(prev.as_slice(), "interior"), (cur.as_slice(), "topedge")] {
let mut bot_simd = std::vec![0u16; 2 * w];
let mut bot_scalar = std::vec![0u16; 2 * w];
let mut bl_simd = std::vec![0u16; 2 * w];
let mut bl_scalar = std::vec![0u16; 2 * w];
unsafe {
super::chroma_upsample_420_bottom_even_h_p0xx_row::<BITS>(
prev_row,
&cur,
&mut bot_simd,
w,
big_endian,
);
super::chroma_upsample_420_bottomleft_even_h_p0xx_row::<BITS>(
prev_row,
&cur,
&mut bl_simd,
w,
big_endian,
);
}
scalar::chroma_upsample_420_bottom_even_h_p0xx::<BITS>(
prev_row,
&cur,
&mut bot_scalar,
w,
big_endian,
);
scalar::chroma_upsample_420_bottomleft_even_h_p0xx::<BITS>(
prev_row,
&cur,
&mut bl_scalar,
w,
big_endian,
);
assert_eq!(
bot_simd, bot_scalar,
"p0xx bottom BITS={BITS} be={big_endian} {tag} width={w}"
);
assert_eq!(
bl_simd, bl_scalar,
"p0xx bottomleft BITS={BITS} be={big_endian} {tag} width={w}"
);
}
}
}
#[test]
#[cfg_attr(
miri,
ignore = "SIMD-dispatched row kernels use intrinsics unsupported by Miri"
)]
fn sse41_p0xx_vertical_matches_scalar_widths() {
if !std::arch::is_x86_feature_detected!("sse4.1") {
return;
}
check_p0xx_vertical::<10>(false);
check_p0xx_vertical::<10>(true);
check_p0xx_vertical::<12>(false);
check_p0xx_vertical::<12>(true);
check_p0xx_vertical::<16>(false);
check_p0xx_vertical::<16>(true);
}
}
#[cfg(all(test, feature = "std", feature = "yuv-planar"))]
mod tests_planar {
use crate::row::scalar;
fn pseudo_random_u8(out: &mut [u8], seed: u32) {
let mut state = seed;
for v in out.iter_mut() {
state = state.wrapping_mul(1664525).wrapping_add(1013904223);
*v = (state >> 16) as u8;
}
}
fn pseudo_random_u16(out: &mut [u16], seed: u32) {
let mut state = seed;
for v in out.iter_mut() {
state = state.wrapping_mul(1664525).wrapping_add(1013904223);
*v = (state >> 8) as u16;
}
}
const WIDTHS_440: &[usize] = &[
1, 2, 3, 4, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, 128, 129, 1920,
];
const WIDTHS_4TO1: &[usize] = &[
1, 2, 3, 4, 5, 6, 7, 8, 16, 20, 63, 64, 65, 66, 67, 68, 69, 72, 73, 128, 129, 130, 131, 260,
];
const WIDTHS: &[usize] = &[2, 4, 6, 8, 16, 18, 30, 32, 34, 62, 64, 66, 128, 130];
#[test]
#[cfg_attr(
miri,
ignore = "SIMD-dispatched row kernels use intrinsics unsupported by Miri"
)]
fn sse41_u8_matches_scalar_widths() {
if !std::arch::is_x86_feature_detected!("sse4.1") {
return;
}
for &w in WIDTHS {
let half = w / 2;
let mut c_half = std::vec![0u8; half];
let mut state = 0xC0FFEEu32;
for v in c_half.iter_mut() {
state = state.wrapping_mul(1664525).wrapping_add(1013904223);
*v = (state >> 16) as u8;
}
let mut out_simd = std::vec![0u8; w];
let mut out_scalar = std::vec![0u8; w];
unsafe { super::chroma_upsample_2to1_center_h_row(&c_half, &mut out_simd, w) };
scalar::chroma_upsample_2to1_center_h(&c_half, &mut out_scalar, w);
assert_eq!(out_simd, out_scalar, "u8 width={w}");
}
}
fn check_u16<const BITS: u32>(big_endian: bool) {
for &w in WIDTHS {
let half = w / 2;
let mut c_half = std::vec![0u16; half];
pseudo_random_u16(&mut c_half, 0x1234 ^ BITS ^ (big_endian as u32));
let mut out_simd = std::vec![0u16; w];
let mut out_scalar = std::vec![0u16; w];
unsafe {
super::chroma_upsample_2to1_center_h_u16_row::<BITS>(&c_half, &mut out_simd, w, big_endian)
};
scalar::chroma_upsample_2to1_center_h_u16::<BITS>(&c_half, &mut out_scalar, w, big_endian);
assert_eq!(
out_simd, out_scalar,
"u16 BITS={BITS} be={big_endian} width={w}"
);
}
}
#[test]
#[cfg_attr(
miri,
ignore = "SIMD-dispatched row kernels use intrinsics unsupported by Miri"
)]
fn sse41_u16_matches_scalar_widths() {
if !std::arch::is_x86_feature_detected!("sse4.1") {
return;
}
check_u16::<10>(false);
check_u16::<10>(true);
check_u16::<12>(false);
check_u16::<12>(true);
check_u16::<16>(false);
check_u16::<16>(true);
}
#[test]
#[cfg_attr(
miri,
ignore = "SIMD-dispatched row kernels use intrinsics unsupported by Miri"
)]
fn sse41_u8_vertical_matches_scalar_widths() {
if !std::arch::is_x86_feature_detected!("sse4.1") {
return;
}
for &w in WIDTHS {
let half = w / 2;
let mut prev = std::vec![0u8; half];
let mut cur = std::vec![0u8; half];
pseudo_random_u8(&mut prev, 0xBEEF);
pseudo_random_u8(&mut cur, 0xFACE);
for (prev_row, tag) in [(prev.as_slice(), "interior"), (cur.as_slice(), "topedge")] {
let mut bot_simd = std::vec![0u8; w];
let mut bot_scalar = std::vec![0u8; w];
let mut bl_simd = std::vec![0u8; w];
let mut bl_scalar = std::vec![0u8; w];
unsafe {
super::chroma_upsample_420_bottom_even_h_row(prev_row, &cur, &mut bot_simd, w);
super::chroma_upsample_420_bottomleft_even_h_row(prev_row, &cur, &mut bl_simd, w);
}
scalar::chroma_upsample_420_bottom_even_h(prev_row, &cur, &mut bot_scalar, w);
scalar::chroma_upsample_420_bottomleft_even_h(prev_row, &cur, &mut bl_scalar, w);
assert_eq!(bot_simd, bot_scalar, "u8 bottom {tag} width={w}");
assert_eq!(bl_simd, bl_scalar, "u8 bottomleft {tag} width={w}");
}
}
}
fn check_u16_vertical<const BITS: u32>(big_endian: bool) {
for &w in WIDTHS {
let half = w / 2;
let mut prev = std::vec![0u16; half];
let mut cur = std::vec![0u16; half];
pseudo_random_u16(&mut prev, 0x51A5 ^ BITS ^ (big_endian as u32));
pseudo_random_u16(&mut cur, 0xC0DE ^ BITS ^ (big_endian as u32));
for (prev_row, tag) in [(prev.as_slice(), "interior"), (cur.as_slice(), "topedge")] {
let mut bot_simd = std::vec![0u16; w];
let mut bot_scalar = std::vec![0u16; w];
let mut bl_simd = std::vec![0u16; w];
let mut bl_scalar = std::vec![0u16; w];
unsafe {
super::chroma_upsample_420_bottom_even_h_u16_row::<BITS>(
prev_row,
&cur,
&mut bot_simd,
w,
big_endian,
);
super::chroma_upsample_420_bottomleft_even_h_u16_row::<BITS>(
prev_row,
&cur,
&mut bl_simd,
w,
big_endian,
);
}
scalar::chroma_upsample_420_bottom_even_h_u16::<BITS>(
prev_row,
&cur,
&mut bot_scalar,
w,
big_endian,
);
scalar::chroma_upsample_420_bottomleft_even_h_u16::<BITS>(
prev_row,
&cur,
&mut bl_scalar,
w,
big_endian,
);
assert_eq!(
bot_simd, bot_scalar,
"u16 bottom BITS={BITS} be={big_endian} {tag} width={w}"
);
assert_eq!(
bl_simd, bl_scalar,
"u16 bottomleft BITS={BITS} be={big_endian} {tag} width={w}"
);
}
}
}
#[test]
#[cfg_attr(
miri,
ignore = "SIMD-dispatched row kernels use intrinsics unsupported by Miri"
)]
fn sse41_u16_vertical_matches_scalar_widths() {
if !std::arch::is_x86_feature_detected!("sse4.1") {
return;
}
check_u16_vertical::<10>(false);
check_u16_vertical::<10>(true);
check_u16_vertical::<12>(false);
check_u16_vertical::<12>(true);
check_u16_vertical::<16>(false);
check_u16_vertical::<16>(true);
}
#[test]
#[cfg_attr(
miri,
ignore = "SIMD-dispatched row kernels use intrinsics unsupported by Miri"
)]
fn sse41_440_bottom_v_matches_scalar_widths() {
if !std::arch::is_x86_feature_detected!("sse4.1") {
return;
}
for &w in WIDTHS_440 {
let mut prev = std::vec![0u8; w];
let mut cur = std::vec![0u8; w];
pseudo_random_u8(&mut prev, 0x440B);
pseudo_random_u8(&mut cur, 0x440C);
for (prev_row, tag) in [(prev.as_slice(), "interior"), (cur.as_slice(), "topedge")] {
let mut simd = std::vec![0u8; w];
let mut sc = std::vec![0u8; w];
unsafe { super::chroma_upsample_440_bottom_v_row(prev_row, &cur, &mut simd, w) };
scalar::chroma_upsample_440_bottom_v(prev_row, &cur, &mut sc, w);
assert_eq!(simd, sc, "u8 440 {tag} width={w}");
}
}
}
fn check_440_u16<const BITS: u32>(big_endian: bool) {
for &w in WIDTHS_440 {
let mut prev = std::vec![0u16; w];
let mut cur = std::vec![0u16; w];
pseudo_random_u16(&mut prev, 0x4416 ^ BITS ^ (big_endian as u32));
pseudo_random_u16(&mut cur, 0x4417 ^ BITS ^ (big_endian as u32));
for (prev_row, tag) in [(prev.as_slice(), "interior"), (cur.as_slice(), "topedge")] {
let mut simd = std::vec![0u16; w];
let mut sc = std::vec![0u16; w];
unsafe {
super::chroma_upsample_440_bottom_v_u16_row::<BITS>(
prev_row, &cur, &mut simd, w, big_endian,
)
};
scalar::chroma_upsample_440_bottom_v_u16_wire::<BITS>(
prev_row, &cur, &mut sc, w, big_endian,
);
assert_eq!(
simd, sc,
"u16 440 BITS={BITS} be={big_endian} {tag} width={w}"
);
}
}
}
#[test]
#[cfg_attr(
miri,
ignore = "SIMD-dispatched row kernels use intrinsics unsupported by Miri"
)]
fn sse41_440_bottom_v_u16_matches_scalar_widths() {
if !std::arch::is_x86_feature_detected!("sse4.1") {
return;
}
check_440_u16::<10>(false);
check_440_u16::<10>(true);
check_440_u16::<12>(false);
check_440_u16::<12>(true);
check_440_u16::<16>(false);
check_440_u16::<16>(true);
}
#[test]
#[cfg_attr(
miri,
ignore = "SIMD-dispatched row kernels use intrinsics unsupported by Miri"
)]
fn sse41_4to1_center_h_matches_scalar_widths() {
if !std::arch::is_x86_feature_detected!("sse4.1") {
return;
}
for &w in WIDTHS_4TO1 {
let quarter = w.div_ceil(4);
let mut cq = std::vec![0u8; quarter];
pseudo_random_u8(&mut cq, 0x4701 ^ w as u32);
let mut simd = std::vec![0u8; w];
let mut sc = std::vec![0u8; w];
unsafe { super::chroma_upsample_4to1_center_h_row(&cq, &mut simd, w) };
scalar::chroma_upsample_4to1_center_h(&cq, &mut sc, w);
assert_eq!(simd, sc, "u8 4to1 width={w}");
}
}
}