#[cfg(target_arch = "x86")]
use std::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use std::arch::x86_64::*;
#[derive(Clone, Copy)]
struct V8(__m128i, __m128i);
trait WarpPx: Copy {
fn load8_i32(s: &[Self]) -> V8;
}
impl WarpPx for u8 {
#[inline(always)]
fn load8_i32(s: &[u8]) -> V8 {
let s = &s[..8];
unsafe {
let v = _mm_loadl_epi64(s.as_ptr().cast());
V8(
_mm_cvtepu8_epi32(v),
_mm_cvtepu8_epi32(_mm_srli_si128::<4>(v)),
)
}
}
}
impl WarpPx for u16 {
#[inline(always)]
fn load8_i32(s: &[u16]) -> V8 {
let s = &s[..8];
unsafe {
let v = _mm_loadu_si128(s.as_ptr().cast());
V8(
_mm_cvtepu16_epi32(v),
_mm_cvtepu16_epi32(_mm_srli_si128::<8>(v)),
)
}
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn coeff(pos: i32, step: i32, tap: usize) -> V8 {
let f = |k: i32| {
crate::tables::MC_WARP_FILTER[(192 + ((pos + step * k + 512) >> 10)) as usize][tap] as i32
};
V8(
_mm_setr_epi32(f(0), f(1), f(2), f(3)),
_mm_setr_epi32(f(4), f(5), f(6), f(7)),
)
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn madd(acc: V8, px: V8, c: V8) -> V8 {
V8(
_mm_add_epi32(acc.0, _mm_mullo_epi32(px.0, c.0)),
_mm_add_epi32(acc.1, _mm_mullo_epi32(px.1, c.1)),
)
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn round(v: V8, rnd: i32, shift: i32) -> V8 {
let (r, s) = (_mm_set1_epi32(rnd), _mm_cvtsi32_si128(shift));
V8(
_mm_sra_epi32(_mm_add_epi32(v.0, r), s),
_mm_sra_epi32(_mm_add_epi32(v.1, r), s),
)
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn horz<P: WarpPx>(
mid: &mut [i16; 15 * 8],
src: &[P],
src_stride: usize,
src_off: usize,
mut mx: i32,
alpha: i32,
beta: i32,
rnd: i32,
shift: i32,
) {
let mut row_base = src_off.wrapping_sub(3 * src_stride + 3);
for mid_row in mid.as_chunks_mut::<8>().0.iter_mut() {
let z = _mm_setzero_si128();
let mut acc = V8(z, z);
for tap in 0..8 {
acc = madd(
acc,
P::load8_i32(&src[row_base + tap..]),
coeff(mx, alpha, tap),
);
}
let v = round(acc, rnd, shift);
unsafe { _mm_storeu_si128(mid_row.as_mut_ptr().cast(), _mm_packs_epi32(v.0, v.1)) };
row_base += src_stride;
mx += beta;
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn vert(mid: &[i16; 15 * 8], y: usize, my: i32, gamma: i32) -> V8 {
let z = _mm_setzero_si128();
let mut acc = V8(z, z);
for tap in 0..8 {
let row = &mid[(y + tap) * 8..(y + tap) * 8 + 8];
let v = unsafe { _mm_loadu_si128(row.as_ptr().cast()) };
let px = V8(
_mm_cvtepi16_epi32(v),
_mm_cvtepi16_epi32(_mm_srli_si128::<8>(v)),
);
acc = madd(acc, px, coeff(my, gamma, tap));
}
acc
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "sse4.1")]
pub(crate) fn warp_affine_8x8_8bpc_sse41(
dst: &mut [u8],
dst_stride: usize,
src: &[u8],
src_stride: usize,
src_off: usize,
abcd: &[i16; 4],
mx: i32,
mut my: i32,
) {
let [alpha, beta, gamma, delta] = abcd.map(i32::from);
let mut mid = [0i16; 15 * 8];
horz(&mut mid, src, src_stride, src_off, mx, alpha, beta, 4, 3);
for (y, dst_row) in dst.chunks_mut(dst_stride).take(8).enumerate() {
let v = round(vert(&mid, y, my, gamma), 1024, 11);
let p16 = _mm_packus_epi32(v.0, v.1);
unsafe { _mm_storel_epi64(dst_row[..8].as_mut_ptr().cast(), _mm_packus_epi16(p16, p16)) };
my += delta;
}
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "sse4.1")]
pub(crate) fn warp_affine_8x8t_8bpc_sse41(
tmp: &mut [i16],
tmp_stride: usize,
src: &[u8],
src_stride: usize,
src_off: usize,
abcd: &[i16; 4],
mx: i32,
mut my: i32,
) {
let [alpha, beta, gamma, delta] = abcd.map(i32::from);
let mut mid = [0i16; 15 * 8];
horz(&mut mid, src, src_stride, src_off, mx, alpha, beta, 4, 3);
for (y, tmp_row) in tmp.chunks_mut(tmp_stride).take(8).enumerate() {
let v = round(vert(&mid, y, my, gamma), 64, 7);
unsafe { _mm_storeu_si128(tmp_row[..8].as_mut_ptr().cast(), _mm_packs_epi32(v.0, v.1)) };
my += delta;
}
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "sse4.1")]
pub(crate) fn warp_affine_8x8_hbd_sse41(
dst: &mut [u16],
dst_stride: usize,
src: &[u16],
src_stride: usize,
src_off: usize,
abcd: &[i16; 4],
mx: i32,
mut my: i32,
bitdepth: u8,
) {
let ib = 14 - bitdepth as i32;
let (h_shift, v_shift) = (7 - ib, 7 + ib);
let max = _mm_set1_epi16(((1 << bitdepth) - 1) as i16);
let [alpha, beta, gamma, delta] = abcd.map(i32::from);
let mut mid = [0i16; 15 * 8];
horz(
&mut mid,
src,
src_stride,
src_off,
mx,
alpha,
beta,
(1 << h_shift) >> 1,
h_shift,
);
for (y, dst_row) in dst.chunks_mut(dst_stride).take(8).enumerate() {
let v = round(vert(&mid, y, my, gamma), (1 << v_shift) >> 1, v_shift);
let p = _mm_min_epu16(_mm_packus_epi32(v.0, v.1), max);
unsafe { _mm_storeu_si128(dst_row[..8].as_mut_ptr().cast(), p) };
my += delta;
}
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "sse4.1")]
pub(crate) fn warp_affine_8x8t_hbd_sse41(
tmp: &mut [i16],
tmp_stride: usize,
src: &[u16],
src_stride: usize,
src_off: usize,
abcd: &[i16; 4],
mx: i32,
mut my: i32,
bitdepth: u8,
) {
let h_shift = 7 - (14 - bitdepth as i32);
let [alpha, beta, gamma, delta] = abcd.map(i32::from);
let mut mid = [0i16; 15 * 8];
horz(
&mut mid,
src,
src_stride,
src_off,
mx,
alpha,
beta,
(1 << h_shift) >> 1,
h_shift,
);
let bias = _mm_set1_epi32(8192);
for (y, tmp_row) in tmp.chunks_mut(tmp_stride).take(8).enumerate() {
let v = round(vert(&mid, y, my, gamma), 64, 7);
let p = _mm_packs_epi32(_mm_sub_epi32(v.0, bias), _mm_sub_epi32(v.1, bias));
unsafe { _mm_storeu_si128(tmp_row[..8].as_mut_ptr().cast(), p) };
my += delta;
}
}