#[cfg(target_arch = "x86")]
use std::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use std::arch::x86_64::*;
#[inline]
#[target_feature(enable = "sse4.1")]
fn load_u16x4_i32(p: &[u16]) -> __m128i {
_mm_cvtepu16_epi32(unsafe { _mm_loadl_epi64(p.as_ptr().cast()) })
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn sll_s32(v: __m128i, shift: i32) -> __m128i {
_mm_sll_epi32(v, _mm_cvtsi32_si128(shift))
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn round_s32(v: __m128i, rnd: i32, shift: i32) -> __m128i {
_mm_sra_epi32(
_mm_add_epi32(v, _mm_set1_epi32(rnd)),
_mm_cvtsi32_si128(shift),
)
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn store_clip_u16x4(dst: &mut [u16], v: __m128i, rnd: i32, shift: i32, max: __m128i) {
let v = round_s32(v, rnd, shift);
let p = _mm_packus_epi32(v, _mm_setzero_si128());
let p = _mm_min_epu16(p, max);
unsafe {
_mm_storel_epi64(dst.as_mut_ptr().cast(), p);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn store_i16x4(dst: &mut [i16], v: __m128i, rnd: i32, shift: i32, bias: i32) {
let v = _mm_sub_epi32(round_s32(v, rnd, shift), _mm_set1_epi32(bias));
let p = _mm_packs_epi32(v, _mm_setzero_si128());
unsafe {
_mm_storel_epi64(dst.as_mut_ptr().cast(), p);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn load_u16x4(p: &[u16]) -> __m128i {
unsafe { _mm_loadl_epi64(p.as_ptr().cast()) }
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn load_i16x4(p: &[i16]) -> __m128i {
unsafe { _mm_loadl_epi64(p.as_ptr().cast()) }
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn mul_i16x4_n_s32(x: __m128i, k: i32) -> __m128i {
let xz = _mm_unpacklo_epi16(x, _mm_setzero_si128());
let kz = _mm_set1_epi32((k as i16 as u16) as i32);
_mm_madd_epi16(xz, kz)
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn filter_u16x4(src: &[u16], base: usize, stride: isize, f: &[i8; 8]) -> __m128i {
let mut sum = _mm_setzero_si128();
static OFFSETS: [isize; 8] = [-3isize, -2, -1, 0, 1, 2, 3, 4];
for k in 0..8 {
let idx = (base as isize + OFFSETS[k] * stride) as usize;
let s = load_u16x4(unsafe { src.get_unchecked(idx..) });
sum = _mm_add_epi32(sum, mul_i16x4_n_s32(s, f[k] as i32));
}
sum
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn filter_i16x4(src: &[i16], base: usize, stride: isize, f: &[i8; 8]) -> __m128i {
let mut sum = _mm_setzero_si128();
static OFFSETS: [isize; 8] = [-3isize, -2, -1, 0, 1, 2, 3, 4];
for k in 0..8 {
let idx = (base as isize + OFFSETS[k] * stride) as usize;
let s = load_i16x4(unsafe { src.get_unchecked(idx..) });
sum = _mm_add_epi32(sum, mul_i16x4_n_s32(s, f[k] as i32));
}
sum
}
#[inline(always)]
fn filter_u16_scalar(src: &[u16], base: usize, stride: isize, f: &[i8; 8]) -> i32 {
let c = base as isize;
f[0] as i32 * src[(c - 3 * stride) as usize] as i32
+ f[1] as i32 * src[(c - 2 * stride) as usize] as i32
+ f[2] as i32 * src[(c - stride) as usize] as i32
+ f[3] as i32 * src[base] as i32
+ f[4] as i32 * src[(c + stride) as usize] as i32
+ f[5] as i32 * src[(c + 2 * stride) as usize] as i32
+ f[6] as i32 * src[(c + 3 * stride) as usize] as i32
+ f[7] as i32 * src[(c + 4 * stride) as usize] as i32
}
#[inline(always)]
fn filter_i16_scalar(src: &[i16], base: usize, stride: isize, f: &[i8; 8]) -> i32 {
let c = base as isize;
f[0] as i32 * src[(c - 3 * stride) as usize] as i32
+ f[1] as i32 * src[(c - 2 * stride) as usize] as i32
+ f[2] as i32 * src[(c - stride) as usize] as i32
+ f[3] as i32 * src[base] as i32
+ f[4] as i32 * src[(c + stride) as usize] as i32
+ f[5] as i32 * src[(c + 2 * stride) as usize] as i32
+ f[6] as i32 * src[(c + 3 * stride) as usize] as i32
+ f[7] as i32 * src[(c + 4 * stride) as usize] as i32
}
#[inline(always)]
fn clip(v: i32, bitdepth: u8) -> u16 {
v.clamp(0, (1 << bitdepth) - 1) as u16
}
#[inline(always)]
fn round_scalar(v: i32, rnd: i32, shift: i32) -> i32 {
if shift == 0 {
v + rnd
} else {
(v + rnd) >> shift
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn bilin_u16x4(src: &[u16], base: usize, stride: usize, mxy: i32) -> __m128i {
let a16 = load_u16x4(unsafe { src.get_unchecked(base..) });
let b16 = load_u16x4(unsafe { src.get_unchecked(base + stride..) });
let a = _mm_cvtepu16_epi32(a16);
let diff = _mm_sub_epi16(b16, a16);
_mm_add_epi32(_mm_slli_epi32::<4>(a), mul_i16x4_n_s32(diff, mxy))
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn bilin_i16x4(a16: __m128i, b16: __m128i, mxy: i32) -> __m128i {
let a = _mm_cvtepi16_epi32(a16);
let diff = _mm_sub_epi16(b16, a16);
_mm_add_epi32(_mm_slli_epi32::<4>(a), mul_i16x4_n_s32(diff, mxy))
}
#[target_feature(enable = "sse4.1")]
#[allow(clippy::too_many_arguments)]
pub(crate) fn prep_hbd_sse41(
tmp: &mut [i16],
tmp_stride: usize,
src: &[u16],
src_stride: usize,
w: usize,
h: usize,
bitdepth: u8,
) {
let ib = 14 - bitdepth as i32;
let bias = 8192i32;
for (y, tmp_row) in tmp.chunks_mut(tmp_stride).take(h).enumerate() {
let src_row = unsafe { src.get_unchecked(y * src_stride..) };
let (tmp_chunks, tmp_rem) = tmp_row[..w].as_chunks_mut::<4>();
for (chunk_idx, tmp_chunk) in tmp_chunks.iter_mut().enumerate() {
let x = chunk_idx * 4;
let s = load_u16x4_i32(unsafe { src_row.get_unchecked(x..) });
let v = _mm_sub_epi32(sll_s32(s, ib), _mm_set1_epi32(bias));
let p = _mm_packs_epi32(v, _mm_setzero_si128());
unsafe { _mm_storel_epi64(tmp_chunk.as_mut_ptr().cast(), p) };
}
let processed = tmp_chunks.len() * 4;
for (x, tmp_px) in (processed..w).zip(tmp_rem.iter_mut()) {
*tmp_px = (((src_row[x] as i32) << ib) - bias) as i16;
}
}
}
#[target_feature(enable = "sse4.1")]
#[allow(clippy::too_many_arguments)]
pub(crate) fn put_bilin_hbd_sse41(
dst: &mut [u16],
dst_stride: usize,
src: &[u16],
src_stride: usize,
w: usize,
h: usize,
mx: i32,
my: i32,
bitdepth: u8,
mid_scratch: &mut [i16],
) {
let ib = 14 - bitdepth as i32;
let maxv = _mm_set1_epi16(((1 << bitdepth) - 1) as i16);
let intermediate_rnd = (1 << ib) >> 1;
if mx != 0 && my != 0 {
let ms = w.next_multiple_of(16).max(64);
let mid = &mut mid_scratch[..ms * (h + 1)];
let sh0 = 4 - ib;
let rnd0 = if sh0 == 0 { 0 } else { 1 << (sh0 - 1) };
for (y, mid_row) in mid.chunks_mut(ms).take(h + 1).enumerate() {
let src_row = unsafe { src.get_unchecked(y * src_stride..) };
let (mid_chunks, mid_rem) = mid_row[..w].as_chunks_mut::<4>();
for (chunk_idx, mid_chunk) in mid_chunks.iter_mut().enumerate() {
let x = chunk_idx * 4;
let v = bilin_u16x4(src_row, x, 1, mx);
store_i16x4(mid_chunk, v, rnd0, sh0, 0);
}
let processed = mid_chunks.len() * 4;
for (x, mid_px) in (processed..w).zip(mid_rem.iter_mut()) {
let a = src_row[x] as i32;
let b = src_row[x + 1] as i32;
*mid_px = round_scalar(16 * a + mx * (b - a), rnd0, sh0) as i16;
}
}
for (y, dst_row) in dst.chunks_mut(dst_stride).take(h).enumerate() {
let mid_row = unsafe { mid.get_unchecked(y * ms..) };
let mid_next_row = unsafe { mid.get_unchecked((y + 1) * ms..) };
let (dst_chunks, dst_rem) = dst_row[..w].as_chunks_mut::<4>();
for (chunk_idx, dst_chunk) in dst_chunks.iter_mut().enumerate() {
let x = chunk_idx * 4;
let a = load_i16x4(unsafe { mid.get_unchecked(y * ms + x..) });
let b = load_i16x4(unsafe { mid.get_unchecked((y + 1) * ms + x..) });
let v = bilin_i16x4(a, b, my);
store_clip_u16x4(dst_chunk, v, 1 << (3 + ib), 4 + ib, maxv);
}
let processed = dst_chunks.len() * 4;
for (x, dst_px) in (processed..w).zip(dst_rem.iter_mut()) {
let a = mid_row[x] as i32;
let b = mid_next_row[x] as i32;
*dst_px = clip(
round_scalar(16 * a + my * (b - a), 1 << (3 + ib), 4 + ib),
bitdepth,
);
}
}
} else if mx != 0 {
let sh0 = 4 - ib;
let rnd0 = if sh0 == 0 { 0 } else { 1 << (sh0 - 1) };
for (y, dst_row) in dst.chunks_mut(dst_stride).take(h).enumerate() {
let src_row = unsafe { src.get_unchecked(y * src_stride..) };
let (dst_chunks, dst_rem) = dst_row[..w].as_chunks_mut::<4>();
for (chunk_idx, dst_chunk) in dst_chunks.iter_mut().enumerate() {
let x = chunk_idx * 4;
let px = round_s32(bilin_u16x4(src_row, x, 1, mx), rnd0, sh0);
store_clip_u16x4(dst_chunk, px, intermediate_rnd, ib, maxv);
}
let processed = dst_chunks.len() * 4;
for (x, dst_px) in (processed..w).zip(dst_rem.iter_mut()) {
let a = src_row[x] as i32;
let b = src_row[x + 1] as i32;
let px = round_scalar(16 * a + mx * (b - a), rnd0, sh0);
*dst_px = clip(round_scalar(px, intermediate_rnd, ib), bitdepth);
}
}
} else if my != 0 {
for (y, dst_row) in dst.chunks_mut(dst_stride).take(h).enumerate() {
let src_row = unsafe { src.get_unchecked(y * src_stride..) };
let src_next_row = unsafe { src.get_unchecked((y + 1) * src_stride..) };
let (dst_chunks, dst_rem) = dst_row[..w].as_chunks_mut::<4>();
for (chunk_idx, dst_chunk) in dst_chunks.iter_mut().enumerate() {
let x = chunk_idx * 4;
let v = bilin_u16x4(src_row, x, src_stride, my);
store_clip_u16x4(dst_chunk, v, 8, 4, maxv);
}
let processed = dst_chunks.len() * 4;
for (x, dst_px) in (processed..w).zip(dst_rem.iter_mut()) {
let a = src_row[x] as i32;
let b = src_next_row[x] as i32;
*dst_px = clip(round_scalar(16 * a + my * (b - a), 8, 4), bitdepth);
}
}
} else {
for (src_row, dst_row) in src
.chunks(src_stride)
.zip(dst.chunks_mut(dst_stride))
.take(h)
{
dst_row[..w].copy_from_slice(&src_row[..w]);
}
}
}
#[target_feature(enable = "sse4.1")]
#[allow(clippy::too_many_arguments)]
pub(crate) fn prep_bilin_hbd_sse41(
tmp: &mut [i16],
tmp_stride: usize,
src: &[u16],
src_stride: usize,
w: usize,
h: usize,
mx: i32,
my: i32,
bitdepth: u8,
mid_scratch: &mut [i16],
) {
let ib = 14 - bitdepth as i32;
let bias = 8192i32;
if mx != 0 && my != 0 {
let ms = w.next_multiple_of(16).max(64);
let mid = &mut mid_scratch[..ms * (h + 1)];
let sh0 = 4 - ib;
let rnd0 = if sh0 == 0 { 0 } else { 1 << (sh0 - 1) };
for (y, mid_row) in mid.chunks_mut(ms).take(h + 1).enumerate() {
let src_row = unsafe { src.get_unchecked(y * src_stride..) };
let (mid_chunks, mid_rem) = mid_row[..w].as_chunks_mut::<4>();
for (chunk_idx, mid_chunk) in mid_chunks.iter_mut().enumerate() {
let x = chunk_idx * 4;
store_i16x4(mid_chunk, bilin_u16x4(src_row, x, 1, mx), rnd0, sh0, 0);
}
let processed = mid_chunks.len() * 4;
for (x, mid_px) in (processed..w).zip(mid_rem.iter_mut()) {
let a = src_row[x] as i32;
let b = src_row[x + 1] as i32;
*mid_px = round_scalar(16 * a + mx * (b - a), rnd0, sh0) as i16;
}
}
for (y, tmp_row) in tmp.chunks_mut(tmp_stride).take(h).enumerate() {
let mid_row = unsafe { mid.get_unchecked(y * ms..) };
let mid_next_row = unsafe { mid.get_unchecked((y + 1) * ms..) };
let (tmp_chunks, tmp_rem) = tmp_row[..w].as_chunks_mut::<4>();
for (chunk_idx, tmp_chunk) in tmp_chunks.iter_mut().enumerate() {
let x = chunk_idx * 4;
let a = load_i16x4(unsafe { mid.get_unchecked(y * ms + x..) });
let b = load_i16x4(unsafe { mid.get_unchecked((y + 1) * ms + x..) });
let v = bilin_i16x4(a, b, my);
store_i16x4(tmp_chunk, v, 8, 4, bias);
}
let processed = tmp_chunks.len() * 4;
for (x, tmp_px) in (processed..w).zip(tmp_rem.iter_mut()) {
let a = mid_row[x] as i32;
let b = mid_next_row[x] as i32;
*tmp_px = (round_scalar(16 * a + my * (b - a), 8, 4) - bias) as i16;
}
}
} else if mx != 0 {
let sh0 = 4 - ib;
let rnd0 = if sh0 == 0 { 0 } else { 1 << (sh0 - 1) };
for (y, tmp_row) in tmp.chunks_mut(tmp_stride).take(h).enumerate() {
let src_row = unsafe { src.get_unchecked(y * src_stride..) };
let (tmp_chunks, tmp_rem) = tmp_row[..w].as_chunks_mut::<4>();
for (chunk_idx, tmp_chunk) in tmp_chunks.iter_mut().enumerate() {
let x = chunk_idx * 4;
store_i16x4(tmp_chunk, bilin_u16x4(src_row, x, 1, mx), rnd0, sh0, bias);
}
let processed = tmp_chunks.len() * 4;
for (x, tmp_px) in (processed..w).zip(tmp_rem.iter_mut()) {
let a = src_row[x] as i32;
let b = src_row[x + 1] as i32;
*tmp_px = (round_scalar(16 * a + mx * (b - a), rnd0, sh0) - bias) as i16;
}
}
} else if my != 0 {
let sh0 = 4 - ib;
let rnd0 = if sh0 == 0 { 0 } else { 1 << (sh0 - 1) };
for (y, tmp_row) in tmp.chunks_mut(tmp_stride).take(h).enumerate() {
let src_row = unsafe { src.get_unchecked(y * src_stride..) };
let src_next_row = unsafe { src.get_unchecked((y + 1) * src_stride..) };
let (tmp_chunks, tmp_rem) = tmp_row[..w].as_chunks_mut::<4>();
for (chunk_idx, tmp_chunk) in tmp_chunks.iter_mut().enumerate() {
let x = chunk_idx * 4;
store_i16x4(
tmp_chunk,
bilin_u16x4(src_row, x, src_stride, my),
rnd0,
sh0,
bias,
);
}
let processed = tmp_chunks.len() * 4;
for (x, tmp_px) in (processed..w).zip(tmp_rem.iter_mut()) {
let a = src_row[x] as i32;
let b = src_next_row[x] as i32;
*tmp_px = (round_scalar(16 * a + my * (b - a), rnd0, sh0) - bias) as i16;
}
}
} else {
prep_hbd_sse41(tmp, tmp_stride, src, src_stride, w, h, bitdepth);
}
}
#[target_feature(enable = "sse4.1")]
#[allow(clippy::too_many_arguments)]
pub(crate) fn put_8tap_hbd_sse41(
dst: &mut [u16],
dst_stride: usize,
src: &[u16],
src_off: usize,
src_stride: usize,
w: usize,
h: usize,
mx: i32,
my: i32,
filter_type: i32,
bitdepth: u8,
mid_scratch: &mut [i16],
) {
let bits = 6 + (filter_type < 0) as i32;
let ib = 14 - bitdepth as i32;
let intermediate_rnd = ((1 << bits) >> 1) + ((1 << (bits - ib)) >> 1);
let fh = crate::mc::get_h_filter(mx, filter_type, w);
let fv = crate::mc::get_v_filter(my, filter_type, h);
let maxv = _mm_set1_epi16(((1 << bitdepth) - 1) as i16);
match (fh, fv) {
(Some(fh), Some(fv)) => {
let tmp_h = h + 7;
let ms = w.next_multiple_of(16).max(64);
let mid = &mut mid_scratch[..ms * tmp_h];
let sh0 = bits - ib;
let rnd0 = (1 << sh0) >> 1;
for (y, mid_row) in mid.chunks_mut(ms).take(tmp_h).enumerate() {
let base = (src_off as isize + (y as isize - 3) * src_stride as isize) as usize;
let (mid_chunks, mid_rem) = mid_row[..w].as_chunks_mut::<4>();
for (chunk_idx, mid_chunk) in mid_chunks.iter_mut().enumerate() {
let x = chunk_idx * 4;
store_i16x4(mid_chunk, filter_u16x4(src, base + x, 1, &fh), rnd0, sh0, 0);
}
let processed = mid_chunks.len() * 4;
for (x, mid_px) in (processed..w).zip(mid_rem.iter_mut()) {
*mid_px =
round_scalar(filter_u16_scalar(src, base + x, 1, &fh), rnd0, sh0) as i16;
}
}
let sh1 = bits + ib;
let rnd1 = (1 << sh1) >> 1;
for (y, dst_row) in dst.chunks_mut(dst_stride).take(h).enumerate() {
let (dst_chunks, dst_rem) = dst_row[..w].as_chunks_mut::<4>();
for (chunk_idx, dst_chunk) in dst_chunks.iter_mut().enumerate() {
let x = chunk_idx * 4;
store_clip_u16x4(
dst_chunk,
filter_i16x4(&mid, (y + 3) * ms + x, ms as isize, &fv),
rnd1,
sh1,
maxv,
);
}
let processed = dst_chunks.len() * 4;
for (x, dst_px) in (processed..w).zip(dst_rem.iter_mut()) {
*dst_px = clip(
round_scalar(
filter_i16_scalar(&mid, (y + 3) * ms + x, ms as isize, &fv),
rnd1,
sh1,
),
bitdepth,
);
}
}
}
(Some(fh), None) => {
for (y, dst_row) in dst.chunks_mut(dst_stride).take(h).enumerate() {
let base = src_off + y * src_stride;
let (dst_chunks, dst_rem) = dst_row[..w].as_chunks_mut::<4>();
for (chunk_idx, dst_chunk) in dst_chunks.iter_mut().enumerate() {
let x = chunk_idx * 4;
store_clip_u16x4(
dst_chunk,
filter_u16x4(src, base + x, 1, &fh),
intermediate_rnd,
bits,
maxv,
);
}
let processed = dst_chunks.len() * 4;
for (x, dst_px) in (processed..w).zip(dst_rem.iter_mut()) {
*dst_px = clip(
round_scalar(
filter_u16_scalar(src, base + x, 1, &fh),
intermediate_rnd,
bits,
),
bitdepth,
);
}
}
}
(None, Some(fv)) => {
let ss = src_stride as isize;
for (y, dst_row) in dst.chunks_mut(dst_stride).take(h).enumerate() {
let base = src_off + y * src_stride;
let (dst_chunks, dst_rem) = dst_row[..w].as_chunks_mut::<4>();
for (chunk_idx, dst_chunk) in dst_chunks.iter_mut().enumerate() {
let x = chunk_idx * 4;
store_clip_u16x4(
dst_chunk,
filter_u16x4(src, base + x, ss, &fv),
(1 << bits) >> 1,
bits,
maxv,
);
}
let processed = dst_chunks.len() * 4;
for (x, dst_px) in (processed..w).zip(dst_rem.iter_mut()) {
*dst_px = clip(
round_scalar(
filter_u16_scalar(src, base + x, ss, &fv),
(1 << bits) >> 1,
bits,
),
bitdepth,
);
}
}
}
(None, None) => {
for (src_row, dst_row) in src[src_off..]
.chunks(src_stride)
.zip(dst.chunks_mut(dst_stride))
.take(h)
{
dst_row[..w].copy_from_slice(&src_row[..w]);
}
}
}
}
#[target_feature(enable = "sse4.1")]
#[allow(clippy::too_many_arguments)]
pub(crate) fn prep_8tap_hbd_sse41(
tmp: &mut [i16],
tmp_stride: usize,
src: &[u16],
src_off: usize,
src_stride: usize,
w: usize,
h: usize,
mx: i32,
my: i32,
filter_type: i32,
bitdepth: u8,
mid_scratch: &mut [i16],
) {
let bits = 6 + (filter_type < 0) as i32;
let ib = 14 - bitdepth as i32;
let bias = 8192i32;
let fh = crate::mc::get_h_filter(mx, filter_type, w);
let fv = crate::mc::get_v_filter(my, filter_type, h);
match (fh, fv) {
(Some(fh), Some(fv)) => {
let tmp_h = h + 7;
let ms = w.next_multiple_of(16).max(64);
let mid = &mut mid_scratch[..ms * tmp_h];
let sh0 = bits - ib;
let rnd0 = (1 << sh0) >> 1;
for (y, mid_row) in mid.chunks_mut(ms).take(tmp_h).enumerate() {
let base = (src_off as isize + (y as isize - 3) * src_stride as isize) as usize;
let (mid_chunks, mid_rem) = mid_row[..w].as_chunks_mut::<4>();
for (chunk_idx, mid_chunk) in mid_chunks.iter_mut().enumerate() {
let x = chunk_idx * 4;
store_i16x4(mid_chunk, filter_u16x4(src, base + x, 1, &fh), rnd0, sh0, 0);
}
let processed = mid_chunks.len() * 4;
for (x, mid_px) in (processed..w).zip(mid_rem.iter_mut()) {
*mid_px =
round_scalar(filter_u16_scalar(src, base + x, 1, &fh), rnd0, sh0) as i16;
}
}
let rnd1 = (1 << bits) >> 1;
for (y, tmp_row) in tmp.chunks_mut(tmp_stride).take(h).enumerate() {
let (tmp_chunks, tmp_rem) = tmp_row[..w].as_chunks_mut::<4>();
for (chunk_idx, tmp_chunk) in tmp_chunks.iter_mut().enumerate() {
let x = chunk_idx * 4;
store_i16x4(
tmp_chunk,
filter_i16x4(&mid, (y + 3) * ms + x, ms as isize, &fv),
rnd1,
bits,
bias,
);
}
let processed = tmp_chunks.len() * 4;
for (x, tmp_px) in (processed..w).zip(tmp_rem.iter_mut()) {
*tmp_px = (round_scalar(
filter_i16_scalar(&mid, (y + 3) * ms + x, ms as isize, &fv),
rnd1,
bits,
) - bias) as i16;
}
}
}
(Some(fh), None) => {
let sh0 = bits - ib;
let rnd0 = (1 << sh0) >> 1;
for (y, tmp_row) in tmp.chunks_mut(tmp_stride).take(h).enumerate() {
let base = src_off + y * src_stride;
let (tmp_chunks, tmp_rem) = tmp_row[..w].as_chunks_mut::<4>();
for (chunk_idx, tmp_chunk) in tmp_chunks.iter_mut().enumerate() {
let x = chunk_idx * 4;
store_i16x4(
tmp_chunk,
filter_u16x4(src, base + x, 1, &fh),
rnd0,
sh0,
bias,
);
}
let processed = tmp_chunks.len() * 4;
for (x, tmp_px) in (processed..w).zip(tmp_rem.iter_mut()) {
*tmp_px = (round_scalar(filter_u16_scalar(src, base + x, 1, &fh), rnd0, sh0)
- bias) as i16;
}
}
}
(None, Some(fv)) => {
let ss = src_stride as isize;
let sh0 = bits - ib;
let rnd0 = (1 << sh0) >> 1;
for (y, tmp_row) in tmp.chunks_mut(tmp_stride).take(h).enumerate() {
let base = src_off + y * src_stride;
let (tmp_chunks, tmp_rem) = tmp_row[..w].as_chunks_mut::<4>();
for (chunk_idx, tmp_chunk) in tmp_chunks.iter_mut().enumerate() {
let x = chunk_idx * 4;
store_i16x4(
tmp_chunk,
filter_u16x4(src, base + x, ss, &fv),
rnd0,
sh0,
bias,
);
}
let processed = tmp_chunks.len() * 4;
for (x, tmp_px) in (processed..w).zip(tmp_rem.iter_mut()) {
*tmp_px = (round_scalar(filter_u16_scalar(src, base + x, ss, &fv), rnd0, sh0)
- bias) as i16;
}
}
}
(None, None) => {
prep_hbd_sse41(tmp, tmp_stride, &src[src_off..], src_stride, w, h, bitdepth)
}
}
}