use super::deblocking::deblock_apply_8bpc_sse41;
#[cfg(target_arch = "x86")]
use std::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use std::arch::x86_64::*;
#[inline(always)]
unsafe fn load_deblock_unchecked<T: Copy>(buf: &[T], idx: isize) -> T {
unsafe { *buf.get_unchecked(idx as usize) }
}
#[inline(always)]
unsafe fn store_deblock_unchecked<T>(buf: &mut [T], idx: isize, value: T) {
unsafe { *buf.get_unchecked_mut(idx as usize) = value };
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn load4_u8_i16_oriented<const CONTIG: bool>(
dst: &[u8],
base: isize,
stride_line: isize,
) -> __m128i {
unsafe {
let p = dst.as_ptr();
if CONTIG {
_mm_cvtepu8_epi16(_mm_castps_si128(_mm_load_ss(p.add(base as usize).cast())))
} else {
_mm_setr_epi16(
load_deblock_unchecked(dst, base) as i16,
load_deblock_unchecked(dst, base + stride_line) as i16,
load_deblock_unchecked(dst, base + 2 * stride_line) as i16,
load_deblock_unchecked(dst, base + 3 * stride_line) as i16,
0,
0,
0,
0,
)
}
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn store4_clip_u8_i16_oriented<const CONTIG: bool>(
dst: &mut [u8],
base: isize,
stride_line: isize,
v: __m128i,
) {
unsafe {
let p = dst.as_mut_ptr();
let p8 = _mm_packus_epi16(v, v);
if CONTIG {
_mm_store_ss(p.add(base as usize).cast(), _mm_castsi128_ps(p8));
} else {
let packed = _mm_cvtsi128_si32(p8) as u32;
store_deblock_unchecked(dst, base, (packed & 0xff) as u8);
store_deblock_unchecked(dst, base + stride_line, ((packed >> 8) & 0xff) as u8);
store_deblock_unchecked(dst, base + 2 * stride_line, ((packed >> 16) & 0xff) as u8);
store_deblock_unchecked(dst, base + 3 * stride_line, (packed >> 24) as u8);
}
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn deblock_delta_i16(
d0: __m128i,
dm1: __m128i,
dp1: __m128i,
dm2: __m128i,
nqc: __m128i,
qc: __m128i,
) -> __m128i {
let d0_m1 = _mm_sub_epi16(d0, dm1);
let dp1_m2 = _mm_sub_epi16(dp1, dm2);
let inner = _mm_sub_epi16(_mm_add_epi16(d0_m1, _mm_slli_epi16::<1>(d0_m1)), dp1_m2);
let delta = _mm_slli_epi16::<2>(inner);
_mm_min_epi16(_mm_max_epi16(delta, nqc), qc)
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn deblock_diff_i16(delta: __m128i, width: i32, tap: i32) -> __m128i {
let coeff = (crate::deblock::W_MULT[(width - 1) as usize] as i32 * tap * 16) as i16;
_mm_mulhrs_epi16(delta, _mm_set1_epi16(coeff))
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn deblock_extract_i16<const LANE: i32>(v: __m128i) -> i16 {
_mm_extract_epi16::<LANE>(v) as u16 as i16
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn deblock_apply_8bpc_sse41_h_sym4_rows(
dst: &mut [u8],
off: isize,
stride_line: isize,
delta: __m128i,
apply_neg: bool,
apply_pos: bool,
) {
let wm = (crate::deblock::W_MULT[3] as i16) * 16;
let neg = if apply_neg { 1 } else { 0 };
let pos = if apply_pos { -1 } else { 0 };
let coeff = _mm_setr_epi16(wm, wm * 2, wm * 3, wm * 4, wm * 4, wm * 3, wm * 2, wm);
let sign = _mm_setr_epi16(neg, neg, neg, neg, pos, pos, pos, pos);
let p = dst.as_mut_ptr();
let mut r = 0;
while r < 4 {
let row = off + r * stride_line - 4;
unsafe {
let bytes = _mm_loadl_epi64(p.add(row as usize).cast());
let pix = _mm_cvtepu8_epi16(bytes);
let d = match r {
0 => deblock_extract_i16::<0>(delta),
1 => deblock_extract_i16::<1>(delta),
2 => deblock_extract_i16::<2>(delta),
_ => deblock_extract_i16::<3>(delta),
};
let diff = _mm_mullo_epi16(_mm_mulhrs_epi16(_mm_set1_epi16(d), coeff), sign);
let res = _mm_add_epi16(pix, diff);
let packed = _mm_packus_epi16(res, res);
_mm_storel_epi64(p.add(row as usize).cast(), packed);
}
r += 1;
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn deblock_apply_8bpc_sse41_h_sym8_rows(
dst: &mut [u8],
off: isize,
stride_line: isize,
delta: __m128i,
apply_neg: bool,
apply_pos: bool,
) {
let wm = (crate::deblock::W_MULT[7] as i16) * 16;
let neg = if apply_neg { 1 } else { 0 };
let pos = if apply_pos { -1 } else { 0 };
let coeff_lo = _mm_setr_epi16(wm, wm * 2, wm * 3, wm * 4, wm * 5, wm * 6, wm * 7, wm * 8);
let coeff_hi = _mm_setr_epi16(wm * 8, wm * 7, wm * 6, wm * 5, wm * 4, wm * 3, wm * 2, wm);
let (sign_lo, sign_hi) = (_mm_set1_epi16(neg), _mm_set1_epi16(pos));
let p = dst.as_mut_ptr();
let mut r = 0;
while r < 4 {
let row = off + r * stride_line - 8;
unsafe {
let bytes = _mm_loadu_si128(p.add(row as usize).cast());
let d = _mm_set1_epi16(match r {
0 => deblock_extract_i16::<0>(delta),
1 => deblock_extract_i16::<1>(delta),
2 => deblock_extract_i16::<2>(delta),
_ => deblock_extract_i16::<3>(delta),
});
let lo = _mm_add_epi16(
_mm_cvtepu8_epi16(bytes),
_mm_mullo_epi16(_mm_mulhrs_epi16(d, coeff_lo), sign_lo),
);
let hi = _mm_add_epi16(
_mm_cvtepu8_epi16(_mm_srli_si128::<8>(bytes)),
_mm_mullo_epi16(_mm_mulhrs_epi16(d, coeff_hi), sign_hi),
);
_mm_storeu_si128(p.add(row as usize).cast(), _mm_packus_epi16(lo, hi));
}
r += 1;
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "sse4.1")]
fn deblock_apply_8bpc_sse41_const_oriented<const WN: i32, const WP: i32, const CONTIG: bool>(
dst: &mut [u8],
off: isize,
stride_line: isize,
stride_tap: isize,
q_thr_clamp: i32,
neg_lossless: bool,
pos_lossless: bool,
) {
debug_assert!((1..=8).contains(&WN));
debug_assert!((1..=8).contains(&WP));
let apply_neg = !neg_lossless;
let apply_pos = !pos_lossless;
debug_assert!(apply_neg || apply_pos);
debug_assert!(q_thr_clamp <= i16::MAX as i32);
let qc = _mm_set1_epi16(q_thr_clamp as i16);
let nqc = _mm_set1_epi16(-(q_thr_clamp as i16));
let d0 = load4_u8_i16_oriented::<CONTIG>(dst, off, stride_line);
let dm1 = load4_u8_i16_oriented::<CONTIG>(dst, off - stride_tap, stride_line);
let dp1 = load4_u8_i16_oriented::<CONTIG>(dst, off + stride_tap, stride_line);
let dm2 = load4_u8_i16_oriented::<CONTIG>(dst, off - 2 * stride_tap, stride_line);
let delta = deblock_delta_i16(d0, dm1, dp1, dm2, nqc, qc);
if !CONTIG && stride_tap == 1 && WN == WP {
if WN == 8 {
deblock_apply_8bpc_sse41_h_sym8_rows(
dst,
off,
stride_line,
delta,
apply_neg,
apply_pos,
);
return;
}
if WN == 4 {
deblock_apply_8bpc_sse41_h_sym4_rows(
dst,
off,
stride_line,
delta,
apply_neg,
apply_pos,
);
return;
}
}
if apply_neg {
let mut j = 0;
while j < WN {
let base = off + (-(j as isize) - 1) * stride_tap;
let cur = load4_u8_i16_oriented::<CONTIG>(dst, base, stride_line);
let diff = deblock_diff_i16(delta, WN, WN - j);
store4_clip_u8_i16_oriented::<CONTIG>(dst, base, stride_line, _mm_add_epi16(cur, diff));
j += 1;
}
}
if apply_pos {
let mut j = 0;
while j < WP {
let base = off + (j as isize) * stride_tap;
let cur = load4_u8_i16_oriented::<CONTIG>(dst, base, stride_line);
let diff = deblock_diff_i16(delta, WP, WP - j);
store4_clip_u8_i16_oriented::<CONTIG>(dst, base, stride_line, _mm_sub_epi16(cur, diff));
j += 1;
}
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn transpose16x16_u8_sse41(r: &mut [__m128i; 16]) {
let z = _mm_setzero_si128();
let mut t = [z; 16];
let mut u = [z; 16];
let mut v = [z; 16];
let mut i = 0;
while i < 8 {
let a = r[i * 2];
let b = r[i * 2 + 1];
t[i * 2] = _mm_unpacklo_epi8(a, b);
t[i * 2 + 1] = _mm_unpackhi_epi8(a, b);
i += 1;
}
i = 0;
while i < 4 {
let b = i * 4;
u[b] = _mm_unpacklo_epi16(t[b], t[b + 2]);
u[b + 1] = _mm_unpackhi_epi16(t[b], t[b + 2]);
u[b + 2] = _mm_unpacklo_epi16(t[b + 1], t[b + 3]);
u[b + 3] = _mm_unpackhi_epi16(t[b + 1], t[b + 3]);
i += 1;
}
i = 0;
while i < 2 {
let b = i * 8;
v[b] = _mm_unpacklo_epi32(u[b], u[b + 4]);
v[b + 1] = _mm_unpackhi_epi32(u[b], u[b + 4]);
v[b + 2] = _mm_unpacklo_epi32(u[b + 1], u[b + 5]);
v[b + 3] = _mm_unpackhi_epi32(u[b + 1], u[b + 5]);
v[b + 4] = _mm_unpacklo_epi32(u[b + 2], u[b + 6]);
v[b + 5] = _mm_unpackhi_epi32(u[b + 2], u[b + 6]);
v[b + 6] = _mm_unpacklo_epi32(u[b + 3], u[b + 7]);
v[b + 7] = _mm_unpackhi_epi32(u[b + 3], u[b + 7]);
i += 1;
}
r[0] = _mm_unpacklo_epi64(v[0], v[8]);
r[1] = _mm_unpackhi_epi64(v[0], v[8]);
r[2] = _mm_unpacklo_epi64(v[1], v[9]);
r[3] = _mm_unpackhi_epi64(v[1], v[9]);
r[4] = _mm_unpacklo_epi64(v[2], v[10]);
r[5] = _mm_unpackhi_epi64(v[2], v[10]);
r[6] = _mm_unpacklo_epi64(v[3], v[11]);
r[7] = _mm_unpackhi_epi64(v[3], v[11]);
r[8] = _mm_unpacklo_epi64(v[4], v[12]);
r[9] = _mm_unpackhi_epi64(v[4], v[12]);
r[10] = _mm_unpacklo_epi64(v[5], v[13]);
r[11] = _mm_unpackhi_epi64(v[5], v[13]);
r[12] = _mm_unpacklo_epi64(v[6], v[14]);
r[13] = _mm_unpackhi_epi64(v[6], v[14]);
r[14] = _mm_unpacklo_epi64(v[7], v[15]);
r[15] = _mm_unpackhi_epi64(v[7], v[15]);
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn cvtepu8_hi_epi16(v: __m128i) -> __m128i {
_mm_unpackhi_epi8(v, _mm_setzero_si128())
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn pack_u8_from_i16x2(lo: __m128i, hi: __m128i) -> __m128i {
_mm_packus_epi16(lo, hi)
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn repeated_qclamp4_w8(q_thr: &[u8], qi: usize) -> (__m128i, __m128i) {
let m = crate::deblock::Q_THRESH_MULTS[7] as i16;
let q0 = (q_thr[qi] as i16) * m;
let q1 = (q_thr[qi + 1] as i16) * m;
let q2 = (q_thr[qi + 2] as i16) * m;
let q3 = (q_thr[qi + 3] as i16) * m;
(
_mm_setr_epi16(q0, q0, q0, q0, q1, q1, q1, q1),
_mm_setr_epi16(q2, q2, q2, q2, q3, q3, q3, q3),
)
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn repeated_apply_mask4(ll: u16, qi: usize) -> (__m128i, __m128i) {
let m0 = if (ll & (1u16 << qi)) == 0 { -1i16 } else { 0 };
let m1 = if (ll & (1u16 << (qi + 1))) == 0 {
-1i16
} else {
0
};
let m2 = if (ll & (1u16 << (qi + 2))) == 0 {
-1i16
} else {
0
};
let m3 = if (ll & (1u16 << (qi + 3))) == 0 {
-1i16
} else {
0
};
(
_mm_setr_epi16(m0, m0, m0, m0, m1, m1, m1, m1),
_mm_setr_epi16(m2, m2, m2, m2, m3, m3, m3, m3),
)
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn deblock_apply_8bpc_sse41_h_w8x4_transpose(
dst: &mut [u8],
off: isize,
stride: isize,
qlo: __m128i,
qhi: __m128i,
neg_mask_lo: __m128i,
neg_mask_hi: __m128i,
pos_mask_lo: __m128i,
pos_mask_hi: __m128i,
) {
let p = dst.as_mut_ptr();
let z = _mm_setzero_si128();
let mut cols = [z; 16];
let mut r = 0;
while r < 16 {
let row = off + r as isize * stride - 8;
unsafe {
cols[r] = _mm_loadu_si128(p.add(row as usize).cast());
}
r += 1;
}
transpose16x16_u8_sse41(&mut cols);
let d0_lo = _mm_cvtepu8_epi16(cols[8]);
let d0_hi = cvtepu8_hi_epi16(cols[8]);
let dm1_lo = _mm_cvtepu8_epi16(cols[7]);
let dm1_hi = cvtepu8_hi_epi16(cols[7]);
let dp1_lo = _mm_cvtepu8_epi16(cols[9]);
let dp1_hi = cvtepu8_hi_epi16(cols[9]);
let dm2_lo = _mm_cvtepu8_epi16(cols[6]);
let dm2_hi = cvtepu8_hi_epi16(cols[6]);
let delta_lo = deblock_delta_i16(d0_lo, dm1_lo, dp1_lo, dm2_lo, _mm_sub_epi16(z, qlo), qlo);
let delta_hi = deblock_delta_i16(d0_hi, dm1_hi, dp1_hi, dm2_hi, _mm_sub_epi16(z, qhi), qhi);
let wm = (crate::deblock::W_MULT[7] as i16) * 16;
let mut c = 0;
while c < 8 {
let tap = (c + 1) as i16;
let coeff = _mm_set1_epi16(wm * tap);
let diff_lo = _mm_and_si128(_mm_mulhrs_epi16(delta_lo, coeff), neg_mask_lo);
let diff_hi = _mm_and_si128(_mm_mulhrs_epi16(delta_hi, coeff), neg_mask_hi);
let pix_lo = _mm_cvtepu8_epi16(cols[c]);
let pix_hi = cvtepu8_hi_epi16(cols[c]);
cols[c] = pack_u8_from_i16x2(
_mm_add_epi16(pix_lo, diff_lo),
_mm_add_epi16(pix_hi, diff_hi),
);
c += 1;
}
c = 8;
while c < 16 {
let tap = (16 - c) as i16;
let coeff = _mm_set1_epi16(wm * tap);
let diff_lo = _mm_and_si128(_mm_mulhrs_epi16(delta_lo, coeff), pos_mask_lo);
let diff_hi = _mm_and_si128(_mm_mulhrs_epi16(delta_hi, coeff), pos_mask_hi);
let pix_lo = _mm_cvtepu8_epi16(cols[c]);
let pix_hi = cvtepu8_hi_epi16(cols[c]);
cols[c] = pack_u8_from_i16x2(
_mm_sub_epi16(pix_lo, diff_lo),
_mm_sub_epi16(pix_hi, diff_hi),
);
c += 1;
}
transpose16x16_u8_sse41(&mut cols);
r = 0;
while r < 16 {
let row = off + r as isize * stride - 8;
unsafe {
_mm_storeu_si128(p.add(row as usize).cast(), cols[r]);
}
r += 1;
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn select_i32(mask: bool, yes: i32, no: i32) -> i32 {
let m = -(mask as i32);
(yes & m) | (no & !m)
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn filter_avg_abs2_from_lanes(v: __m128i) -> u32 {
let sum = _mm_add_epi16(v, _mm_srli_si128::<2>(v));
((_mm_extract_epi16::<0>(sum) as u32) + 1) >> 1
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn filter_second_deriv_8bpc_sse41(
buf: &[u8],
s: isize,
t: isize,
stride: isize,
dist: isize,
) -> u32 {
unsafe {
let s0 = load_deblock_unchecked(buf, s + (dist - 1) * stride) as i16;
let s1 = load_deblock_unchecked(buf, s + dist * stride) as i16;
let s2 = load_deblock_unchecked(buf, s + (dist + 1) * stride) as i16;
let t0 = load_deblock_unchecked(buf, t + (dist - 1) * stride) as i16;
let t1 = load_deblock_unchecked(buf, t + dist * stride) as i16;
let t2 = load_deblock_unchecked(buf, t + (dist + 1) * stride) as i16;
let a = _mm_setr_epi16(s0, t0, 0, 0, 0, 0, 0, 0);
let b = _mm_setr_epi16(s1, t1, 0, 0, 0, 0, 0, 0);
let c = _mm_setr_epi16(s2, t2, 0, 0, 0, 0, 0, 0);
let deriv = _mm_add_epi16(_mm_sub_epi16(a, _mm_slli_epi16::<1>(b)), c);
filter_avg_abs2_from_lanes(_mm_abs_epi16(deriv))
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn filter_end_deriv_8bpc_sse41(
buf: &[u8],
s0: isize,
s1: isize,
s2: isize,
t0: isize,
t1: isize,
t2: isize,
c0: i16,
c1: i16,
c2: i16,
) -> u32 {
unsafe {
let a = _mm_setr_epi16(
load_deblock_unchecked(buf, s0) as i16,
load_deblock_unchecked(buf, t0) as i16,
0,
0,
0,
0,
0,
0,
);
let b = _mm_setr_epi16(
load_deblock_unchecked(buf, s1) as i16,
load_deblock_unchecked(buf, t1) as i16,
0,
0,
0,
0,
0,
0,
);
let c = _mm_setr_epi16(
load_deblock_unchecked(buf, s2) as i16,
load_deblock_unchecked(buf, t2) as i16,
0,
0,
0,
0,
0,
0,
);
let v = _mm_add_epi16(
_mm_add_epi16(
_mm_mullo_epi16(a, _mm_set1_epi16(c0)),
_mm_mullo_epi16(b, _mm_set1_epi16(c1)),
),
_mm_mullo_epi16(c, _mm_set1_epi16(c2)),
);
filter_avg_abs2_from_lanes(_mm_abs_epi16(v))
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "sse4.1")]
fn filter_choice_8bpc_sse41_const<const MAX_WIDTH_NEG: i32, const MAX_WIDTH_POS: i32>(
buf: &[u8],
s: isize,
t: isize,
stride: isize,
q_thr: u32,
side_thr: u32,
) -> i32 {
debug_assert!((1..=8).contains(&MAX_WIDTH_POS));
debug_assert!((1..=8).contains(&MAX_WIDTH_NEG));
debug_assert!(MAX_WIDTH_NEG <= MAX_WIDTH_POS);
let sd_m2 = filter_second_deriv_8bpc_sse41(buf, s, t, stride, -2);
let sd_m1 = filter_second_deriv_8bpc_sse41(buf, s, t, stride, -1);
let sd_0 = filter_second_deriv_8bpc_sse41(buf, s, t, stride, 0);
let sd_1 = filter_second_deriv_8bpc_sse41(buf, s, t, stride, 1);
let high_deriv = sd_m2.max(sd_1);
let transition = sd_m1 + sd_0;
let fail0 = high_deriv > side_thr;
if MAX_WIDTH_POS == 1 {
return select_i32(fail0, 0, 1);
}
let fail1 = high_deriv > (side_thr >> 2) || transition > q_thr * 4;
let end_thr = (side_thr * 3) >> 4;
let neg3_fail = if MAX_WIDTH_NEG >= 3 {
filter_end_deriv_8bpc_sse41(
buf,
s - stride,
s - 2 * stride,
s - 4 * stride,
t - stride,
t - 2 * stride,
t - 4 * stride,
-2,
3,
-1,
) > end_thr
} else {
false
};
let pos3_fail = filter_end_deriv_8bpc_sse41(
buf,
s,
s + stride,
s + 3 * stride,
t,
t + stride,
t + 3 * stride,
-2,
3,
-1,
) > end_thr;
let fail2 = high_deriv > (side_thr >> 3) || transition > q_thr * 3 || neg3_fail || pos3_fail;
if MAX_WIDTH_POS == 3 {
let mut width = 3;
width = select_i32(fail2, 2, width);
width = select_i32(fail1, 1, width);
return select_i32(fail0, 0, width);
}
let transition4 = transition << 4;
let mut fail4 = false;
let mut fail6 = false;
let mut fail8 = false;
if MAX_WIDTH_POS >= 4 {
let dist = 4i32;
let dist2 = 4i32;
let end_thr4 = (side_thr * dist as u32) >> 4;
let neg_fail = if MAX_WIDTH_NEG >= dist2 {
filter_end_deriv_8bpc_sse41(
buf,
s - stride,
s - (dist2 as isize + 1) * stride,
s - 2 * stride,
t - stride,
t - (dist2 as isize + 1) * stride,
t - 2 * stride,
(1 - dist2) as i16,
-1,
dist2 as i16,
) > end_thr4
} else {
false
};
let pos_fail = filter_end_deriv_8bpc_sse41(
buf,
s,
s + dist2 as isize * stride,
s + stride,
t,
t + dist2 as isize * stride,
t + stride,
(1 - dist2) as i16,
-1,
dist2 as i16,
) > end_thr4;
fail4 = transition4 > q_thr * crate::deblock::Q_FIRST[0] as u32 || neg_fail || pos_fail;
}
if MAX_WIDTH_POS >= 6 {
let dist = 6i32;
let dist2 = 6i32;
let end_thr4 = (side_thr * dist as u32) >> 4;
let neg_fail = if MAX_WIDTH_NEG >= dist2 {
filter_end_deriv_8bpc_sse41(
buf,
s - stride,
s - (dist2 as isize + 1) * stride,
s - 2 * stride,
t - stride,
t - (dist2 as isize + 1) * stride,
t - 2 * stride,
(1 - dist2) as i16,
-1,
dist2 as i16,
) > end_thr4
} else {
false
};
let pos_fail = filter_end_deriv_8bpc_sse41(
buf,
s,
s + dist2 as isize * stride,
s + stride,
t,
t + dist2 as isize * stride,
t + stride,
(1 - dist2) as i16,
-1,
dist2 as i16,
) > end_thr4;
fail6 = transition4 > q_thr * crate::deblock::Q_FIRST[1] as u32 || neg_fail || pos_fail;
}
if MAX_WIDTH_POS >= 8 {
let dist = 8i32;
let dist2 = 7i32;
let end_thr4 = (side_thr * dist as u32) >> 4;
let neg_fail = if MAX_WIDTH_NEG >= dist2 {
filter_end_deriv_8bpc_sse41(
buf,
s - stride,
s - (dist2 as isize + 1) * stride,
s - 2 * stride,
t - stride,
t - (dist2 as isize + 1) * stride,
t - 2 * stride,
(1 - dist2) as i16,
-1,
dist2 as i16,
) > end_thr4
} else {
false
};
let pos_fail = filter_end_deriv_8bpc_sse41(
buf,
s,
s + dist2 as isize * stride,
s + stride,
t,
t + dist2 as isize * stride,
t + stride,
(1 - dist2) as i16,
-1,
dist2 as i16,
) > end_thr4;
fail8 = transition4 > q_thr * crate::deblock::Q_FIRST[2] as u32 || neg_fail || pos_fail;
}
let mut width = MAX_WIDTH_POS;
width = select_i32(MAX_WIDTH_POS >= 8 && fail8, 6, width);
width = select_i32(MAX_WIDTH_POS >= 6 && fail6, 4, width);
width = select_i32(MAX_WIDTH_POS >= 4 && fail4, 3, width);
width = select_i32(fail2, 2, width);
width = select_i32(fail1, 1, width);
select_i32(fail0, 0, width)
}
#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "sse4.1")]
fn deblock_apply_8bpc_sse41_width_constmax<const MAX_WIDTH_NEG: i32, const CONTIG: bool>(
dst: &mut [u8],
off: isize,
stride_line: isize,
stride_tap: isize,
width: i32,
q_thr_clamp: i32,
neg_lossless: bool,
pos_lossless: bool,
) {
debug_assert!((1..=8).contains(&MAX_WIDTH_NEG));
debug_assert!(q_thr_clamp <= i16::MAX as i32);
match width {
1 => deblock_apply_8bpc_sse41_const_oriented::<1, 1, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
),
2 => {
if MAX_WIDTH_NEG >= 2 {
deblock_apply_8bpc_sse41_const_oriented::<2, 2, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
} else {
deblock_apply_8bpc_sse41_const_oriented::<1, 2, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
}
}
3 => {
if MAX_WIDTH_NEG >= 3 {
deblock_apply_8bpc_sse41_const_oriented::<3, 3, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
} else if MAX_WIDTH_NEG >= 2 {
deblock_apply_8bpc_sse41_const_oriented::<2, 3, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
} else {
deblock_apply_8bpc_sse41_const_oriented::<1, 3, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
}
}
4 => {
if MAX_WIDTH_NEG >= 4 {
deblock_apply_8bpc_sse41_const_oriented::<4, 4, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
} else if MAX_WIDTH_NEG >= 3 {
deblock_apply_8bpc_sse41_const_oriented::<3, 4, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
} else if MAX_WIDTH_NEG >= 2 {
deblock_apply_8bpc_sse41_const_oriented::<2, 4, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
} else {
deblock_apply_8bpc_sse41_const_oriented::<1, 4, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
}
}
6 => {
if MAX_WIDTH_NEG >= 6 {
deblock_apply_8bpc_sse41_const_oriented::<6, 6, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
} else if MAX_WIDTH_NEG >= 4 {
deblock_apply_8bpc_sse41_const_oriented::<4, 6, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
} else if MAX_WIDTH_NEG >= 3 {
deblock_apply_8bpc_sse41_const_oriented::<3, 6, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
} else if MAX_WIDTH_NEG >= 2 {
deblock_apply_8bpc_sse41_const_oriented::<2, 6, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
} else {
deblock_apply_8bpc_sse41_const_oriented::<1, 6, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
}
}
8 => {
if MAX_WIDTH_NEG >= 8 {
deblock_apply_8bpc_sse41_const_oriented::<8, 8, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
} else if MAX_WIDTH_NEG >= 6 {
deblock_apply_8bpc_sse41_const_oriented::<6, 8, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
} else if MAX_WIDTH_NEG >= 4 {
deblock_apply_8bpc_sse41_const_oriented::<4, 8, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
} else if MAX_WIDTH_NEG >= 3 {
deblock_apply_8bpc_sse41_const_oriented::<3, 8, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
} else if MAX_WIDTH_NEG >= 2 {
deblock_apply_8bpc_sse41_const_oriented::<2, 8, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
} else {
deblock_apply_8bpc_sse41_const_oriented::<1, 8, CONTIG>(
dst,
off,
stride_line,
stride_tap,
q_thr_clamp,
neg_lossless,
pos_lossless,
)
}
}
_ => {}
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "sse4.1")]
fn deblock_8bpc_sse41_const_max<
const MAX_WIDTH_NEG: i32,
const MAX_WIDTH_POS: i32,
const CONTIG: bool,
>(
dst: &mut [u8],
off: isize,
q_thr: u32,
side_thr: u32,
stridea: isize,
strideb: isize,
pos_lossless: bool,
neg_lossless: bool,
) {
debug_assert!((1..=8).contains(&MAX_WIDTH_POS));
debug_assert!((1..=8).contains(&MAX_WIDTH_NEG));
debug_assert!(MAX_WIDTH_NEG <= MAX_WIDTH_POS);
let width = filter_choice_8bpc_sse41_const::<MAX_WIDTH_NEG, MAX_WIDTH_POS>(
dst,
off,
off + 3 * stridea,
strideb,
q_thr,
side_thr,
);
if width < 1 || (neg_lossless && pos_lossless) {
return;
}
let q_thr_clamp = q_thr as i32 * crate::deblock::Q_THRESH_MULTS[(width - 1) as usize] as i32;
if q_thr_clamp <= 0 {
return;
}
if q_thr_clamp > i16::MAX as i32 {
deblock_apply_8bpc_sse41(
dst,
off,
stridea,
strideb,
width.min(MAX_WIDTH_NEG),
width,
q_thr_clamp,
neg_lossless,
pos_lossless,
);
return;
}
deblock_apply_8bpc_sse41_width_constmax::<MAX_WIDTH_NEG, CONTIG>(
dst,
off,
stridea,
strideb,
width,
q_thr_clamp,
neg_lossless,
pos_lossless,
);
}
#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "sse4.1")]
fn try_deblock_h_sb64_w8_run4_transpose(
dst: &mut [u8],
dst_off: usize,
stride: usize,
qi: usize,
vm: u32,
ll_mask: &[u16],
q_thr: &[u8],
side_thr: &[u8],
) -> bool {
debug_assert!(qi + 3 < 16);
let run = 0x0fu32 << qi;
if (vm & run) != run {
return false;
}
let mut i = 0;
while i < 4 {
let bit = 1u16 << (qi + i);
let q = q_thr[qi + i] as u32;
if q == 0 || ((ll_mask[0] & bit) != 0 && (ll_mask[1] & bit) != 0) {
return false;
}
let off = (dst_off + (qi + i) * 4 * stride) as isize;
let width = filter_choice_8bpc_sse41_const::<8, 8>(
dst,
off,
off + 3 * stride as isize,
1,
q,
side_thr[qi + i] as u32,
);
if width != 8 {
return false;
}
i += 1;
}
let (qlo, qhi) = repeated_qclamp4_w8(q_thr, qi);
let (neg_mask_lo, neg_mask_hi) = repeated_apply_mask4(ll_mask[0], qi);
let (pos_mask_lo, pos_mask_hi) = repeated_apply_mask4(ll_mask[1], qi);
let off = (dst_off + qi * 4 * stride) as isize;
deblock_apply_8bpc_sse41_h_w8x4_transpose(
dst,
off,
stride as isize,
qlo,
qhi,
neg_mask_lo,
neg_mask_hi,
pos_mask_lo,
pos_mask_hi,
);
true
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn deblock_mask_class_bits(mask: u16, higher: u16, both_lossless: u16) -> u32 {
(mask & !higher & !both_lossless) as u32
}
#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "sse4.1")]
fn deblock_sb64_8bpc_sse41_mask<
const MAX_WIDTH_NEG: i32,
const MAX_WIDTH_POS: i32,
const HORIZONTAL: bool,
const CONTIG: bool,
>(
dst: &mut [u8],
dst_off: usize,
stride: usize,
mut vm: u32,
ll_mask: &[u16],
q_thr: &[u8],
side_thr: &[u8],
) {
debug_assert!(MAX_WIDTH_NEG <= MAX_WIDTH_POS);
if HORIZONTAL && !CONTIG && MAX_WIDTH_NEG == 8 && MAX_WIDTH_POS == 8 {
let mut qi = 0usize;
while qi <= 12 {
let run = 0x0fu32 << qi;
if (vm & run) == run
&& try_deblock_h_sb64_w8_run4_transpose(
dst, dst_off, stride, qi, vm, ll_mask, q_thr, side_thr,
)
{
vm &= !run;
}
qi += 4;
}
}
while vm != 0 {
let qi = vm.trailing_zeros() as usize;
let bit = 1u32 << qi;
let q = q_thr[qi] as u32;
if q != 0 {
let pos_ll = (ll_mask[1] as u32 & bit) != 0;
let neg_ll = (ll_mask[0] as u32 & bit) != 0;
if !(pos_ll && neg_ll) {
let side = side_thr[qi] as u32;
let off = if HORIZONTAL {
(dst_off + qi * 4 * stride) as isize
} else {
(dst_off + qi * 4) as isize
};
let stridea = if HORIZONTAL { stride as isize } else { 1 };
let strideb = if HORIZONTAL { 1 } else { stride as isize };
deblock_8bpc_sse41_const_max::<MAX_WIDTH_NEG, MAX_WIDTH_POS, CONTIG>(
dst, off, q, side, stridea, strideb, pos_ll, neg_ll,
);
}
}
vm &= vm - 1;
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) fn deblock_h_sb64y_8bpc_sse41(
dst: &mut [u8],
dst_off: usize,
stride: usize,
vmask: &[u16],
ll_mask: &[u16],
q_thr: &[u8],
side_thr: &[u8],
edge: bool,
) {
let both_lossless = ll_mask[0] & ll_mask[1];
let m3 = deblock_mask_class_bits(vmask[3], 0, both_lossless);
let m2 = deblock_mask_class_bits(vmask[2], vmask[3], both_lossless);
let m1 = deblock_mask_class_bits(vmask[1], vmask[2] | vmask[3], both_lossless);
let m0 = deblock_mask_class_bits(vmask[0], vmask[1] | vmask[2] | vmask[3], both_lossless);
if m0 != 0 {
deblock_sb64_8bpc_sse41_mask::<1, 1, true, false>(
dst, dst_off, stride, m0, ll_mask, q_thr, side_thr,
);
}
if m1 != 0 {
deblock_sb64_8bpc_sse41_mask::<3, 3, true, false>(
dst, dst_off, stride, m1, ll_mask, q_thr, side_thr,
);
}
if m2 != 0 {
deblock_sb64_8bpc_sse41_mask::<6, 6, true, false>(
dst, dst_off, stride, m2, ll_mask, q_thr, side_thr,
);
}
if m3 != 0 {
if edge {
deblock_sb64_8bpc_sse41_mask::<6, 8, true, false>(
dst, dst_off, stride, m3, ll_mask, q_thr, side_thr,
);
} else {
deblock_sb64_8bpc_sse41_mask::<8, 8, true, false>(
dst, dst_off, stride, m3, ll_mask, q_thr, side_thr,
);
}
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) fn deblock_v_sb64y_8bpc_sse41(
dst: &mut [u8],
dst_off: usize,
stride: usize,
vmask: &[u16],
ll_mask: &[u16],
q_thr: &[u8],
side_thr: &[u8],
edge: bool,
) {
let both_lossless = ll_mask[0] & ll_mask[1];
let m3 = deblock_mask_class_bits(vmask[3], 0, both_lossless);
let m2 = deblock_mask_class_bits(vmask[2], vmask[3], both_lossless);
let m1 = deblock_mask_class_bits(vmask[1], vmask[2] | vmask[3], both_lossless);
let m0 = deblock_mask_class_bits(vmask[0], vmask[1] | vmask[2] | vmask[3], both_lossless);
if m0 != 0 {
deblock_sb64_8bpc_sse41_mask::<1, 1, false, true>(
dst, dst_off, stride, m0, ll_mask, q_thr, side_thr,
);
}
if m1 != 0 {
deblock_sb64_8bpc_sse41_mask::<3, 3, false, true>(
dst, dst_off, stride, m1, ll_mask, q_thr, side_thr,
);
}
if m2 != 0 {
deblock_sb64_8bpc_sse41_mask::<6, 6, false, true>(
dst, dst_off, stride, m2, ll_mask, q_thr, side_thr,
);
}
if m3 != 0 {
if edge {
deblock_sb64_8bpc_sse41_mask::<6, 8, false, true>(
dst, dst_off, stride, m3, ll_mask, q_thr, side_thr,
);
} else {
deblock_sb64_8bpc_sse41_mask::<8, 8, false, true>(
dst, dst_off, stride, m3, ll_mask, q_thr, side_thr,
);
}
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) fn deblock_h_sb64uv_8bpc_sse41(
dst: &mut [u8],
dst_off: usize,
stride: usize,
vmask: &[u16],
ll_mask: &[u16],
q_thr: &[u8],
side_thr: &[u8],
edge: bool,
) {
let both_lossless = ll_mask[0] & ll_mask[1];
let m2 = deblock_mask_class_bits(vmask[2], 0, both_lossless);
let m1 = deblock_mask_class_bits(vmask[1], vmask[2], both_lossless);
let m0 = deblock_mask_class_bits(vmask[0], vmask[1] | vmask[2], both_lossless);
if m0 != 0 {
deblock_sb64_8bpc_sse41_mask::<1, 1, true, false>(
dst, dst_off, stride, m0, ll_mask, q_thr, side_thr,
);
}
if m1 != 0 {
if edge {
deblock_sb64_8bpc_sse41_mask::<2, 3, true, false>(
dst, dst_off, stride, m1, ll_mask, q_thr, side_thr,
);
} else {
deblock_sb64_8bpc_sse41_mask::<3, 3, true, false>(
dst, dst_off, stride, m1, ll_mask, q_thr, side_thr,
);
}
}
if m2 != 0 {
if edge {
deblock_sb64_8bpc_sse41_mask::<2, 4, true, false>(
dst, dst_off, stride, m2, ll_mask, q_thr, side_thr,
);
} else {
deblock_sb64_8bpc_sse41_mask::<4, 4, true, false>(
dst, dst_off, stride, m2, ll_mask, q_thr, side_thr,
);
}
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) fn deblock_v_sb64uv_8bpc_sse41(
dst: &mut [u8],
dst_off: usize,
stride: usize,
vmask: &[u16],
ll_mask: &[u16],
q_thr: &[u8],
side_thr: &[u8],
edge: bool,
) {
let both_lossless = ll_mask[0] & ll_mask[1];
let m2 = deblock_mask_class_bits(vmask[2], 0, both_lossless);
let m1 = deblock_mask_class_bits(vmask[1], vmask[2], both_lossless);
let m0 = deblock_mask_class_bits(vmask[0], vmask[1] | vmask[2], both_lossless);
if m0 != 0 {
deblock_sb64_8bpc_sse41_mask::<1, 1, false, true>(
dst, dst_off, stride, m0, ll_mask, q_thr, side_thr,
);
}
if m1 != 0 {
if edge {
deblock_sb64_8bpc_sse41_mask::<2, 3, false, true>(
dst, dst_off, stride, m1, ll_mask, q_thr, side_thr,
);
} else {
deblock_sb64_8bpc_sse41_mask::<3, 3, false, true>(
dst, dst_off, stride, m1, ll_mask, q_thr, side_thr,
);
}
}
if m2 != 0 {
if edge {
deblock_sb64_8bpc_sse41_mask::<2, 4, false, true>(
dst, dst_off, stride, m2, ll_mask, q_thr, side_thr,
);
} else {
deblock_sb64_8bpc_sse41_mask::<4, 4, false, true>(
dst, dst_off, stride, m2, ll_mask, q_thr, side_thr,
);
}
}
}