#[cfg(target_arch = "x86")]
use std::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use std::arch::x86_64::*;
#[inline(always)]
fn ccso_tail_8bpc(
dst_row: &mut [u8],
tmp: &[u8],
row: usize,
x0: usize,
x1: usize,
ss_hor: usize,
shift: u32,
luma_offset: isize,
quant_step: i32,
edge_clf: u32,
bo_only: bool,
) {
for (x, out) in dst_row[x0..x1].iter_mut().enumerate() {
let x = x + x0;
let ti = row + (x << ss_hor);
let c = tmp[ti] as i32;
let band = (c as u32 >> shift) as u8;
if bo_only {
*out = band;
} else {
let cls0 = crate::ccso::ccso_score(
tmp[(ti as isize + luma_offset) as usize] as i32 - c,
quant_step,
edge_clf,
);
let cls1 = crate::ccso::ccso_score(
tmp[(ti as isize - luma_offset) as usize] as i32 - c,
quant_step,
edge_clf,
);
*out = ((cls0 << 5) | (cls1 << 3)) as u8 | band;
}
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn ccso_classify_epi16(diff: __m128i, q: __m128i, nq: __m128i, edge_clf: u32) -> __m128i {
let zero = _mm_setzero_si128();
let one = _mm_set1_epi16(1);
let two = _mm_set1_epi16(2);
let gt = if edge_clf == 0 {
_mm_cmpgt_epi16(diff, q)
} else {
zero
};
let lt = _mm_cmpgt_epi16(nq, diff);
let cls = _mm_blendv_epi8(one, two, gt);
_mm_blendv_epi8(cls, zero, lt)
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn ccso_make_idx_8x16(
c: __m128i,
p0: __m128i,
p1: __m128i,
shiftv: __m128i,
q: __m128i,
nq: __m128i,
edge_clf: u32,
) -> __m128i {
let band = _mm_srl_epi16(c, shiftv);
let cls0 = ccso_classify_epi16(_mm_sub_epi16(p0, c), q, nq, edge_clf);
let cls1 = ccso_classify_epi16(_mm_sub_epi16(p1, c), q, nq, edge_clf);
_mm_or_si128(
band,
_mm_or_si128(_mm_slli_epi16::<5>(cls0), _mm_slli_epi16::<3>(cls1)),
)
}
#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) fn ccso_prep_lut_8bpc_sse41(
dst: &mut [u8],
dst_stride: usize,
tmp: &[u8],
tmp_stride: usize,
o: usize,
w: usize,
h: usize,
ss_hor: usize,
ss_ver: usize,
shift: u32,
luma_offset: isize,
quant_step: i32,
edge_clf: u32,
bo_only: bool,
) {
if ss_hor != 0 {
crate::ccso::ccso_prep_lut_8bpc_scalar(
dst,
dst_stride,
tmp,
tmp_stride,
o,
w,
h,
ss_hor,
ss_ver,
shift,
luma_offset,
quant_step,
edge_clf,
bo_only,
);
return;
}
let zero = _mm_setzero_si128();
let q = _mm_set1_epi16(quant_step as i16);
let nq = _mm_set1_epi16((-quant_step) as i16);
let shiftv = _mm_cvtsi32_si128(shift as i32);
for (y, dst_row) in dst.chunks_mut(dst_stride).take(h).enumerate() {
let row = o + (y << ss_ver) * tmp_stride;
let dst_row = &mut dst_row[..w];
let n16 = {
let (dst16, _) = dst_row.as_chunks_mut::<16>();
if bo_only {
for (chunk_idx, out16) in dst16.iter_mut().enumerate() {
let x = chunk_idx * 16;
let c = unsafe { _mm_loadu_si128(tmp.as_ptr().add(row + x) as *const __m128i) };
let lo = _mm_srl_epi16(_mm_unpacklo_epi8(c, zero), shiftv);
let hi = _mm_srl_epi16(_mm_unpackhi_epi8(c, zero), shiftv);
let out = _mm_packus_epi16(lo, hi);
unsafe { _mm_storeu_si128(out16.as_mut_ptr() as *mut __m128i, out) };
}
} else {
for (chunk_idx, out16) in dst16.iter_mut().enumerate() {
let x = chunk_idx * 16;
let c8 =
unsafe { _mm_loadu_si128(tmp.as_ptr().add(row + x) as *const __m128i) };
let p08 = unsafe {
_mm_loadu_si128(
tmp.as_ptr()
.add(((row + x) as isize + luma_offset) as usize)
as *const __m128i,
)
};
let p18 = unsafe {
_mm_loadu_si128(
tmp.as_ptr()
.add(((row + x) as isize - luma_offset) as usize)
as *const __m128i,
)
};
let clo = _mm_unpacklo_epi8(c8, zero);
let chi = _mm_unpackhi_epi8(c8, zero);
let p0lo = _mm_unpacklo_epi8(p08, zero);
let p0hi = _mm_unpackhi_epi8(p08, zero);
let p1lo = _mm_unpacklo_epi8(p18, zero);
let p1hi = _mm_unpackhi_epi8(p18, zero);
let lo = ccso_make_idx_8x16(clo, p0lo, p1lo, shiftv, q, nq, edge_clf);
let hi = ccso_make_idx_8x16(chi, p0hi, p1hi, shiftv, q, nq, edge_clf);
let out = _mm_packus_epi16(lo, hi);
unsafe { _mm_storeu_si128(out16.as_mut_ptr() as *mut __m128i, out) };
}
}
dst16.len()
};
ccso_tail_8bpc(
dst_row,
tmp,
row,
n16 * 16,
w,
ss_hor,
shift,
luma_offset,
quant_step,
edge_clf,
bo_only,
);
}
}
#[inline(always)]
fn fill_offsets_16(out: &mut [i8; 16], idx: &[u8; 16], offset_map: &[i8; 256]) {
for (out, &idx) in out.iter_mut().zip(idx.iter()) {
*out = offset_map[idx as usize];
}
}
#[inline(always)]
fn ccso_add_4x4_8bpc(
dst_rows: &mut [u8],
dst_stride: usize,
idx_rows: &[u8],
idx_stride: usize,
block_h: usize,
offset_map: &[i8; 256],
xx: usize,
) {
for (yy, idx_row) in idx_rows.chunks(idx_stride).take(block_h).enumerate() {
let dst_row = &mut dst_rows[yy * dst_stride..];
let dst4 = &mut dst_row[xx..xx + 4].as_chunks_mut::<4>().0[0];
let idx4 = &idx_row[xx..xx + 4].as_chunks::<4>().0[0];
for (dst, &idx) in dst4.iter_mut().zip(idx4.iter()) {
let off = offset_map[idx as usize];
let cur = *dst as i32;
*dst = (cur + off as i32).clamp(0, 255) as u8;
}
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) fn ccso_add_8bpc_sse41(
dst: &mut [u8],
dst_stride: usize,
idx_buf: &[u8],
idx_stride: usize,
offset_idxs: &[u8],
offset_lut: &[i8],
w: usize,
h: usize,
ll_mask: &[[u16; 4]],
) {
let zero = _mm_setzero_si128();
let offset_map = crate::ccso::ccso_build_offset_map(offset_idxs, offset_lut);
let mut off_tmp = [0i8; 16];
let n_blocks = (h + 3) >> 2;
for (by, mask) in (0..h).step_by(4).zip(ll_mask[..n_blocks].iter()) {
let block_h = (h - by).min(4);
let dst_rows = &mut dst[by * dst_stride..];
let idx_row_start = by * idx_stride;
let idx_rows = &idx_buf[idx_row_start..idx_row_start + block_h * idx_stride];
let row_mask = mask[0];
let mut xx = 0usize;
while xx + 16 <= w {
let bx = xx >> 2;
if ((row_mask >> bx) & 0x0f) == 0 {
for (yy, idx_row) in idx_rows.chunks(idx_stride).take(block_h).enumerate() {
let dst_row = &mut dst_rows[yy * dst_stride..];
let idx16 = &idx_row[xx..xx + 16].as_chunks::<16>().0[0];
fill_offsets_16(&mut off_tmp, idx16, &offset_map);
let off = unsafe { _mm_loadu_si128(off_tmp.as_ptr() as *const __m128i) };
let off_lo = _mm_cvtepi8_epi16(off);
let off_hi = _mm_cvtepi8_epi16(_mm_srli_si128(off, 8));
let dst16 = &mut dst_row[xx..xx + 16].as_chunks_mut::<16>().0[0];
let cur = unsafe { _mm_loadu_si128(dst16.as_ptr() as *const __m128i) };
let cur_lo = _mm_unpacklo_epi8(cur, zero);
let cur_hi = _mm_unpackhi_epi8(cur, zero);
let out = _mm_packus_epi16(
_mm_add_epi16(cur_lo, off_lo),
_mm_add_epi16(cur_hi, off_hi),
);
unsafe { _mm_storeu_si128(dst16.as_mut_ptr() as *mut __m128i, out) };
}
xx += 16;
} else {
for _ in 0..4 {
let bx = xx >> 2;
if row_mask & (1 << bx) == 0 {
ccso_add_4x4_8bpc(
dst_rows,
dst_stride,
idx_rows,
idx_stride,
block_h,
&offset_map,
xx,
);
}
xx += 4;
}
}
}
while xx < w {
let bx = xx >> 2;
if row_mask & (1 << bx) == 0 {
ccso_add_4x4_8bpc(
dst_rows,
dst_stride,
idx_rows,
idx_stride,
block_h,
&offset_map,
xx,
);
}
xx += 4;
}
}
}