#[cfg(target_arch = "x86")]
use std::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use std::arch::x86_64::*;
#[inline(always)]
fn ccso_tail_hbd(
dst_row: &mut [u8],
tmp: &[u16],
row: usize,
x0: usize,
x1: usize,
ss_hor: usize,
shift: u32,
luma_offset: isize,
quant_step: i32,
edge_clf: u32,
bo_only: bool,
) {
for (x, out) in dst_row[x0..x1].iter_mut().enumerate() {
let x = x + x0;
let ti = row + (x << ss_hor);
let c = tmp[ti] as i32;
let band = (c as u32 >> shift) as u8;
if bo_only {
*out = band;
} else {
let cls0 = crate::ccso::ccso_score(
tmp[(ti as isize + luma_offset) as usize] as i32 - c,
quant_step,
edge_clf,
);
let cls1 = crate::ccso::ccso_score(
tmp[(ti as isize - luma_offset) as usize] as i32 - c,
quant_step,
edge_clf,
);
*out = ((cls0 << 5) | (cls1 << 3)) as u8 | band;
}
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn ccso_classify_epi16(diff: __m128i, q: __m128i, nq: __m128i, edge_clf: u32) -> __m128i {
let zero = _mm_setzero_si128();
let one = _mm_set1_epi16(1);
let two = _mm_set1_epi16(2);
let gt = if edge_clf == 0 {
_mm_cmpgt_epi16(diff, q)
} else {
zero
};
let lt = _mm_cmpgt_epi16(nq, diff);
let cls = _mm_blendv_epi8(one, two, gt);
_mm_blendv_epi8(cls, zero, lt)
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn ccso_make_idx_8x16(
c: __m128i,
p0: __m128i,
p1: __m128i,
shiftv: __m128i,
q: __m128i,
nq: __m128i,
edge_clf: u32,
) -> __m128i {
let band = _mm_srl_epi16(c, shiftv);
let cls0 = ccso_classify_epi16(_mm_sub_epi16(p0, c), q, nq, edge_clf);
let cls1 = ccso_classify_epi16(_mm_sub_epi16(p1, c), q, nq, edge_clf);
_mm_or_si128(
band,
_mm_or_si128(_mm_slli_epi16(cls0, 5), _mm_slli_epi16(cls1, 3)),
)
}
#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) fn ccso_prep_lut_hbd_sse41(
dst: &mut [u8],
dst_stride: usize,
tmp: &[u16],
tmp_stride: usize,
o: usize,
w: usize,
h: usize,
ss_hor: usize,
ss_ver: usize,
shift: u32,
luma_offset: isize,
quant_step: i32,
edge_clf: u32,
bo_only: bool,
) {
if ss_hor != 0 {
crate::ccso::ccso_prep_lut_hbd_scalar(
dst,
dst_stride,
tmp,
tmp_stride,
o,
w,
h,
ss_hor,
ss_ver,
shift,
luma_offset,
quant_step,
edge_clf,
bo_only,
);
return;
}
let q = _mm_set1_epi16(quant_step as i16);
let nq = _mm_set1_epi16((-quant_step) as i16);
let shiftv = _mm_cvtsi32_si128(shift as i32);
for (y, dst_row) in dst.chunks_mut(dst_stride).take(h).enumerate() {
let row = o + (y << ss_ver) * tmp_stride;
let dst_row = &mut dst_row[..w];
let n8 = {
let (dst8, _) = dst_row.as_chunks_mut::<8>();
if bo_only {
for (chunk_idx, out8) in dst8.iter_mut().enumerate() {
let x = chunk_idx * 8;
let c = unsafe { _mm_loadu_si128(tmp.as_ptr().add(row + x) as *const __m128i) };
let out16 = _mm_srl_epi16(c, shiftv);
let out = _mm_packus_epi16(out16, out16);
unsafe { _mm_storel_epi64(out8.as_mut_ptr() as *mut __m128i, out) };
}
} else {
for (chunk_idx, out8) in dst8.iter_mut().enumerate() {
let x = chunk_idx * 8;
let c = unsafe { _mm_loadu_si128(tmp.as_ptr().add(row + x) as *const __m128i) };
let p0 = unsafe {
_mm_loadu_si128(
tmp.as_ptr()
.add(((row + x) as isize + luma_offset) as usize)
as *const __m128i,
)
};
let p1 = unsafe {
_mm_loadu_si128(
tmp.as_ptr()
.add(((row + x) as isize - luma_offset) as usize)
as *const __m128i,
)
};
let out16 = ccso_make_idx_8x16(c, p0, p1, shiftv, q, nq, edge_clf);
let out = _mm_packus_epi16(out16, out16);
unsafe { _mm_storel_epi64(out8.as_mut_ptr() as *mut __m128i, out) };
}
}
dst8.len()
};
ccso_tail_hbd(
dst_row,
tmp,
row,
n8 * 8,
w,
ss_hor,
shift,
luma_offset,
quant_step,
edge_clf,
bo_only,
);
}
}
#[inline(always)]
fn fill_offsets_8(out: &mut [i16; 8], idx: &[u8; 8], offset_map: &[i8; 256]) {
for (out, &idx) in out.iter_mut().zip(idx.iter()) {
*out = offset_map[idx as usize] as i16;
}
}
#[inline(always)]
fn ccso_add_4x4_hbd(
dst_rows: &mut [u16],
dst_stride: usize,
idx_rows: &[u8],
idx_stride: usize,
block_h: usize,
offset_map: &[i8; 256],
xx: usize,
bitdepth_max: i32,
) {
for (yy, idx_row) in idx_rows.chunks(idx_stride).take(block_h).enumerate() {
let dst_row = &mut dst_rows[yy * dst_stride..];
let dst4 = &mut dst_row[xx..xx + 4].as_chunks_mut::<4>().0[0];
let idx4 = &idx_row[xx..xx + 4].as_chunks::<4>().0[0];
for (dst, &idx) in dst4.iter_mut().zip(idx4.iter()) {
let off = offset_map[idx as usize];
let cur = *dst as i32;
*dst = (cur + off as i32).clamp(0, bitdepth_max) as u16;
}
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) fn ccso_add_hbd_sse41(
dst: &mut [u16],
dst_stride: usize,
idx_buf: &[u8],
idx_stride: usize,
offset_idxs: &[u8],
offset_lut: &[i8],
w: usize,
h: usize,
ll_mask: &[[u16; 4]],
bitdepth_max: i32,
) {
let zero = _mm_setzero_si128();
let maxv = _mm_set1_epi16(bitdepth_max as i16);
let offset_map = crate::ccso::ccso_build_offset_map(offset_idxs, offset_lut);
let mut off_tmp = [0i16; 8];
let n_blocks = (h + 3) >> 2;
for (by, mask) in (0..h).step_by(4).zip(ll_mask[..n_blocks].iter()) {
let block_h = (h - by).min(4);
let dst_rows = &mut dst[by * dst_stride..];
let idx_row_start = by * idx_stride;
let idx_rows = &idx_buf[idx_row_start..idx_row_start + block_h * idx_stride];
let row_mask = mask[0];
let mut xx = 0usize;
while xx + 8 <= w {
let bx = xx >> 2;
if ((row_mask >> bx) & 0x03) == 0 {
for (yy, idx_row) in idx_rows.chunks(idx_stride).take(block_h).enumerate() {
let dst_row = &mut dst_rows[yy * dst_stride..];
let idx8 = &idx_row[xx..xx + 8].as_chunks::<8>().0[0];
fill_offsets_8(&mut off_tmp, idx8, &offset_map);
let off = unsafe { _mm_loadu_si128(off_tmp.as_ptr() as *const __m128i) };
let dst8 = &mut dst_row[xx..xx + 8].as_chunks_mut::<8>().0[0];
let cur = unsafe { _mm_loadu_si128(dst8.as_ptr() as *const __m128i) };
let out = _mm_min_epi16(_mm_max_epi16(_mm_add_epi16(cur, off), zero), maxv);
unsafe { _mm_storeu_si128(dst8.as_mut_ptr() as *mut __m128i, out) };
}
xx += 8;
} else {
for _ in 0..2 {
let bx = xx >> 2;
if row_mask & (1 << bx) == 0 {
ccso_add_4x4_hbd(
dst_rows,
dst_stride,
idx_rows,
idx_stride,
block_h,
&offset_map,
xx,
bitdepth_max,
);
}
xx += 4;
}
}
}
while xx < w {
let bx = xx >> 2;
if row_mask & (1 << bx) == 0 {
ccso_add_4x4_hbd(
dst_rows,
dst_stride,
idx_rows,
idx_stride,
block_h,
&offset_map,
xx,
bitdepth_max,
);
}
xx += 4;
}
}
}