#[cfg(target_arch = "x86")]
use std::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use std::arch::x86_64::*;
#[inline(always)]
fn ccso_tail_8bpc<const SS_HOR: usize, const BO_ONLY: bool>(
dst_row: &mut [u8],
tmp: &[u8],
row: usize,
x0: usize,
x1: usize,
shift: u32,
luma_offset: isize,
quant_step: i32,
edge_clf: u32,
) {
for (x, out) in dst_row[x0..x1].iter_mut().enumerate() {
let x = x + x0;
let ti = row + (x << SS_HOR);
let c = tmp[ti] as i32;
let band = (c as u32 >> shift) as u8;
if BO_ONLY {
*out = band;
} else {
let cls0 = crate::ccso::ccso_score(
tmp[(ti as isize + luma_offset) as usize] as i32 - c,
quant_step,
edge_clf,
);
let cls1 = crate::ccso::ccso_score(
tmp[(ti as isize - luma_offset) as usize] as i32 - c,
quant_step,
edge_clf,
);
*out = ((cls0 << 5) | (cls1 << 3)) as u8 | band;
}
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn ccso_classify_epi16(diff: __m128i, q: __m128i, nq: __m128i, edge_clf: u32) -> __m128i {
let zero = _mm_setzero_si128();
let one = _mm_set1_epi16(1);
let two = _mm_set1_epi16(2);
let gt = if edge_clf == 0 {
_mm_cmpgt_epi16(diff, q)
} else {
zero
};
let lt = _mm_cmpgt_epi16(nq, diff);
let cls = _mm_blendv_epi8(one, two, gt);
_mm_blendv_epi8(cls, zero, lt)
}
#[inline]
#[target_feature(enable = "avx2")]
fn ccso_make_idx_8x16(
c: __m128i,
p0: __m128i,
p1: __m128i,
shiftv: __m128i,
q: __m128i,
nq: __m128i,
edge_clf: u32,
) -> __m128i {
let band = _mm_srl_epi16(c, shiftv);
let cls0 = ccso_classify_epi16(_mm_sub_epi16(p0, c), q, nq, edge_clf);
let cls1 = ccso_classify_epi16(_mm_sub_epi16(p1, c), q, nq, edge_clf);
_mm_or_si128(
band,
_mm_or_si128(_mm_slli_epi16(cls0, 5), _mm_slli_epi16(cls1, 3)),
)
}
#[inline]
#[target_feature(enable = "avx2")]
fn load_u8x16_step2(ptr: *const u8) -> __m128i {
let mask = _mm256_setr_epi8(
0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, 0, 2, 4, 6, 8, 10, 12, 14, -1,
-1, -1, -1, -1, -1, -1, -1,
);
let v = unsafe { _mm256_loadu_si256(ptr as *const __m256i) };
let v = _mm256_shuffle_epi8(v, mask);
_mm_unpacklo_epi64(_mm256_castsi256_si128(v), _mm256_extracti128_si256::<1>(v))
}
#[inline]
#[target_feature(enable = "avx2")]
fn load_u8x8_step2(ptr: *const u8) -> __m128i {
let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1);
let v = unsafe { _mm_loadu_si128(ptr as *const __m128i) };
_mm_shuffle_epi8(v, mask)
}
#[inline]
#[target_feature(enable = "avx2")]
fn load_u8x16_subsampled<const SS_HOR: usize>(tmp: &[u8], base: usize) -> __m128i {
if SS_HOR == 0 {
unsafe { _mm_loadu_si128(tmp.as_ptr().add(base) as *const __m128i) }
} else {
load_u8x16_step2(unsafe { tmp.as_ptr().add(base) })
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn load_u8x8_subsampled<const SS_HOR: usize>(tmp: &[u8], base: usize) -> __m128i {
if SS_HOR == 0 {
unsafe { _mm_loadl_epi64(tmp.as_ptr().add(base) as *const __m128i) }
} else {
load_u8x8_step2(unsafe { tmp.as_ptr().add(base) })
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn store_idx8(dst: &mut [u8; 8], out16: __m128i) {
let out = _mm_packus_epi16(out16, out16);
unsafe { _mm_storel_epi64(dst.as_mut_ptr() as *mut __m128i, out) };
}
#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "avx2")]
fn ccso_prep_lut_8bpc_avx2_impl<const SS_HOR: usize, const SS_VER: usize, const BO_ONLY: bool>(
dst: &mut [u8],
dst_stride: usize,
tmp: &[u8],
tmp_stride: usize,
o: usize,
w: usize,
h: usize,
shift: u32,
luma_offset: isize,
quant_step: i32,
edge_clf: u32,
) {
let zero = _mm_setzero_si128();
let q = _mm_set1_epi16(quant_step as i16);
let nq = _mm_set1_epi16((-quant_step) as i16);
let shiftv = _mm_cvtsi32_si128(shift as i32);
for (y, dst_row) in dst.chunks_mut(dst_stride).take(h).enumerate() {
let row = o + (y << SS_VER) * tmp_stride;
let dst_row = &mut dst_row[..w];
let mut x;
if BO_ONLY {
let n16 = {
let (dst16, _) = dst_row.as_chunks_mut::<16>();
for (chunk_idx, out16) in dst16.iter_mut().enumerate() {
let x = chunk_idx * 16;
let base = row + (x << SS_HOR);
let c = load_u8x16_subsampled::<SS_HOR>(tmp, base);
let lo = _mm_srl_epi16(_mm_unpacklo_epi8(c, zero), shiftv);
let hi = _mm_srl_epi16(_mm_unpackhi_epi8(c, zero), shiftv);
let out = _mm_packus_epi16(lo, hi);
unsafe { _mm_storeu_si128(out16.as_mut_ptr() as *mut __m128i, out) };
}
dst16.len()
};
x = n16 * 16;
if x + 8 <= w {
let base = row + (x << SS_HOR);
let c = load_u8x8_subsampled::<SS_HOR>(tmp, base);
let out16 = _mm_srl_epi16(_mm_unpacklo_epi8(c, zero), shiftv);
let out8 = &mut dst_row[x..x + 8].as_chunks_mut::<8>().0[0];
store_idx8(out8, out16);
x += 8;
}
} else {
let n16 = {
let (dst16, _) = dst_row.as_chunks_mut::<16>();
for (chunk_idx, out16) in dst16.iter_mut().enumerate() {
let x = chunk_idx * 16;
let base = row + (x << SS_HOR);
let c8 = load_u8x16_subsampled::<SS_HOR>(tmp, base);
let p08 = load_u8x16_subsampled::<SS_HOR>(
tmp,
(base as isize + luma_offset) as usize,
);
let p18 = load_u8x16_subsampled::<SS_HOR>(
tmp,
(base as isize - luma_offset) as usize,
);
let clo = _mm_unpacklo_epi8(c8, zero);
let chi = _mm_unpackhi_epi8(c8, zero);
let p0lo = _mm_unpacklo_epi8(p08, zero);
let p0hi = _mm_unpackhi_epi8(p08, zero);
let p1lo = _mm_unpacklo_epi8(p18, zero);
let p1hi = _mm_unpackhi_epi8(p18, zero);
let lo = ccso_make_idx_8x16(clo, p0lo, p1lo, shiftv, q, nq, edge_clf);
let hi = ccso_make_idx_8x16(chi, p0hi, p1hi, shiftv, q, nq, edge_clf);
let out = _mm_packus_epi16(lo, hi);
unsafe { _mm_storeu_si128(out16.as_mut_ptr() as *mut __m128i, out) };
}
dst16.len()
};
x = n16 * 16;
if x + 8 <= w {
let base = row + (x << SS_HOR);
let c8 = load_u8x8_subsampled::<SS_HOR>(tmp, base);
let p08 =
load_u8x8_subsampled::<SS_HOR>(tmp, (base as isize + luma_offset) as usize);
let p18 =
load_u8x8_subsampled::<SS_HOR>(tmp, (base as isize - luma_offset) as usize);
let out16 = ccso_make_idx_8x16(
_mm_unpacklo_epi8(c8, zero),
_mm_unpacklo_epi8(p08, zero),
_mm_unpacklo_epi8(p18, zero),
shiftv,
q,
nq,
edge_clf,
);
let out8 = &mut dst_row[x..x + 8].as_chunks_mut::<8>().0[0];
store_idx8(out8, out16);
x += 8;
}
}
ccso_tail_8bpc::<SS_HOR, BO_ONLY>(
dst_row,
tmp,
row,
x,
w,
shift,
luma_offset,
quant_step,
edge_clf,
);
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn ccso_prep_lut_8bpc_avx2(
dst: &mut [u8],
dst_stride: usize,
tmp: &[u8],
tmp_stride: usize,
o: usize,
w: usize,
h: usize,
ss_hor: usize,
ss_ver: usize,
shift: u32,
luma_offset: isize,
quant_step: i32,
edge_clf: u32,
bo_only: bool,
) {
match (ss_hor, ss_ver, bo_only) {
(0, 0, true) => ccso_prep_lut_8bpc_avx2_impl::<0, 0, true>(
dst,
dst_stride,
tmp,
tmp_stride,
o,
w,
h,
shift,
luma_offset,
quant_step,
edge_clf,
),
(0, 0, false) => ccso_prep_lut_8bpc_avx2_impl::<0, 0, false>(
dst,
dst_stride,
tmp,
tmp_stride,
o,
w,
h,
shift,
luma_offset,
quant_step,
edge_clf,
),
(1, 0, true) => ccso_prep_lut_8bpc_avx2_impl::<1, 0, true>(
dst,
dst_stride,
tmp,
tmp_stride,
o,
w,
h,
shift,
luma_offset,
quant_step,
edge_clf,
),
(1, 0, false) => ccso_prep_lut_8bpc_avx2_impl::<1, 0, false>(
dst,
dst_stride,
tmp,
tmp_stride,
o,
w,
h,
shift,
luma_offset,
quant_step,
edge_clf,
),
(1, 1, true) => ccso_prep_lut_8bpc_avx2_impl::<1, 1, true>(
dst,
dst_stride,
tmp,
tmp_stride,
o,
w,
h,
shift,
luma_offset,
quant_step,
edge_clf,
),
(1, 1, false) => ccso_prep_lut_8bpc_avx2_impl::<1, 1, false>(
dst,
dst_stride,
tmp,
tmp_stride,
o,
w,
h,
shift,
luma_offset,
quant_step,
edge_clf,
),
_ => crate::ccso::ccso_prep_lut_8bpc_scalar(
dst,
dst_stride,
tmp,
tmp_stride,
o,
w,
h,
ss_hor,
ss_ver,
shift,
luma_offset,
quant_step,
edge_clf,
bo_only,
),
}
}
#[inline(always)]
fn fill_offsets_16(out: &mut [i8; 16], idx: &[u8; 16], offset_map: &[i8; 256]) {
for (out, &idx) in out.iter_mut().zip(idx.iter()) {
*out = offset_map[idx as usize];
}
}
#[inline(always)]
fn fill_offsets_32(out: &mut [i8; 32], idx: &[u8; 32], offset_map: &[i8; 256]) {
for (out, &idx) in out.iter_mut().zip(idx.iter()) {
*out = offset_map[idx as usize];
}
}
#[inline(always)]
fn fill_offsets_4_i16(out: &mut [i16; 8], idx: &[u8; 4], offset_map: &[i8; 256]) {
for (out, &idx) in out[..4].iter_mut().zip(idx.iter()) {
*out = offset_map[idx as usize] as i16;
}
}
#[repr(C, align(16))]
pub(crate) struct AlignedSseS16(pub(crate) [i16; 8]);
#[inline]
#[target_feature(enable = "avx2")]
fn ccso_add_4x4_8bpc(
dst_rows: &mut [u8],
dst_stride: usize,
idx_rows: &[u8],
idx_stride: usize,
block_h: usize,
offset_map: &[i8; 256],
xx: usize,
) {
let zero = _mm_setzero_si128();
let mut off_tmp = AlignedSseS16([0; 8]);
for (yy, idx_row) in idx_rows.chunks(idx_stride).take(block_h).enumerate() {
let dst_row = &mut dst_rows[yy * dst_stride..];
let idx4 = &idx_row[xx..xx + 4].as_chunks::<4>().0[0];
fill_offsets_4_i16(&mut off_tmp.0, idx4, offset_map);
let off = unsafe { _mm_load_si128(off_tmp.0.as_ptr().cast()) };
let dst4 = &mut dst_row[xx..xx + 4].as_chunks_mut::<4>().0[0];
let cur = unsafe { _mm_castps_si128(_mm_load_ss(dst4.as_ptr().cast())) };
let cur = _mm_unpacklo_epi8(cur, zero);
let out = _mm_packus_epi16(_mm_add_epi16(cur, off), zero);
unsafe {
_mm_store_ss(dst4.as_mut_ptr().cast(), _mm_castsi128_ps(out));
}
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn ccso_add_8bpc_avx2(
dst: &mut [u8],
dst_stride: usize,
idx_buf: &[u8],
idx_stride: usize,
offset_idxs: &[u8],
offset_lut: &[i8],
w: usize,
h: usize,
ll_mask: &[[u16; 4]],
) {
let zero = _mm_setzero_si128();
let zero256 = _mm256_setzero_si256();
let offset_map = crate::ccso::ccso_build_offset_map(offset_idxs, offset_lut);
let mut off_tmp = [0i8; 32];
let mut off_tmp16 = [0i8; 16];
let n_blocks = (h + 3) >> 2;
for (by, mask) in (0..h).step_by(4).zip(ll_mask[..n_blocks].iter()) {
let block_h = (h - by).min(4);
let dst_rows = &mut dst[by * dst_stride..];
let idx_row_start = by * idx_stride;
let idx_rows = &idx_buf[idx_row_start..idx_row_start + block_h * idx_stride];
let row_mask = mask[0];
let mut xx = 0usize;
while xx + 32 <= w {
let bx = xx >> 2;
if ((row_mask >> bx) & 0xff) != 0 {
break;
}
for (yy, idx_row) in idx_rows.chunks(idx_stride).take(block_h).enumerate() {
let dst_row = &mut dst_rows[yy * dst_stride..];
let idx32 = &idx_row[xx..xx + 32].as_chunks::<32>().0[0];
fill_offsets_32(&mut off_tmp, idx32, &offset_map);
let off = unsafe { _mm256_loadu_si256(off_tmp.as_ptr().cast()) };
let off0 = _mm256_castsi256_si128(off);
let off1 = _mm256_extracti128_si256::<1>(off);
let off_lo = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(_mm_cvtepi8_epi16(off0)),
_mm_cvtepi8_epi16(off1),
);
let off_hi = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(_mm_cvtepi8_epi16(_mm_srli_si128(off0, 8))),
_mm_cvtepi8_epi16(_mm_srli_si128(off1, 8)),
);
let dst32 = &mut dst_row[xx..xx + 32].as_chunks_mut::<32>().0[0];
let cur = unsafe { _mm256_loadu_si256(dst32.as_ptr().cast()) };
let cur_lo = _mm256_unpacklo_epi8(cur, zero256);
let cur_hi = _mm256_unpackhi_epi8(cur, zero256);
let out = _mm256_packus_epi16(
_mm256_add_epi16(cur_lo, off_lo),
_mm256_add_epi16(cur_hi, off_hi),
);
unsafe { _mm256_storeu_si256(dst32.as_mut_ptr().cast(), out) };
}
xx += 32;
}
while xx + 16 <= w {
let bx = xx >> 2;
if ((row_mask >> bx) & 0x0f) == 0 {
for (yy, idx_row) in idx_rows.chunks(idx_stride).take(block_h).enumerate() {
let dst_row = &mut dst_rows[yy * dst_stride..];
let idx16 = &idx_row[xx..xx + 16].as_chunks::<16>().0[0];
fill_offsets_16(&mut off_tmp16, idx16, &offset_map);
let off = unsafe { _mm_loadu_si128(off_tmp16.as_ptr() as *const __m128i) };
let off_lo = _mm_cvtepi8_epi16(off);
let off_hi = _mm_cvtepi8_epi16(_mm_srli_si128(off, 8));
let dst16 = &mut dst_row[xx..xx + 16].as_chunks_mut::<16>().0[0];
let cur = unsafe { _mm_loadu_si128(dst16.as_ptr() as *const __m128i) };
let cur_lo = _mm_unpacklo_epi8(cur, zero);
let cur_hi = _mm_unpackhi_epi8(cur, zero);
let out = _mm_packus_epi16(
_mm_add_epi16(cur_lo, off_lo),
_mm_add_epi16(cur_hi, off_hi),
);
unsafe { _mm_storeu_si128(dst16.as_mut_ptr() as *mut __m128i, out) };
}
xx += 16;
} else {
for _ in 0..4 {
let bx = xx >> 2;
if row_mask & (1 << bx) == 0 {
ccso_add_4x4_8bpc(
dst_rows,
dst_stride,
idx_rows,
idx_stride,
block_h,
&offset_map,
xx,
);
}
xx += 4;
}
}
}
while xx < w {
let bx = xx >> 2;
if row_mask & (1 << bx) == 0 {
ccso_add_4x4_8bpc(
dst_rows,
dst_stride,
idx_rows,
idx_stride,
block_h,
&offset_map,
xx,
);
}
xx += 4;
}
}
}