Documentation
/*
 * Copyright (c) Radzivon Bartoshyk 6/2026. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 * 1.  Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 *
 * 2.  Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * 3.  Neither the name of the copyright holder nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#[cfg(target_arch = "x86")]
use std::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use std::arch::x86_64::*;

#[inline(always)]
fn ccso_tail_8bpc(
    dst_row: &mut [u8],
    tmp: &[u8],
    row: usize,
    x0: usize,
    x1: usize,
    ss_hor: usize,
    shift: u32,
    luma_offset: isize,
    quant_step: i32,
    edge_clf: u32,
    bo_only: bool,
) {
    for (x, out) in dst_row[x0..x1].iter_mut().enumerate() {
        let x = x + x0;
        let ti = row + (x << ss_hor);
        let c = tmp[ti] as i32;
        let band = (c as u32 >> shift) as u8;
        if bo_only {
            *out = band;
        } else {
            let cls0 = crate::ccso::ccso_score(
                tmp[(ti as isize + luma_offset) as usize] as i32 - c,
                quant_step,
                edge_clf,
            );
            let cls1 = crate::ccso::ccso_score(
                tmp[(ti as isize - luma_offset) as usize] as i32 - c,
                quant_step,
                edge_clf,
            );
            *out = ((cls0 << 5) | (cls1 << 3)) as u8 | band;
        }
    }
}

#[inline]
#[target_feature(enable = "sse4.1")]
fn ccso_classify_epi16(diff: __m128i, q: __m128i, nq: __m128i, edge_clf: u32) -> __m128i {
    let zero = _mm_setzero_si128();
    let one = _mm_set1_epi16(1);
    let two = _mm_set1_epi16(2);
    let gt = if edge_clf == 0 {
        _mm_cmpgt_epi16(diff, q)
    } else {
        zero
    };
    let lt = _mm_cmpgt_epi16(nq, diff);
    let cls = _mm_blendv_epi8(one, two, gt);
    _mm_blendv_epi8(cls, zero, lt)
}

#[inline]
#[target_feature(enable = "sse4.1")]
fn ccso_make_idx_8x16(
    c: __m128i,
    p0: __m128i,
    p1: __m128i,
    shiftv: __m128i,
    q: __m128i,
    nq: __m128i,
    edge_clf: u32,
) -> __m128i {
    let band = _mm_srl_epi16(c, shiftv);
    let cls0 = ccso_classify_epi16(_mm_sub_epi16(p0, c), q, nq, edge_clf);
    let cls1 = ccso_classify_epi16(_mm_sub_epi16(p1, c), q, nq, edge_clf);
    _mm_or_si128(
        band,
        _mm_or_si128(_mm_slli_epi16::<5>(cls0), _mm_slli_epi16::<3>(cls1)),
    )
}

#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) fn ccso_prep_lut_8bpc_sse41(
    dst: &mut [u8],
    dst_stride: usize,
    tmp: &[u8],
    tmp_stride: usize,
    o: usize,
    w: usize,
    h: usize,
    ss_hor: usize,
    ss_ver: usize,
    shift: u32,
    luma_offset: isize,
    quant_step: i32,
    edge_clf: u32,
    bo_only: bool,
) {
    if ss_hor != 0 {
        crate::ccso::ccso_prep_lut_8bpc_scalar(
            dst,
            dst_stride,
            tmp,
            tmp_stride,
            o,
            w,
            h,
            ss_hor,
            ss_ver,
            shift,
            luma_offset,
            quant_step,
            edge_clf,
            bo_only,
        );
        return;
    }

    let zero = _mm_setzero_si128();
    let q = _mm_set1_epi16(quant_step as i16);
    let nq = _mm_set1_epi16((-quant_step) as i16);
    let shiftv = _mm_cvtsi32_si128(shift as i32);

    for (y, dst_row) in dst.chunks_mut(dst_stride).take(h).enumerate() {
        let row = o + (y << ss_ver) * tmp_stride;
        let dst_row = &mut dst_row[..w];
        let n16 = {
            let (dst16, _) = dst_row.as_chunks_mut::<16>();
            if bo_only {
                for (chunk_idx, out16) in dst16.iter_mut().enumerate() {
                    let x = chunk_idx * 16;
                    let c = unsafe { _mm_loadu_si128(tmp.as_ptr().add(row + x) as *const __m128i) };
                    let lo = _mm_srl_epi16(_mm_unpacklo_epi8(c, zero), shiftv);
                    let hi = _mm_srl_epi16(_mm_unpackhi_epi8(c, zero), shiftv);
                    let out = _mm_packus_epi16(lo, hi);
                    unsafe { _mm_storeu_si128(out16.as_mut_ptr() as *mut __m128i, out) };
                }
            } else {
                for (chunk_idx, out16) in dst16.iter_mut().enumerate() {
                    let x = chunk_idx * 16;
                    let c8 =
                        unsafe { _mm_loadu_si128(tmp.as_ptr().add(row + x) as *const __m128i) };
                    let p08 = unsafe {
                        _mm_loadu_si128(
                            tmp.as_ptr()
                                .add(((row + x) as isize + luma_offset) as usize)
                                as *const __m128i,
                        )
                    };
                    let p18 = unsafe {
                        _mm_loadu_si128(
                            tmp.as_ptr()
                                .add(((row + x) as isize - luma_offset) as usize)
                                as *const __m128i,
                        )
                    };
                    let clo = _mm_unpacklo_epi8(c8, zero);
                    let chi = _mm_unpackhi_epi8(c8, zero);
                    let p0lo = _mm_unpacklo_epi8(p08, zero);
                    let p0hi = _mm_unpackhi_epi8(p08, zero);
                    let p1lo = _mm_unpacklo_epi8(p18, zero);
                    let p1hi = _mm_unpackhi_epi8(p18, zero);
                    let lo = ccso_make_idx_8x16(clo, p0lo, p1lo, shiftv, q, nq, edge_clf);
                    let hi = ccso_make_idx_8x16(chi, p0hi, p1hi, shiftv, q, nq, edge_clf);
                    let out = _mm_packus_epi16(lo, hi);
                    unsafe { _mm_storeu_si128(out16.as_mut_ptr() as *mut __m128i, out) };
                }
            }
            dst16.len()
        };
        ccso_tail_8bpc(
            dst_row,
            tmp,
            row,
            n16 * 16,
            w,
            ss_hor,
            shift,
            luma_offset,
            quant_step,
            edge_clf,
            bo_only,
        );
    }
}

#[inline(always)]
fn fill_offsets_16(out: &mut [i8; 16], idx: &[u8; 16], offset_map: &[i8; 256]) {
    for (out, &idx) in out.iter_mut().zip(idx.iter()) {
        *out = offset_map[idx as usize];
    }
}

#[inline(always)]
fn ccso_add_4x4_8bpc(
    dst_rows: &mut [u8],
    dst_stride: usize,
    idx_rows: &[u8],
    idx_stride: usize,
    block_h: usize,
    offset_map: &[i8; 256],
    xx: usize,
) {
    for (yy, idx_row) in idx_rows.chunks(idx_stride).take(block_h).enumerate() {
        let dst_row = &mut dst_rows[yy * dst_stride..];
        let dst4 = &mut dst_row[xx..xx + 4].as_chunks_mut::<4>().0[0];
        let idx4 = &idx_row[xx..xx + 4].as_chunks::<4>().0[0];
        for (dst, &idx) in dst4.iter_mut().zip(idx4.iter()) {
            let off = offset_map[idx as usize];
            let cur = *dst as i32;
            *dst = (cur + off as i32).clamp(0, 255) as u8;
        }
    }
}

#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) fn ccso_add_8bpc_sse41(
    dst: &mut [u8],
    dst_stride: usize,
    idx_buf: &[u8],
    idx_stride: usize,
    offset_idxs: &[u8],
    offset_lut: &[i8],
    w: usize,
    h: usize,
    ll_mask: &[[u16; 4]],
) {
    let zero = _mm_setzero_si128();
    let offset_map = crate::ccso::ccso_build_offset_map(offset_idxs, offset_lut);
    let mut off_tmp = [0i8; 16];
    let n_blocks = (h + 3) >> 2;
    for (by, mask) in (0..h).step_by(4).zip(ll_mask[..n_blocks].iter()) {
        let block_h = (h - by).min(4);
        // `dst` may already start at an x-offset inside the picture row, so
        // chunking it by `dst_stride` would mix row tails with the next row.
        let dst_rows = &mut dst[by * dst_stride..];
        let idx_row_start = by * idx_stride;
        let idx_rows = &idx_buf[idx_row_start..idx_row_start + block_h * idx_stride];
        let row_mask = mask[0];
        let mut xx = 0usize;
        while xx + 16 <= w {
            let bx = xx >> 2;
            if ((row_mask >> bx) & 0x0f) == 0 {
                for (yy, idx_row) in idx_rows.chunks(idx_stride).take(block_h).enumerate() {
                    let dst_row = &mut dst_rows[yy * dst_stride..];
                    let idx16 = &idx_row[xx..xx + 16].as_chunks::<16>().0[0];
                    fill_offsets_16(&mut off_tmp, idx16, &offset_map);
                    let off = unsafe { _mm_loadu_si128(off_tmp.as_ptr() as *const __m128i) };
                    let off_lo = _mm_cvtepi8_epi16(off);
                    let off_hi = _mm_cvtepi8_epi16(_mm_srli_si128(off, 8));
                    let dst16 = &mut dst_row[xx..xx + 16].as_chunks_mut::<16>().0[0];
                    let cur = unsafe { _mm_loadu_si128(dst16.as_ptr() as *const __m128i) };
                    let cur_lo = _mm_unpacklo_epi8(cur, zero);
                    let cur_hi = _mm_unpackhi_epi8(cur, zero);
                    let out = _mm_packus_epi16(
                        _mm_add_epi16(cur_lo, off_lo),
                        _mm_add_epi16(cur_hi, off_hi),
                    );
                    unsafe { _mm_storeu_si128(dst16.as_mut_ptr() as *mut __m128i, out) };
                }
                xx += 16;
            } else {
                for _ in 0..4 {
                    let bx = xx >> 2;
                    if row_mask & (1 << bx) == 0 {
                        ccso_add_4x4_8bpc(
                            dst_rows,
                            dst_stride,
                            idx_rows,
                            idx_stride,
                            block_h,
                            &offset_map,
                            xx,
                        );
                    }
                    xx += 4;
                }
            }
        }
        while xx < w {
            let bx = xx >> 2;
            if row_mask & (1 << bx) == 0 {
                ccso_add_4x4_8bpc(
                    dst_rows,
                    dst_stride,
                    idx_rows,
                    idx_stride,
                    block_h,
                    &offset_map,
                    xx,
                );
            }
            xx += 4;
        }
    }
}