use crate::cdef::{CDEF_HAVE_BOTTOM, CDEF_HAVE_LEFT, CDEF_HAVE_RIGHT, CDEF_HAVE_TOP, constrain};
use crate::tables::CDEF_DIRECTIONS;
use std::sync::OnceLock;
pub(crate) type CdefFilterFn = unsafe fn(
&mut [u8],
usize,
usize,
&[i16],
usize,
usize,
i32,
i32,
i32,
i32,
i32,
usize,
usize,
usize,
);
pub(crate) type CdefFilterHbdFn = unsafe fn(
&mut [u16],
usize,
usize,
&[i16],
usize,
usize,
i32,
i32,
i32,
i32,
i32,
usize,
usize,
usize,
);
pub(crate) type CdefFilterShapeFn =
unsafe fn(&mut [u8], usize, usize, &[i16], usize, usize, i32, i32, i32, i32, i32, usize);
pub(crate) type CdefFilterHbdShapeFn =
unsafe fn(&mut [u16], usize, usize, &[i16], usize, usize, i32, i32, i32, i32, i32, usize);
pub(crate) type CdefDir8Fn = unsafe fn(&[u8], usize, &mut u32) -> i32;
pub(crate) type CdefDirHbdFn = unsafe fn(&[u16], usize, i32, &mut u32) -> i32;
pub(crate) type CdefPadding8Fn = unsafe fn(
&mut [i16],
usize,
&[u8],
usize,
usize,
&[[u8; 2]],
&[u8],
usize,
&[u8],
usize,
usize,
usize,
usize,
u8,
);
pub(crate) type CdefPaddingHbdFn = unsafe fn(
&mut [i16],
usize,
&[u16],
usize,
usize,
&[[u16; 2]],
&[u16],
usize,
&[u16],
usize,
usize,
usize,
usize,
u8,
);
#[inline(always)]
fn cdef_fill_i16(tmp: &mut [i16], stride: usize, w: usize, h: usize) {
for row in tmp.chunks_mut(stride).take(h) {
row[..w].fill(i16::MIN);
}
}
#[inline(always)]
fn copy_u8_to_i16_scalar(dst: &mut [i16], src: &[u8]) {
debug_assert_eq!(dst.len(), src.len());
for (d, &s) in dst.iter_mut().zip(src) {
*d = s as i16;
}
}
#[inline(always)]
fn copy_u16_to_i16_scalar(dst: &mut [i16], src: &[u16]) {
debug_assert_eq!(dst.len(), src.len());
for (d, &s) in dst.iter_mut().zip(src) {
*d = s as i16;
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
pub(crate) fn cdef_padding_8bpc_scalar(
tmp: &mut [i16],
tmp_stride: usize,
src: &[u8],
src_stride: usize,
src_off: usize,
left: &[[u8; 2]],
top: &[u8],
top_off: usize,
bottom: &[u8],
bottom_off: usize,
bottom_stride: usize,
w: usize,
h: usize,
edges: u8,
) {
let o = 2 * tmp_stride + 2;
let mut x_start: i32 = -2;
let mut x_end: i32 = w as i32 + 2;
let mut y_start: i32 = -2;
let mut y_end: i32 = h as i32 + 2;
if edges & CDEF_HAVE_TOP == 0 {
let base = o.wrapping_sub(2).wrapping_sub(2 * tmp_stride);
cdef_fill_i16(&mut tmp[base..], tmp_stride, w + 4, 2);
y_start = 0;
}
if edges & CDEF_HAVE_BOTTOM == 0 {
let base = o + h * tmp_stride - 2;
cdef_fill_i16(&mut tmp[base..], tmp_stride, w + 4, 2);
y_end -= 2;
}
if edges & CDEF_HAVE_LEFT == 0 {
let base = (o as i32 + y_start * tmp_stride as i32 - 2) as usize;
cdef_fill_i16(&mut tmp[base..], tmp_stride, 2, (y_end - y_start) as usize);
x_start = 0;
}
if edges & CDEF_HAVE_RIGHT == 0 {
let base = (o as i32 + y_start * tmp_stride as i32 + w as i32) as usize;
cdef_fill_i16(&mut tmp[base..], tmp_stride, 2, (y_end - y_start) as usize);
x_end -= 2;
}
let copy_w = (x_end - x_start) as usize;
let mut toff = top_off;
for y in y_start..0 {
let ti = (o as i32 + x_start + y * tmp_stride as i32) as usize;
let si = (toff as i32 + x_start) as usize;
copy_u8_to_i16_scalar(&mut tmp[ti..ti + copy_w], &top[si..si + copy_w]);
toff += src_stride;
}
for y in 0..h as i32 {
let ti = (o as i32 + y * tmp_stride as i32 - 2) as usize;
for x in x_start..0 {
tmp[ti + (x + 2) as usize] = left[y as usize][(x + 2) as usize] as i16;
}
}
let copy_w = x_end as usize;
let mut soff = src_off;
for y in 0..h as i32 {
let ti = (o as i32 + y * tmp_stride as i32) as usize;
copy_u8_to_i16_scalar(&mut tmp[ti..ti + copy_w], &src[soff..soff + copy_w]);
soff += src_stride;
}
let copy_w = (x_end - x_start) as usize;
let mut boff = bottom_off;
for y in h as i32..y_end {
let ti = (o as i32 + x_start + y * tmp_stride as i32) as usize;
let si = (boff as i32 + x_start) as usize;
copy_u8_to_i16_scalar(&mut tmp[ti..ti + copy_w], &bottom[si..si + copy_w]);
boff += bottom_stride;
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
pub(crate) fn cdef_padding_hbd_scalar(
tmp: &mut [i16],
tmp_stride: usize,
src: &[u16],
src_stride: usize,
src_off: usize,
left: &[[u16; 2]],
top: &[u16],
top_off: usize,
bottom: &[u16],
bottom_off: usize,
bottom_stride: usize,
w: usize,
h: usize,
edges: u8,
) {
let o = 2 * tmp_stride + 2;
let mut x_start: i32 = -2;
let mut x_end: i32 = w as i32 + 2;
let mut y_start: i32 = -2;
let mut y_end: i32 = h as i32 + 2;
if edges & CDEF_HAVE_TOP == 0 {
let base = o.wrapping_sub(2).wrapping_sub(2 * tmp_stride);
cdef_fill_i16(&mut tmp[base..], tmp_stride, w + 4, 2);
y_start = 0;
}
if edges & CDEF_HAVE_BOTTOM == 0 {
let base = o + h * tmp_stride - 2;
cdef_fill_i16(&mut tmp[base..], tmp_stride, w + 4, 2);
y_end -= 2;
}
if edges & CDEF_HAVE_LEFT == 0 {
let base = (o as i32 + y_start * tmp_stride as i32 - 2) as usize;
cdef_fill_i16(&mut tmp[base..], tmp_stride, 2, (y_end - y_start) as usize);
x_start = 0;
}
if edges & CDEF_HAVE_RIGHT == 0 {
let base = (o as i32 + y_start * tmp_stride as i32 + w as i32) as usize;
cdef_fill_i16(&mut tmp[base..], tmp_stride, 2, (y_end - y_start) as usize);
x_end -= 2;
}
let copy_w = (x_end - x_start) as usize;
let mut toff = top_off;
for y in y_start..0 {
let ti = (o as i32 + x_start + y * tmp_stride as i32) as usize;
let si = (toff as i32 + x_start) as usize;
copy_u16_to_i16_scalar(&mut tmp[ti..ti + copy_w], &top[si..si + copy_w]);
toff += src_stride;
}
for y in 0..h as i32 {
let ti = (o as i32 + y * tmp_stride as i32 - 2) as usize;
for x in x_start..0 {
tmp[ti + (x + 2) as usize] = left[y as usize][(x + 2) as usize] as i16;
}
}
let copy_w = x_end as usize;
let mut soff = src_off;
for y in 0..h as i32 {
let ti = (o as i32 + y * tmp_stride as i32) as usize;
copy_u16_to_i16_scalar(&mut tmp[ti..ti + copy_w], &src[soff..soff + copy_w]);
soff += src_stride;
}
let copy_w = (x_end - x_start) as usize;
let mut boff = bottom_off;
for y in h as i32..y_end {
let ti = (o as i32 + x_start + y * tmp_stride as i32) as usize;
let si = (boff as i32 + x_start) as usize;
copy_u16_to_i16_scalar(&mut tmp[ti..ti + copy_w], &bottom[si..si + copy_w]);
boff += bottom_stride;
}
}
#[inline(always)]
fn cdef_min(a: i32, b: i32) -> i32 {
if (a as u32) < (b as u32) { a } else { b }
}
#[inline(always)]
pub(crate) fn cdef_find_dir_from_i16_rows(rows: &[[i16; 8]; 8], var: &mut u32) -> i32 {
let mut partial_sum_hv = [[0i32; 8]; 2];
let mut partial_sum_diag = [[0i32; 15]; 2];
let mut partial_sum_alt = [[0i32; 11]; 4];
for y in 0..8usize {
let mut row_sum = 0i32;
for x in 0..8usize {
let px = rows[y][x] as i32;
row_sum += px;
partial_sum_diag[0][y + x] += px;
partial_sum_alt[0][y + (x >> 1)] += px;
partial_sum_alt[1][3 + y - (x >> 1)] += px;
partial_sum_diag[1][7 + y - x] += px;
partial_sum_alt[2][3 - (y >> 1) + x] += px;
partial_sum_hv[1][x] += px;
partial_sum_alt[3][(y >> 1) + x] += px;
}
partial_sum_hv[0][y] = row_sum;
}
cdef_find_dir_from_partials(&partial_sum_hv, &partial_sum_diag, &partial_sum_alt, var)
}
#[inline(always)]
pub(crate) fn cdef_find_dir_from_partials(
partial_sum_hv: &[[i32; 8]; 2],
partial_sum_diag: &[[i32; 15]; 2],
partial_sum_alt: &[[i32; 11]; 4],
var: &mut u32,
) -> i32 {
let mut cost = [0u32; 8];
for n in 0..8usize {
cost[2] += (partial_sum_hv[0][n] * partial_sum_hv[0][n]) as u32;
cost[6] += (partial_sum_hv[1][n] * partial_sum_hv[1][n]) as u32;
}
cost[2] *= 105;
cost[6] *= 105;
const DIV_TABLE: [u32; 7] = [840, 420, 280, 210, 168, 140, 120];
for n in 0..7usize {
let d = DIV_TABLE[n];
cost[0] += ((partial_sum_diag[0][n] * partial_sum_diag[0][n]
+ partial_sum_diag[0][14 - n] * partial_sum_diag[0][14 - n])
as u32)
* d;
cost[4] += ((partial_sum_diag[1][n] * partial_sum_diag[1][n]
+ partial_sum_diag[1][14 - n] * partial_sum_diag[1][14 - n])
as u32)
* d;
}
cost[0] += (partial_sum_diag[0][7] * partial_sum_diag[0][7]) as u32 * 105;
cost[4] += (partial_sum_diag[1][7] * partial_sum_diag[1][7]) as u32 * 105;
for n in 0..4usize {
let ci = n * 2 + 1;
for m in 0..5usize {
cost[ci] += (partial_sum_alt[n][3 + m] * partial_sum_alt[n][3 + m]) as u32;
}
cost[ci] *= 105;
for m in 0..3usize {
let d = DIV_TABLE[2 * m + 1];
cost[ci] += ((partial_sum_alt[n][m] * partial_sum_alt[n][m]
+ partial_sum_alt[n][10 - m] * partial_sum_alt[n][10 - m])
as u32)
* d;
}
}
let mut best_dir = 0i32;
let mut best_cost = cost[0];
for (n, &c) in cost.iter().enumerate().skip(1) {
if c > best_cost {
best_cost = c;
best_dir = n as i32;
}
}
*var = (best_cost - cost[(best_dir ^ 4) as usize]) >> 10;
best_dir
}
#[inline]
pub(crate) fn cdef_find_dir_8bpc_scalar(img: &[u8], stride: usize, var: &mut u32) -> i32 {
let mut rows = [[0i16; 8]; 8];
for y in 0..8usize {
for x in 0..8usize {
rows[y][x] = img[y * stride + x] as i16 - 128;
}
}
cdef_find_dir_from_i16_rows(&rows, var)
}
#[inline]
pub(crate) fn cdef_find_dir_hbd_scalar(
img: &[u16],
stride: usize,
bitdepth_min_8: i32,
var: &mut u32,
) -> i32 {
let shift = bitdepth_min_8 as u32;
let mut rows = [[0i16; 8]; 8];
for y in 0..8usize {
for x in 0..8usize {
rows[y][x] = (img[y * stride + x] >> shift) as i16 - 128;
}
}
cdef_find_dir_from_i16_rows(&rows, var)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn cdef_filter_block_hbd_scalar(
dst: &mut [u16],
dst_stride: usize,
dst_off: usize,
tmp: &[i16],
tmp_stride: usize,
o: usize,
pri_strength: i32,
sec_strength: i32,
pri_shift: i32,
sec_shift: i32,
pri_tap: i32,
dir: usize,
w: usize,
h: usize,
) {
let mut dp = dst_off;
let mut tp = o;
if pri_strength != 0 && sec_strength != 0 {
for _y in 0..h {
for x in 0..w {
let px = tmp[tp + x] as i32;
let mut sum = 0i32;
let mut min_v = px;
let mut max_v = px;
let mut ptap = pri_tap;
for k in 0..2 {
let off1 = CDEF_DIRECTIONS[dir + 2][k] as isize;
let p0 = tmp[((tp + x) as isize + off1) as usize] as i32;
let p1 = tmp[((tp + x) as isize - off1) as usize] as i32;
sum += ptap * constrain(p0 - px, pri_strength, pri_shift);
sum += ptap * constrain(p1 - px, pri_strength, pri_shift);
ptap = (ptap & 3) | 2;
min_v = cdef_min(cdef_min(min_v, p0), p1);
max_v = p0.max(max_v).max(p1);
let off2 = CDEF_DIRECTIONS[dir + 4][k] as isize;
let off3 = CDEF_DIRECTIONS[dir][k] as isize;
let s0 = tmp[((tp + x) as isize + off2) as usize] as i32;
let s1 = tmp[((tp + x) as isize - off2) as usize] as i32;
let s2 = tmp[((tp + x) as isize + off3) as usize] as i32;
let s3 = tmp[((tp + x) as isize - off3) as usize] as i32;
let st = 2 - k as i32;
sum += st * constrain(s0 - px, sec_strength, sec_shift);
sum += st * constrain(s1 - px, sec_strength, sec_shift);
sum += st * constrain(s2 - px, sec_strength, sec_shift);
sum += st * constrain(s3 - px, sec_strength, sec_shift);
min_v = cdef_min(cdef_min(cdef_min(cdef_min(min_v, s0), s1), s2), s3);
max_v = s0.max(max_v).max(s1).max(s2).max(s3);
}
let v = px + ((sum - (sum < 0) as i32 + 8) >> 4);
dst[dp + x] = v.clamp(min_v, max_v) as u16;
}
dp += dst_stride;
tp += tmp_stride;
}
} else if pri_strength != 0 {
for _y in 0..h {
for x in 0..w {
let px = tmp[tp + x] as i32;
let mut sum = 0i32;
let mut ptap = pri_tap;
for k in 0..2 {
let off = CDEF_DIRECTIONS[dir + 2][k] as isize;
let p0 = tmp[((tp + x) as isize + off) as usize] as i32;
let p1 = tmp[((tp + x) as isize - off) as usize] as i32;
sum += ptap * constrain(p0 - px, pri_strength, pri_shift);
sum += ptap * constrain(p1 - px, pri_strength, pri_shift);
ptap = (ptap & 3) | 2;
}
dst[dp + x] = (px + ((sum - (sum < 0) as i32 + 8) >> 4)) as u16;
}
dp += dst_stride;
tp += tmp_stride;
}
} else {
for _y in 0..h {
for x in 0..w {
let px = tmp[tp + x] as i32;
let mut sum = 0i32;
for k in 0..2 {
let off1 = CDEF_DIRECTIONS[dir + 4][k] as isize;
let off2 = CDEF_DIRECTIONS[dir][k] as isize;
let s0 = tmp[((tp + x) as isize + off1) as usize] as i32;
let s1 = tmp[((tp + x) as isize - off1) as usize] as i32;
let s2 = tmp[((tp + x) as isize + off2) as usize] as i32;
let s3 = tmp[((tp + x) as isize - off2) as usize] as i32;
let st = 2 - k as i32;
sum += st * constrain(s0 - px, sec_strength, sec_shift);
sum += st * constrain(s1 - px, sec_strength, sec_shift);
sum += st * constrain(s2 - px, sec_strength, sec_shift);
sum += st * constrain(s3 - px, sec_strength, sec_shift);
}
dst[dp + x] = (px + ((sum - (sum < 0) as i32 + 8) >> 4)) as u16;
}
dp += dst_stride;
tp += tmp_stride;
}
}
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn cdef_filter_block_8bpc_scalar(
dst: &mut [u8],
dst_stride: usize,
dst_off: usize,
tmp: &[i16],
tmp_stride: usize,
o: usize,
pri_strength: i32,
sec_strength: i32,
pri_shift: i32,
sec_shift: i32,
pri_tap: i32,
dir: usize,
w: usize,
h: usize,
) {
let mut dp = dst_off;
let mut tp = o;
if pri_strength != 0 && sec_strength != 0 {
for _y in 0..h {
for x in 0..w {
let px = tmp[tp + x] as i32;
let mut sum = 0i32;
let mut min_v = px;
let mut max_v = px;
let mut ptap = pri_tap;
for k in 0..2 {
let off1 = CDEF_DIRECTIONS[dir + 2][k] as isize;
let p0 = tmp[((tp + x) as isize + off1) as usize] as i32;
let p1 = tmp[((tp + x) as isize - off1) as usize] as i32;
sum += ptap * constrain(p0 - px, pri_strength, pri_shift);
sum += ptap * constrain(p1 - px, pri_strength, pri_shift);
ptap = (ptap & 3) | 2;
min_v = cdef_min(cdef_min(min_v, p0), p1);
max_v = p0.max(max_v).max(p1);
let off2 = CDEF_DIRECTIONS[dir + 4][k] as isize;
let off3 = CDEF_DIRECTIONS[dir][k] as isize;
let s0 = tmp[((tp + x) as isize + off2) as usize] as i32;
let s1 = tmp[((tp + x) as isize - off2) as usize] as i32;
let s2 = tmp[((tp + x) as isize + off3) as usize] as i32;
let s3 = tmp[((tp + x) as isize - off3) as usize] as i32;
let st = 2 - k as i32;
sum += st * constrain(s0 - px, sec_strength, sec_shift);
sum += st * constrain(s1 - px, sec_strength, sec_shift);
sum += st * constrain(s2 - px, sec_strength, sec_shift);
sum += st * constrain(s3 - px, sec_strength, sec_shift);
min_v = cdef_min(cdef_min(cdef_min(cdef_min(min_v, s0), s1), s2), s3);
max_v = s0.max(max_v).max(s1).max(s2).max(s3);
}
let v = px + ((sum - (sum < 0) as i32 + 8) >> 4);
dst[dp + x] = v.clamp(min_v, max_v) as u8;
}
dp += dst_stride;
tp += tmp_stride;
}
} else if pri_strength != 0 {
for _y in 0..h {
for x in 0..w {
let px = tmp[tp + x] as i32;
let mut sum = 0i32;
let mut ptap = pri_tap;
for k in 0..2 {
let off = CDEF_DIRECTIONS[dir + 2][k] as isize;
let p0 = tmp[((tp + x) as isize + off) as usize] as i32;
let p1 = tmp[((tp + x) as isize - off) as usize] as i32;
sum += ptap * constrain(p0 - px, pri_strength, pri_shift);
sum += ptap * constrain(p1 - px, pri_strength, pri_shift);
ptap = (ptap & 3) | 2;
}
dst[dp + x] = (px + ((sum - (sum < 0) as i32 + 8) >> 4)) as u8;
}
dp += dst_stride;
tp += tmp_stride;
}
} else {
for _y in 0..h {
for x in 0..w {
let px = tmp[tp + x] as i32;
let mut sum = 0i32;
for k in 0..2 {
let off1 = CDEF_DIRECTIONS[dir + 4][k] as isize;
let off2 = CDEF_DIRECTIONS[dir][k] as isize;
let s0 = tmp[((tp + x) as isize + off1) as usize] as i32;
let s1 = tmp[((tp + x) as isize - off1) as usize] as i32;
let s2 = tmp[((tp + x) as isize + off2) as usize] as i32;
let s3 = tmp[((tp + x) as isize - off2) as usize] as i32;
let st = 2 - k as i32;
sum += st * constrain(s0 - px, sec_strength, sec_shift);
sum += st * constrain(s1 - px, sec_strength, sec_shift);
sum += st * constrain(s2 - px, sec_strength, sec_shift);
sum += st * constrain(s3 - px, sec_strength, sec_shift);
}
dst[dp + x] = (px + ((sum - (sum < 0) as i32 + 8) >> 4)) as u8;
}
dp += dst_stride;
tp += tmp_stride;
}
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
pub(crate) unsafe fn cdef_filter_block_8x8_8bpc_scalar(
dst: &mut [u8],
dst_stride: usize,
dst_off: usize,
tmp: &[i16],
tmp_stride: usize,
o: usize,
pri_strength: i32,
sec_strength: i32,
pri_shift: i32,
sec_shift: i32,
pri_tap: i32,
dir: usize,
) {
cdef_filter_block_8bpc_scalar(
dst,
dst_stride,
dst_off,
tmp,
tmp_stride,
o,
pri_strength,
sec_strength,
pri_shift,
sec_shift,
pri_tap,
dir,
8,
8,
);
}
#[allow(clippy::too_many_arguments)]
#[inline]
pub(crate) fn cdef_filter_block_8x4_8bpc_scalar(
dst: &mut [u8],
dst_stride: usize,
dst_off: usize,
tmp: &[i16],
tmp_stride: usize,
o: usize,
pri_strength: i32,
sec_strength: i32,
pri_shift: i32,
sec_shift: i32,
pri_tap: i32,
dir: usize,
) {
cdef_filter_block_8bpc_scalar(
dst,
dst_stride,
dst_off,
tmp,
tmp_stride,
o,
pri_strength,
sec_strength,
pri_shift,
sec_shift,
pri_tap,
dir,
8,
4,
);
}
#[allow(clippy::too_many_arguments)]
#[inline]
pub(crate) fn cdef_filter_block_4x8_8bpc_scalar(
dst: &mut [u8],
dst_stride: usize,
dst_off: usize,
tmp: &[i16],
tmp_stride: usize,
o: usize,
pri_strength: i32,
sec_strength: i32,
pri_shift: i32,
sec_shift: i32,
pri_tap: i32,
dir: usize,
) {
cdef_filter_block_8bpc_scalar(
dst,
dst_stride,
dst_off,
tmp,
tmp_stride,
o,
pri_strength,
sec_strength,
pri_shift,
sec_shift,
pri_tap,
dir,
4,
8,
);
}
#[allow(clippy::too_many_arguments)]
#[inline]
pub(crate) fn cdef_filter_block_4x4_8bpc_scalar(
dst: &mut [u8],
dst_stride: usize,
dst_off: usize,
tmp: &[i16],
tmp_stride: usize,
o: usize,
pri_strength: i32,
sec_strength: i32,
pri_shift: i32,
sec_shift: i32,
pri_tap: i32,
dir: usize,
) {
cdef_filter_block_8bpc_scalar(
dst,
dst_stride,
dst_off,
tmp,
tmp_stride,
o,
pri_strength,
sec_strength,
pri_shift,
sec_shift,
pri_tap,
dir,
4,
4,
);
}
#[allow(clippy::too_many_arguments)]
#[inline]
pub(crate) unsafe fn cdef_filter_block_8x8_hbd_scalar(
dst: &mut [u16],
dst_stride: usize,
dst_off: usize,
tmp: &[i16],
tmp_stride: usize,
o: usize,
pri_strength: i32,
sec_strength: i32,
pri_shift: i32,
sec_shift: i32,
pri_tap: i32,
dir: usize,
) {
cdef_filter_block_hbd_scalar(
dst,
dst_stride,
dst_off,
tmp,
tmp_stride,
o,
pri_strength,
sec_strength,
pri_shift,
sec_shift,
pri_tap,
dir,
8,
8,
);
}
#[allow(clippy::too_many_arguments)]
#[inline]
pub(crate) fn cdef_filter_block_8x4_hbd_scalar(
dst: &mut [u16],
dst_stride: usize,
dst_off: usize,
tmp: &[i16],
tmp_stride: usize,
o: usize,
pri_strength: i32,
sec_strength: i32,
pri_shift: i32,
sec_shift: i32,
pri_tap: i32,
dir: usize,
) {
cdef_filter_block_hbd_scalar(
dst,
dst_stride,
dst_off,
tmp,
tmp_stride,
o,
pri_strength,
sec_strength,
pri_shift,
sec_shift,
pri_tap,
dir,
8,
4,
);
}
#[allow(clippy::too_many_arguments)]
#[inline]
pub(crate) fn cdef_filter_block_4x8_hbd_scalar(
dst: &mut [u16],
dst_stride: usize,
dst_off: usize,
tmp: &[i16],
tmp_stride: usize,
o: usize,
pri_strength: i32,
sec_strength: i32,
pri_shift: i32,
sec_shift: i32,
pri_tap: i32,
dir: usize,
) {
cdef_filter_block_hbd_scalar(
dst,
dst_stride,
dst_off,
tmp,
tmp_stride,
o,
pri_strength,
sec_strength,
pri_shift,
sec_shift,
pri_tap,
dir,
4,
8,
);
}
#[allow(clippy::too_many_arguments)]
#[inline]
pub(crate) unsafe fn cdef_filter_block_4x4_hbd_scalar(
dst: &mut [u16],
dst_stride: usize,
dst_off: usize,
tmp: &[i16],
tmp_stride: usize,
o: usize,
pri_strength: i32,
sec_strength: i32,
pri_shift: i32,
sec_shift: i32,
pri_tap: i32,
dir: usize,
) {
cdef_filter_block_hbd_scalar(
dst,
dst_stride,
dst_off,
tmp,
tmp_stride,
o,
pri_strength,
sec_strength,
pri_shift,
sec_shift,
pri_tap,
dir,
4,
4,
);
}
static CDEF_DIR_8BPC: OnceLock<CdefDir8Fn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_cdef_dir_8bpc() -> CdefDir8Fn {
*CDEF_DIR_8BPC.get_or_init(|| {
let mut _f = cdef_find_dir_8bpc_scalar as CdefDir8Fn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::cdef_find_dir_8bpc_neon as CdefDir8Fn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::cdef_find_dir_8bpc_sse41 as CdefDir8Fn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::cdef_find_dir_8bpc_avx2 as CdefDir8Fn;
}
}
_f
})
}
static CDEF_DIR_HBD: OnceLock<CdefDirHbdFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_cdef_dir_hbd() -> CdefDirHbdFn {
*CDEF_DIR_HBD.get_or_init(|| {
let mut _f = cdef_find_dir_hbd_scalar as CdefDirHbdFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::cdef_find_dir_hbd_neon as CdefDirHbdFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::cdef_find_dir_hbd_sse41 as CdefDirHbdFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::cdef_find_dir_hbd_avx2 as CdefDirHbdFn;
}
}
_f
})
}
static CDEF_PADDING_8BPC: OnceLock<CdefPadding8Fn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_cdef_padding_8bpc() -> CdefPadding8Fn {
*CDEF_PADDING_8BPC.get_or_init(|| {
let mut _f = cdef_padding_8bpc_scalar as CdefPadding8Fn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::cdef_padding_8bpc_neon as CdefPadding8Fn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::cdef_padding_8bpc_sse41 as CdefPadding8Fn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::cdef_padding_8bpc_avx2 as CdefPadding8Fn;
}
}
_f
})
}
static CDEF_PADDING_HBD: OnceLock<CdefPaddingHbdFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_cdef_padding_hbd() -> CdefPaddingHbdFn {
*CDEF_PADDING_HBD.get_or_init(|| {
let mut _f = cdef_padding_hbd_scalar as CdefPaddingHbdFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::cdef_padding_hbd_neon as CdefPaddingHbdFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::cdef_padding_hbd_sse41 as CdefPaddingHbdFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::cdef_padding_hbd_avx2 as CdefPaddingHbdFn;
}
}
_f
})
}
static CDEF_FILTER: OnceLock<CdefFilterFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_cdef_filter() -> CdefFilterFn {
*CDEF_FILTER.get_or_init(|| {
let mut _f = cdef_filter_block_8bpc_scalar as CdefFilterFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::cdef_filter_block_8bpc_neon as CdefFilterFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::cdef_filter_block_8bpc_sse41 as CdefFilterFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::cdef_filter_block_8bpc_avx2 as CdefFilterFn;
}
}
_f
})
}
static CDEF_FILTER_HBD: OnceLock<CdefFilterHbdFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_cdef_filter_hbd() -> CdefFilterHbdFn {
*CDEF_FILTER_HBD.get_or_init(|| {
let mut _f = cdef_filter_block_hbd_scalar as CdefFilterHbdFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::cdef_filter_block_hbd_neon as CdefFilterHbdFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::cdef_filter_block_hbd_sse41 as CdefFilterHbdFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::cdef_filter_block_hbd_avx2 as CdefFilterHbdFn;
}
}
_f
})
}
static CDEF_FILTER_SHAPES: OnceLock<[CdefFilterShapeFn; 4]> = OnceLock::new();
#[inline]
pub(crate) fn resolve_cdef_filter_shapes() -> &'static [CdefFilterShapeFn; 4] {
CDEF_FILTER_SHAPES.get_or_init(|| {
let mut _f = [
cdef_filter_block_8x8_8bpc_scalar as CdefFilterShapeFn,
cdef_filter_block_4x8_8bpc_scalar as CdefFilterShapeFn,
cdef_filter_block_4x4_8bpc_scalar as CdefFilterShapeFn,
cdef_filter_block_8x4_8bpc_scalar as CdefFilterShapeFn,
];
#[cfg(target_arch = "aarch64")]
{
_f = [
crate::neon::cdef_filter_block_8x8_8bpc_neon as CdefFilterShapeFn,
crate::neon::cdef_filter_block_4x8_8bpc_neon as CdefFilterShapeFn,
crate::neon::cdef_filter_block_4x4_8bpc_neon as CdefFilterShapeFn,
crate::neon::cdef_filter_block_8x4_8bpc_neon as CdefFilterShapeFn,
];
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = [
crate::sse::cdef_filter_block_8x8_8bpc_sse41 as CdefFilterShapeFn,
crate::sse::cdef_filter_block_4x8_8bpc_sse41 as CdefFilterShapeFn,
crate::sse::cdef_filter_block_4x4_8bpc_sse41 as CdefFilterShapeFn,
cdef_filter_block_8x4_8bpc_scalar as CdefFilterShapeFn,
];
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = [
crate::avx::cdef_filter_block_8x8_8bpc_avx2 as CdefFilterShapeFn,
crate::avx::cdef_filter_block_4x8_8bpc_avx2 as CdefFilterShapeFn,
crate::avx::cdef_filter_block_4x4_8bpc_avx2 as CdefFilterShapeFn,
crate::avx::cdef_filter_block_8x4_8bpc_avx2 as CdefFilterShapeFn,
];
}
}
_f
})
}
static CDEF_FILTER_HBD_SHAPES: OnceLock<[CdefFilterHbdShapeFn; 4]> = OnceLock::new();
#[inline]
pub(crate) fn resolve_cdef_filter_hbd_shapes() -> &'static [CdefFilterHbdShapeFn; 4] {
CDEF_FILTER_HBD_SHAPES.get_or_init(|| {
let mut _f = [
cdef_filter_block_8x8_hbd_scalar as CdefFilterHbdShapeFn,
cdef_filter_block_4x8_hbd_scalar as CdefFilterHbdShapeFn,
cdef_filter_block_4x4_hbd_scalar as CdefFilterHbdShapeFn,
cdef_filter_block_8x4_hbd_scalar as CdefFilterHbdShapeFn,
];
#[cfg(target_arch = "aarch64")]
{
_f = [
crate::neon::cdef_filter_block_8x8_hbd_neon as CdefFilterHbdShapeFn,
crate::neon::cdef_filter_block_4x8_hbd_neon as CdefFilterHbdShapeFn,
crate::neon::cdef_filter_block_4x4_hbd_neon as CdefFilterHbdShapeFn,
crate::neon::cdef_filter_block_8x4_hbd_neon as CdefFilterHbdShapeFn,
];
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = [
crate::sse::cdef_filter_block_8x8_hbd_sse41 as CdefFilterHbdShapeFn,
crate::sse::cdef_filter_block_4x8_hbd_sse41 as CdefFilterHbdShapeFn,
crate::sse::cdef_filter_block_4x4_hbd_sse41 as CdefFilterHbdShapeFn,
cdef_filter_block_8x4_hbd_scalar as CdefFilterHbdShapeFn,
];
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = [
crate::avx::cdef_filter_block_8x8_hbd_avx2 as CdefFilterHbdShapeFn,
crate::avx::cdef_filter_block_4x8_hbd_avx2 as CdefFilterHbdShapeFn,
crate::avx::cdef_filter_block_4x4_hbd_avx2 as CdefFilterHbdShapeFn,
crate::avx::cdef_filter_block_8x4_hbd_avx2 as CdefFilterHbdShapeFn,
];
}
}
_f
})
}
#[cfg(test)]
mod simd_matches_scalar_tests {
use super::*;
use crate::intops::{imax, ulog2};
struct Rng(u64);
impl Rng {
fn r(&mut self, n: u32) -> i32 {
self.0 ^= self.0 << 13;
self.0 ^= self.0 >> 7;
self.0 ^= self.0 << 17;
((self.0 >> 16) as u32 % n) as i32
}
}
fn strengths(rng: &mut Rng, bdm8: i32) -> (i32, i32, i32, i32, i32, usize) {
let pri = if rng.r(4) == 0 {
0
} else {
(1 + rng.r(15)) << bdm8
};
let sec = [0, 1, 2, 4][rng.r(4) as usize] << bdm8;
let pri = if pri == 0 && sec == 0 { 1 << bdm8 } else { pri };
let damping = 3 + rng.r(4) + bdm8;
let pri_tap = if pri != 0 { 4 - ((pri >> bdm8) & 1) } else { 0 };
let pri_shift = if pri != 0 {
imax(0, damping - ulog2(pri as u32))
} else {
0
};
let sec_shift = if sec != 0 {
damping - ulog2(sec as u32)
} else {
0
};
(pri, sec, pri_shift, sec_shift, pri_tap, rng.r(8) as usize)
}
fn tmp_block(rng: &mut Rng, ts: usize, w: usize, h: usize, max: u32) -> Vec<i16> {
(0..ts * 12)
.map(|i| {
let inner = (2..2 + w).contains(&(i % ts)) && (2..2 + h).contains(&(i / ts));
if !inner && rng.r(10) == 0 {
i16::MIN
} else if rng.r(4) == 0 {
0
} else {
rng.r(max) as i16
}
})
.collect()
}
const SHAPES: [(usize, usize); 4] = [(8, 8), (4, 8), (4, 4), (8, 4)];
#[test]
fn find_dir_matches_scalar() {
let mut rng = Rng(0x1234_5678_9abc);
for _ in 0..2000 {
let img: Vec<u8> = (0..64).map(|_| rng.r(256) as u8).collect();
let (mut v0, mut v1) = (0, 0);
let d0 = cdef_find_dir_8bpc_scalar(&img, 8, &mut v0);
let d1 = unsafe { resolve_cdef_dir_8bpc()(&img, 8, &mut v1) };
assert_eq!((d0, v0), (d1, v1));
for bdm8 in [2, 4] {
let img: Vec<u16> = (0..64).map(|_| rng.r(1 << (8 + bdm8)) as u16).collect();
let (mut v0, mut v1) = (0, 0);
let d0 = cdef_find_dir_hbd_scalar(&img, 8, bdm8, &mut v0);
let d1 = unsafe { resolve_cdef_dir_hbd()(&img, 8, bdm8, &mut v1) };
assert_eq!((d0, v0), (d1, v1), "bdm8={bdm8}");
}
}
}
#[test]
fn filter_8bpc_matches_scalar() {
let mut rng = Rng(0xdead_beef_1234);
let ts = 16;
let o = 2 * ts + 2;
for _ in 0..2000 {
let si = rng.r(4) as usize;
let (w, h) = SHAPES[si];
let tmp = tmp_block(&mut rng, ts, w, h, 256);
let (pri, sec, ps, ss, pt, dir) = strengths(&mut rng, 0);
let mut d0 = vec![7u8; 16 * 8];
let mut d1 = d0.clone();
let mut d2 = d0.clone();
cdef_filter_block_8bpc_scalar(
&mut d0, 16, 0, &tmp, ts, o, pri, sec, ps, ss, pt, dir, w, h,
);
unsafe {
resolve_cdef_filter_shapes()[si](
&mut d1, 16, 0, &tmp, ts, o, pri, sec, ps, ss, pt, dir,
);
resolve_cdef_filter()(&mut d2, 16, 0, &tmp, ts, o, pri, sec, ps, ss, pt, dir, w, h);
}
assert_eq!(d0, d1, "shape {w}x{h} pri={pri} sec={sec} dir={dir}");
assert_eq!(d0, d2, "generic {w}x{h} pri={pri} sec={sec} dir={dir}");
}
}
#[test]
fn filter_hbd_matches_scalar() {
let mut rng = Rng(0x00fe_edf0_0d77);
let ts = 16;
let o = 2 * ts + 2;
for _ in 0..2000 {
let bdm8 = [2, 4][rng.r(2) as usize];
let si = rng.r(4) as usize;
let (w, h) = SHAPES[si];
let tmp = tmp_block(&mut rng, ts, w, h, 1 << (8 + bdm8));
let (pri, sec, ps, ss, pt, dir) = strengths(&mut rng, bdm8);
let mut d0 = vec![7u16; 16 * 8];
let mut d1 = d0.clone();
let mut d2 = d0.clone();
cdef_filter_block_hbd_scalar(
&mut d0, 16, 0, &tmp, ts, o, pri, sec, ps, ss, pt, dir, w, h,
);
unsafe {
resolve_cdef_filter_hbd_shapes()[si](
&mut d1, 16, 0, &tmp, ts, o, pri, sec, ps, ss, pt, dir,
);
resolve_cdef_filter_hbd()(
&mut d2, 16, 0, &tmp, ts, o, pri, sec, ps, ss, pt, dir, w, h,
);
}
assert_eq!(
d0, d1,
"shape {w}x{h} bdm8={bdm8} pri={pri} sec={sec} dir={dir}"
);
assert_eq!(
d0, d2,
"generic {w}x{h} bdm8={bdm8} pri={pri} sec={sec} dir={dir}"
);
}
}
}