use crate::exec_context::ExecContext;
use crate::pixel::{BitDepth, Pixel};
macro_rules! call_rowops {
($f:expr, $($arg:expr),+ $(,)?) => {{
unsafe { ($f)($($arg),+) }
}};
}
#[inline]
pub(crate) fn avg_row_ctx<BD: BitDepth>(
exec: &ExecContext,
bd: BD,
dst: &mut [BD::Pixel],
tmp1: &[i16],
tmp2: &[i16],
n: usize,
rnd: i32,
sh: i32,
) {
if BD::BPC == 8 {
if let Some(d8) = <BD::Pixel as Pixel>::try_as_u8_slice_mut(dst) {
call_rowops!(exec.avg, d8, tmp1, tmp2, n, rnd, sh);
return;
}
} else if BD::BPC == 16 {
if let Some(d16) = <BD::Pixel as Pixel>::try_as_u16_slice_mut(dst) {
call_rowops!(exec.avg_hbd, d16, tmp1, tmp2, n, rnd, sh, bd.bitdepth_max());
return;
}
}
let (dc, dr) = dst[..n].as_chunks_mut::<8>();
let (t1c, t1r) = tmp1[..n].as_chunks::<8>();
let (t2c, t2r) = tmp2[..n].as_chunks::<8>();
for ((d, a), b) in dc.iter_mut().zip(t1c).zip(t2c) {
for k in 0..8 {
d[k] = bd.pixel_clip((a[k] as i32 + b[k] as i32 + rnd) >> sh);
}
}
for ((d, &a), &b) in dr.iter_mut().zip(t1r).zip(t2r) {
*d = bd.pixel_clip((a as i32 + b as i32 + rnd) >> sh);
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
pub(crate) fn w_avg_row_ctx<BD: BitDepth>(
exec: &ExecContext,
bd: BD,
dst: &mut [BD::Pixel],
tmp1: &[i16],
tmp2: &[i16],
n: usize,
weight: i32,
rnd: i32,
sh: i32,
) {
if BD::BPC == 8 {
if let Some(d8) = <BD::Pixel as Pixel>::try_as_u8_slice_mut(dst) {
call_rowops!(exec.w_avg, d8, tmp1, tmp2, n, weight, rnd, sh);
return;
}
} else if BD::BPC == 16 {
if let Some(d16) = <BD::Pixel as Pixel>::try_as_u16_slice_mut(dst) {
call_rowops!(
exec.w_avg_hbd,
d16,
tmp1,
tmp2,
n,
weight,
rnd,
sh,
bd.bitdepth_max()
);
return;
}
}
let (dc, dr) = dst[..n].as_chunks_mut::<8>();
let (t1c, t1r) = tmp1[..n].as_chunks::<8>();
let (t2c, t2r) = tmp2[..n].as_chunks::<8>();
for ((d, a), b) in dc.iter_mut().zip(t1c).zip(t2c) {
for k in 0..8 {
d[k] = bd.pixel_clip((a[k] as i32 * weight + b[k] as i32 * (16 - weight) + rnd) >> sh);
}
}
for ((d, &a), &b) in dr.iter_mut().zip(t1r).zip(t2r) {
*d = bd.pixel_clip((a as i32 * weight + b as i32 * (16 - weight) + rnd) >> sh);
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
pub(crate) fn mask_row_ctx<BD: BitDepth>(
exec: &ExecContext,
bd: BD,
dst: &mut [BD::Pixel],
tmp1: &[i16],
tmp2: &[i16],
mask: &[u8],
n: usize,
rnd: i32,
sh: i32,
) {
if BD::BPC == 8 {
if let Some(d8) = <BD::Pixel as Pixel>::try_as_u8_slice_mut(dst) {
call_rowops!(exec.mask, d8, tmp1, tmp2, mask, n, rnd, sh);
return;
}
} else if BD::BPC == 16 {
if let Some(d16) = <BD::Pixel as Pixel>::try_as_u16_slice_mut(dst) {
call_rowops!(
exec.mask_hbd,
d16,
tmp1,
tmp2,
mask,
n,
rnd,
sh,
bd.bitdepth_max()
);
return;
}
}
let (dc, dr) = dst[..n].as_chunks_mut::<8>();
let (t1c, t1r) = tmp1[..n].as_chunks::<8>();
let (t2c, t2r) = tmp2[..n].as_chunks::<8>();
let (mc, mr) = mask[..n].as_chunks::<8>();
for (((d, a), b), m) in dc.iter_mut().zip(t1c).zip(t2c).zip(mc) {
for k in 0..8 {
let mk = m[k] as i32;
d[k] = bd.pixel_clip((a[k] as i32 * mk + b[k] as i32 * (64 - mk) + rnd) >> sh);
}
}
for (((d, &a), &b), &m) in dr.iter_mut().zip(t1r).zip(t2r).zip(mr) {
let mk = m as i32;
*d = bd.pixel_clip((a as i32 * mk + b as i32 * (64 - mk) + rnd) >> sh);
}
}
#[inline]
pub(crate) fn blend_row_ctx<P: Pixel>(
exec: &ExecContext,
dst: &mut [P],
tmp: &[P],
mask: &[u8],
n: usize,
) {
if let (Some(t8), Some(d8)) = (P::try_as_u8_slice(tmp), P::try_as_u8_slice_mut(dst)) {
call_rowops!(exec.blend, d8, t8, mask, n);
return;
}
if let (Some(t16), Some(d16)) = (P::try_as_u16_slice(tmp), P::try_as_u16_slice_mut(dst)) {
call_rowops!(exec.blend_hbd, d16, t16, mask, n);
return;
}
let (dc, dr) = dst[..n].as_chunks_mut::<8>();
let (tc, tr) = tmp[..n].as_chunks::<8>();
let (mc, mr) = mask[..n].as_chunks::<8>();
for ((d, t), m) in dc.iter_mut().zip(tc).zip(mc) {
for k in 0..8 {
let mk = m[k] as i32;
let dv: i32 = d[k].into();
let tv: i32 = t[k].into();
d[k] = P::from_i32((dv * (64 - mk) + tv * mk + 32) >> 6);
}
}
for ((d, &t), &m) in dr.iter_mut().zip(tr).zip(mr) {
let mk = m as i32;
let dv: i32 = (*d).into();
let tv: i32 = t.into();
*d = P::from_i32((dv * (64 - mk) + tv * mk + 32) >> 6);
}
}
pub(crate) fn morph_row_ctx<BD: BitDepth>(
exec: &ExecContext,
bd: BD,
dst: &mut [BD::Pixel],
alpha: i32,
beta: i32,
n: usize,
) {
if BD::BPC == 8 {
if let Some(d8) = <BD::Pixel as Pixel>::try_as_u8_slice_mut(dst) {
call_rowops!(exec.morph, d8, alpha, beta, n);
return;
}
} else if BD::BPC == 16 {
if let Some(d16) = <BD::Pixel as Pixel>::try_as_u16_slice_mut(dst) {
call_rowops!(exec.morph_hbd, d16, alpha, beta, n, bd.bitdepth_max());
return;
}
}
let (dc, dr) = dst[..n].as_chunks_mut::<8>();
for d in dc.iter_mut() {
for k in 0..8 {
let dv: i32 = d[k].into();
d[k] = bd.pixel_clip((alpha * dv + beta) >> 8);
}
}
for d in dr.iter_mut() {
let dv: i32 = (*d).into();
*d = bd.pixel_clip((alpha * dv + beta) >> 8);
}
}
#[inline]
pub(crate) fn dc_add_row_ctx<BD: BitDepth>(
exec: &ExecContext,
bd: BD,
dst: &mut [BD::Pixel],
dc: i32,
n: usize,
) {
if BD::BPC == 8 {
if let Some(d8) = <BD::Pixel as Pixel>::try_as_u8_slice_mut(dst) {
call_rowops!(exec.dc_add, d8, dc, n);
return;
}
} else if BD::BPC == 16 {
if let Some(d16) = <BD::Pixel as Pixel>::try_as_u16_slice_mut(dst) {
call_rowops!(exec.dc_add_hbd, d16, dc, n, bd.bitdepth_max());
return;
}
}
let (dchunks, dr) = dst[..n].as_chunks_mut::<8>();
for d in dchunks.iter_mut() {
for k in 0..8 {
let p: i32 = d[k].into();
d[k] = bd.pixel_clip(p + dc);
}
}
for d in dr.iter_mut() {
let p: i32 = (*d).into();
*d = bd.pixel_clip(p + dc);
}
}
#[inline]
pub(crate) fn row_clip_ctx(
exec: &ExecContext,
tmp: &mut [i32],
n: usize,
rnd: i32,
shift: i32,
min: i32,
max: i32,
) {
call_rowops!(exec.row_clip, tmp, n, rnd, shift, min, max);
}
#[inline]
pub(crate) fn residual_add_row_ctx<BD: BitDepth>(
exec: &ExecContext,
bd: BD,
dst: &mut [BD::Pixel],
c: &[i32],
n: usize,
rnd: i32,
shift: i32,
) {
if BD::BPC == 8 {
if let Some(d8) = <BD::Pixel as Pixel>::try_as_u8_slice_mut(dst) {
call_rowops!(exec.residual_add, d8, c, n, rnd, shift);
return;
}
} else if BD::BPC == 16 {
if let Some(d16) = <BD::Pixel as Pixel>::try_as_u16_slice_mut(dst) {
call_rowops!(
exec.residual_add_hbd,
d16,
c,
n,
rnd,
shift,
bd.bitdepth_max()
);
return;
}
}
let (dc, dr) = dst[..n].as_chunks_mut::<8>();
let (cc, cr) = c[..n].as_chunks::<8>();
for (d, cv) in dc.iter_mut().zip(cc) {
for k in 0..8 {
let p: i32 = d[k].into();
d[k] = bd.pixel_clip(p + ((cv[k] + rnd) >> shift));
}
}
for (d, &cv) in dr.iter_mut().zip(cr) {
let p: i32 = (*d).into();
*d = bd.pixel_clip(p + ((cv + rnd) >> shift));
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
pub(crate) fn cctx_row_ctx(
exec: &ExecContext,
u: &mut [i32],
v: &mut [i32],
sina: i32,
cosa: i32,
sz: usize,
min: i32,
max: i32,
) {
call_rowops!(exec.cctx, u, v, sina, cosa, sz, min, max);
}
pub(crate) struct WienerTap<'a> {
pub row_p: &'a [u8],
pub row_m: &'a [u8],
pub dx: i32,
pub coef: i32,
}
pub(crate) type NsWienerFirFn = unsafe fn(&mut [u8], &[u8], usize, &[WienerTap<'_>], usize);
pub(crate) type PcWienerFirFn = unsafe fn(&mut [u8], &[u8], i32, usize, &[WienerTap<'_>], usize);
static NS_WIENER_FIR: std::sync::OnceLock<NsWienerFirFn> = std::sync::OnceLock::new();
static PC_WIENER_FIR: std::sync::OnceLock<PcWienerFirFn> = std::sync::OnceLock::new();
#[inline]
pub(crate) fn ns_wiener_fir_run() -> NsWienerFirFn {
*NS_WIENER_FIR.get_or_init(|| {
let mut _f: NsWienerFirFn = ns_wiener_fir_run_scalar;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::ns_wiener_fir_run_neon;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::ns_wiener_fir_run_sse41;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::ns_wiener_fir_run_avx2;
}
}
_f
})
}
#[inline]
pub(crate) fn pc_wiener_fir_run() -> PcWienerFirFn {
*PC_WIENER_FIR.get_or_init(|| {
let mut _f: PcWienerFirFn = pc_wiener_fir_run_scalar;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::pc_wiener_fir_run_neon;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::pc_wiener_fir_run_sse41;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::pc_wiener_fir_run_avx2;
}
}
_f
})
}
pub(crate) fn ns_wiener_fir_run_scalar(
dst: &mut [u8],
center: &[u8],
col0: usize,
taps: &[WienerTap],
n: usize,
) {
for x in 0..n {
let c = col0 + x;
let m = center[c] as i32;
let mut s = m << 7;
for t in taps {
let a = t.row_p[(c as i32 + t.dx) as usize] as i32;
let b = t.row_m[(c as i32 - t.dx) as usize] as i32;
s += (a + b - 2 * m) * t.coef;
}
dst[x] = ((s + 64) >> 7).clamp(0, 255) as u8;
}
}
pub(crate) struct UvLumaTap<'a> {
pub(crate) row: &'a [u8],
pub(crate) ldx: i32,
pub(crate) coef: i32,
}
pub(crate) type NsWienerUvFirFn = unsafe fn(
&mut [u8],
&[u8],
usize,
&[WienerTap<'_>],
&[u8],
usize,
&[UvLumaTap<'_>],
usize,
usize,
);
static NS_WIENER_UV_FIR: std::sync::OnceLock<NsWienerUvFirFn> = std::sync::OnceLock::new();
#[inline]
pub(crate) fn ns_wiener_uv_fir_run() -> NsWienerUvFirFn {
*NS_WIENER_UV_FIR.get_or_init(|| {
let mut _f: NsWienerUvFirFn = ns_wiener_uv_fir_run_scalar;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::ns_wiener_uv_fir_run_neon;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::ns_wiener_uv_fir_run_sse41;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::ns_wiener_uv_fir_run_avx2;
}
}
_f
})
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn ns_wiener_uv_fir_run_scalar(
dst: &mut [u8],
c_center: &[u8],
co: usize,
ctaps: &[WienerTap],
l_center: &[u8],
lo: usize,
ltaps: &[UvLumaTap],
lstep: usize,
n: usize,
) {
for x in 0..n {
let cc = co + x;
let m = c_center[cc] as i32;
let mut s = m << 7;
for t in ctaps {
let a = t.row_p[(cc as i32 + t.dx) as usize] as i32;
let b = t.row_m[(cc as i32 - t.dx) as usize] as i32;
s += (a + b - 2 * m) * t.coef;
}
let lcx = lo + x * lstep;
let lc = l_center[lcx] as i32;
for t in ltaps {
let lv = t.row[(lcx as i32 + t.ldx) as usize] as i32;
s += (lv - lc) * t.coef;
}
dst[x] = ((s + 64) >> 7).clamp(0, 255) as u8;
}
}
pub(crate) fn pc_wiener_fir_run_scalar(
dst: &mut [u8],
center: &[u8],
center_coef: i32,
col0: usize,
taps: &[WienerTap],
n: usize,
) {
for x in 0..n {
let c = col0 + x;
let m = center[c] as i32;
let mut s = m * center_coef;
for t in taps {
let a = t.row_p[(c as i32 + t.dx) as usize] as i32;
let b = t.row_m[(c as i32 - t.dx) as usize] as i32;
s += (a + b) * t.coef;
}
dst[x] = ((s + 64) >> 7).clamp(0, 255) as u8;
}
}
pub(crate) struct WienerTapHbd<'a> {
pub row_p: &'a [u16],
pub row_m: &'a [u16],
pub dx: i32,
pub coef: i32,
}
pub(crate) type NsWienerFirHbdFn =
unsafe fn(&mut [u16], &[u16], usize, &[WienerTapHbd<'_>], usize, i32);
pub(crate) type PcWienerFirHbdFn =
unsafe fn(&mut [u16], &[u16], i32, usize, &[WienerTapHbd<'_>], usize, i32);
static NS_WIENER_FIR_HBD: std::sync::OnceLock<NsWienerFirHbdFn> = std::sync::OnceLock::new();
static PC_WIENER_FIR_HBD: std::sync::OnceLock<PcWienerFirHbdFn> = std::sync::OnceLock::new();
#[inline]
pub(crate) fn ns_wiener_fir_run_hbd() -> NsWienerFirHbdFn {
*NS_WIENER_FIR_HBD.get_or_init(|| {
let mut _f: NsWienerFirHbdFn = ns_wiener_fir_run_hbd_scalar;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::ns_wiener_fir_run_hbd_neon;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::ns_wiener_fir_run_hbd_sse41;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::ns_wiener_fir_run_hbd_avx2;
}
}
_f
})
}
#[inline]
pub(crate) fn pc_wiener_fir_run_hbd() -> PcWienerFirHbdFn {
*PC_WIENER_FIR_HBD.get_or_init(|| {
let mut _f: PcWienerFirHbdFn = pc_wiener_fir_run_hbd_scalar;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::pc_wiener_fir_run_hbd_neon;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::pc_wiener_fir_run_hbd_sse41;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::pc_wiener_fir_run_hbd_avx2;
}
}
_f
})
}
pub(crate) fn ns_wiener_fir_run_hbd_scalar(
dst: &mut [u16],
center: &[u16],
col0: usize,
taps: &[WienerTapHbd],
n: usize,
bitdepth_max: i32,
) {
for x in 0..n {
let c = col0 + x;
let m = center[c] as i32;
let mut s = m << 7;
for t in taps {
let a = t.row_p[(c as i32 + t.dx) as usize] as i32;
let b = t.row_m[(c as i32 - t.dx) as usize] as i32;
s += (a + b - 2 * m) * t.coef;
}
dst[x] = ((s + 64) >> 7).clamp(0, bitdepth_max) as u16;
}
}
pub(crate) fn pc_wiener_fir_run_hbd_scalar(
dst: &mut [u16],
center: &[u16],
center_coef: i32,
col0: usize,
taps: &[WienerTapHbd],
n: usize,
bitdepth_max: i32,
) {
for x in 0..n {
let c = col0 + x;
let m = center[c] as i32;
let mut s = m * center_coef;
for t in taps {
let a = t.row_p[(c as i32 + t.dx) as usize] as i32;
let b = t.row_m[(c as i32 - t.dx) as usize] as i32;
s += (a + b) * t.coef;
}
dst[x] = ((s + 64) >> 7).clamp(0, bitdepth_max) as u16;
}
}
pub(crate) struct UvLumaTapHbd<'a> {
pub(crate) row: &'a [u16],
pub(crate) ldx: i32,
pub(crate) coef: i32,
}
pub(crate) type NsWienerUvFirHbdFn = unsafe fn(
&mut [u16],
&[u16],
usize,
&[WienerTapHbd<'_>],
&[u16],
usize,
&[UvLumaTapHbd<'_>],
usize,
usize,
i32,
);
static NS_WIENER_UV_FIR_HBD: std::sync::OnceLock<NsWienerUvFirHbdFn> = std::sync::OnceLock::new();
#[inline]
pub(crate) fn ns_wiener_uv_fir_run_hbd() -> NsWienerUvFirHbdFn {
*NS_WIENER_UV_FIR_HBD.get_or_init(|| {
let mut _f: NsWienerUvFirHbdFn = ns_wiener_uv_fir_run_hbd_scalar;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::ns_wiener_uv_fir_run_hbd_neon;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::ns_wiener_uv_fir_run_hbd_sse41;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::ns_wiener_uv_fir_run_hbd_avx2;
}
}
_f
})
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn ns_wiener_uv_fir_run_hbd_scalar(
dst: &mut [u16],
c_center: &[u16],
co: usize,
ctaps: &[WienerTapHbd],
l_center: &[u16],
lo: usize,
ltaps: &[UvLumaTapHbd],
lstep: usize,
n: usize,
bitdepth_max: i32,
) {
for x in 0..n {
let cc = co + x;
let m = c_center[cc] as i32;
let mut s = m << 7;
for t in ctaps {
let a = t.row_p[(cc as i32 + t.dx) as usize] as i32;
let b = t.row_m[(cc as i32 - t.dx) as usize] as i32;
s += (a + b - 2 * m) * t.coef;
}
let lcx = lo + x * lstep;
let lc = l_center[lcx] as i32;
for t in ltaps {
let lv = t.row[(lcx as i32 + t.ldx) as usize] as i32;
s += (lv - lc) * t.coef;
}
dst[x] = ((s + 64) >> 7).clamp(0, bitdepth_max) as u16;
}
}
#[inline]
pub(crate) fn gdf_add_run_ctx(
exec: &ExecContext,
dst: &mut [u8],
err: &[i8],
scale: i32,
n: usize,
) {
call_rowops!(exec.gdf_add, dst, err, scale, n);
}
#[inline]
pub(crate) fn gdf_add_run_hbd_ctx(
exec: &ExecContext,
dst: &mut [u16],
err: &[i8],
scale: i32,
n: usize,
bitdepth_max: i32,
) {
call_rowops!(exec.gdf_add_hbd, dst, err, scale, n, bitdepth_max);
}
#[allow(clippy::too_many_arguments)]
#[inline]
pub(crate) fn gdf_gradient_group_ctx(
exec: &ExecContext,
dst: &mut [[u16; 4]],
d: usize,
base_cell: usize,
ncells: usize,
center_rows: [&[u8]; 2],
a_rows: [&[u8]; 2],
c_rows: [&[u8]; 2],
col0: usize,
dx: i32,
shift: u32,
) {
call_rowops!(
exec.gdf_grad,
dst,
d,
base_cell,
ncells,
center_rows,
a_rows,
c_rows,
col0,
dx,
shift,
);
}
#[allow(clippy::too_many_arguments)]
#[inline]
pub(crate) fn gdf_gradient_group_hbd_ctx(
exec: &ExecContext,
dst: &mut [[u16; 4]],
d: usize,
base_cell: usize,
ncells: usize,
center_rows: [&[u16]; 2],
a_rows: [&[u16]; 2],
c_rows: [&[u16]; 2],
col0: usize,
dx: i32,
shift: u32,
) {
call_rowops!(
exec.gdf_grad_hbd,
dst,
d,
base_cell,
ncells,
center_rows,
a_rows,
c_rows,
col0,
dx,
shift,
);
}
#[inline]
pub(crate) fn row_clip(tmp: &mut [i32], n: usize, rnd: i32, shift: i32, min: i32, max: i32) {
let exec = ExecContext::new();
row_clip_ctx(&exec, tmp, n, rnd, shift, min, max);
}
#[cfg(test)]
mod wiener_scalar_proof {
use super::*;
struct Rng(u64);
impl Rng {
fn next(&mut self) -> u64 {
self.0 ^= self.0 << 13;
self.0 ^= self.0 >> 7;
self.0 ^= self.0 << 17;
self.0
}
fn u8(&mut self) -> u8 {
(self.next() & 0xff) as u8
}
fn range(&mut self, lo: i32, hi: i32) -> i32 {
lo + (self.next() % ((hi - lo) as u64 + 1)) as i32
}
}
fn buf(rng: &mut Rng, len: usize) -> Vec<u8> {
(0..len).map(|_| rng.u8()).collect()
}
#[test]
fn ns_wiener_dispatch_matches_scalar() {
let mut rng = Rng(0xD15A);
let f = ns_wiener_fir_run();
for _ in 0..400 {
let len = 256usize;
let center = buf(&mut rng, len);
let n_taps = rng.range(1, 8) as usize;
let rows: Vec<(Vec<u8>, Vec<u8>, i32, i32)> = (0..n_taps)
.map(|_| {
(
buf(&mut rng, len),
buf(&mut rng, len),
rng.range(1, 16),
rng.range(-64, 64),
)
})
.collect();
let taps: Vec<WienerTap> = rows
.iter()
.map(|(p, m, dx, coef)| WienerTap {
row_p: p,
row_m: m,
dx: *dx,
coef: *coef,
})
.collect();
let col0 = 64usize;
let n = rng.range(1, 100) as usize;
let mut d_ref = vec![0u8; n];
let mut d_dsp = vec![0u8; n];
ns_wiener_fir_run_scalar(&mut d_ref, ¢er, col0, &taps, n);
unsafe { f(&mut d_dsp, ¢er, col0, &taps, n) };
assert_eq!(d_ref, d_dsp, "ns dispatch mismatch n={} taps={}", n, n_taps);
}
}
#[test]
fn pc_wiener_dispatch_matches_scalar() {
let mut rng = Rng(0xD15B);
let f = pc_wiener_fir_run();
for _ in 0..400 {
let len = 256usize;
let center = buf(&mut rng, len);
let center_coef = rng.range(-128, 128);
let n_taps = rng.range(1, 6) as usize;
let rows: Vec<(Vec<u8>, Vec<u8>, i32, i32)> = (0..n_taps)
.map(|_| {
(
buf(&mut rng, len),
buf(&mut rng, len),
rng.range(1, 16),
rng.range(-64, 64),
)
})
.collect();
let taps: Vec<WienerTap> = rows
.iter()
.map(|(p, m, dx, coef)| WienerTap {
row_p: p,
row_m: m,
dx: *dx,
coef: *coef,
})
.collect();
let col0 = 64usize;
let n = rng.range(1, 100) as usize;
let mut d_ref = vec![0u8; n];
let mut d_dsp = vec![0u8; n];
pc_wiener_fir_run_scalar(&mut d_ref, ¢er, center_coef, col0, &taps, n);
unsafe { f(&mut d_dsp, ¢er, center_coef, col0, &taps, n) };
assert_eq!(d_ref, d_dsp, "pc dispatch mismatch n={} taps={}", n, n_taps);
}
}
fn u16_buf(rng: &mut Rng, len: usize, max: u16) -> Vec<u16> {
(0..len).map(|_| rng.range(0, max as i32) as u16).collect()
}
#[test]
fn ns_wiener_uv_dispatch_matches_scalar() {
let mut rng = Rng(0xD15C);
let f = ns_wiener_uv_fir_run();
for _ in 0..400 {
let len = 320usize;
let c_center = buf(&mut rng, len);
let l_center = buf(&mut rng, len);
let lstep = if rng.range(0, 1) == 0 { 1usize } else { 2usize };
let n_taps = rng.range(1, 8) as usize;
let c_rows: Vec<(Vec<u8>, Vec<u8>, i32, i32)> = (0..n_taps)
.map(|_| {
(
buf(&mut rng, len),
buf(&mut rng, len),
rng.range(-4, 4),
rng.range(-64, 64),
)
})
.collect();
let ctaps: Vec<WienerTap> = c_rows
.iter()
.map(|(p, m, dx, coef)| WienerTap {
row_p: p,
row_m: m,
dx: *dx,
coef: *coef,
})
.collect();
let n_luma_taps = rng.range(1, 12) as usize;
let l_rows: Vec<(Vec<u8>, i32, i32)> = (0..n_luma_taps)
.map(|_| {
(
buf(&mut rng, len),
rng.range(-4, 4) * lstep as i32,
rng.range(-64, 64),
)
})
.collect();
let ltaps: Vec<UvLumaTap> = l_rows
.iter()
.map(|(row, ldx, coef)| UvLumaTap {
row,
ldx: *ldx,
coef: *coef,
})
.collect();
let co = 64usize;
let lo = 32usize;
let n = rng.range(1, 96) as usize;
let mut d_ref = vec![0u8; n];
let mut d_dsp = vec![0u8; n];
ns_wiener_uv_fir_run_scalar(
&mut d_ref, &c_center, co, &ctaps, &l_center, lo, <aps, lstep, n,
);
unsafe {
f(
&mut d_dsp, &c_center, co, &ctaps, &l_center, lo, <aps, lstep, n,
)
};
assert_eq!(
d_ref, d_dsp,
"uv dispatch mismatch n={} ctaps={} ltaps={} lstep={}",
n, n_taps, n_luma_taps, lstep
);
}
}
#[test]
fn ns_wiener_hbd_dispatch_matches_scalar() {
let mut rng = Rng(0xD15D);
let f = ns_wiener_fir_run_hbd();
for _ in 0..400 {
let max = if rng.range(0, 1) == 0 {
1023u16
} else {
4095u16
};
let len = 320usize;
let center = u16_buf(&mut rng, len, max);
let n_taps = rng.range(1, 8) as usize;
let rows: Vec<(Vec<u16>, Vec<u16>, i32, i32)> = (0..n_taps)
.map(|_| {
(
u16_buf(&mut rng, len, max),
u16_buf(&mut rng, len, max),
rng.range(-4, 4),
rng.range(-64, 64),
)
})
.collect();
let taps: Vec<WienerTapHbd> = rows
.iter()
.map(|(p, m, dx, coef)| WienerTapHbd {
row_p: p,
row_m: m,
dx: *dx,
coef: *coef,
})
.collect();
let col0 = 64usize;
let n = rng.range(1, 128) as usize;
let mut d_ref = vec![0u16; n];
let mut d_dsp = vec![0u16; n];
ns_wiener_fir_run_hbd_scalar(&mut d_ref, ¢er, col0, &taps, n, max as i32);
unsafe { f(&mut d_dsp, ¢er, col0, &taps, n, max as i32) };
assert_eq!(
d_ref, d_dsp,
"hbd ns dispatch mismatch n={} taps={}",
n, n_taps
);
}
}
#[test]
fn pc_wiener_hbd_dispatch_matches_scalar() {
let mut rng = Rng(0xD15E);
let f = pc_wiener_fir_run_hbd();
for _ in 0..400 {
let max = if rng.range(0, 1) == 0 {
1023u16
} else {
4095u16
};
let len = 320usize;
let center = u16_buf(&mut rng, len, max);
let center_coef = rng.range(-256, 256);
let n_taps = rng.range(1, 12) as usize;
let rows: Vec<(Vec<u16>, Vec<u16>, i32, i32)> = (0..n_taps)
.map(|_| {
(
u16_buf(&mut rng, len, max),
u16_buf(&mut rng, len, max),
rng.range(-4, 4),
rng.range(-64, 64),
)
})
.collect();
let taps: Vec<WienerTapHbd> = rows
.iter()
.map(|(p, m, dx, coef)| WienerTapHbd {
row_p: p,
row_m: m,
dx: *dx,
coef: *coef,
})
.collect();
let col0 = 64usize;
let n = rng.range(1, 128) as usize;
let mut d_ref = vec![0u16; n];
let mut d_dsp = vec![0u16; n];
pc_wiener_fir_run_hbd_scalar(
&mut d_ref,
¢er,
center_coef,
col0,
&taps,
n,
max as i32,
);
unsafe { f(&mut d_dsp, ¢er, center_coef, col0, &taps, n, max as i32) };
assert_eq!(
d_ref, d_dsp,
"hbd pc dispatch mismatch n={} taps={}",
n, n_taps
);
}
}
#[test]
fn ns_wiener_uv_hbd_dispatch_matches_scalar() {
let mut rng = Rng(0xD15F);
let f = ns_wiener_uv_fir_run_hbd();
for _ in 0..400 {
let max = if rng.range(0, 1) == 0 {
1023u16
} else {
4095u16
};
let len = 360usize;
let c_center = u16_buf(&mut rng, len, max);
let l_center = u16_buf(&mut rng, len, max);
let lstep = if rng.range(0, 1) == 0 { 1usize } else { 2usize };
let n_taps = rng.range(1, 8) as usize;
let c_rows: Vec<(Vec<u16>, Vec<u16>, i32, i32)> = (0..n_taps)
.map(|_| {
(
u16_buf(&mut rng, len, max),
u16_buf(&mut rng, len, max),
rng.range(-4, 4),
rng.range(-64, 64),
)
})
.collect();
let ctaps: Vec<WienerTapHbd> = c_rows
.iter()
.map(|(p, m, dx, coef)| WienerTapHbd {
row_p: p,
row_m: m,
dx: *dx,
coef: *coef,
})
.collect();
let n_luma_taps = rng.range(1, 12) as usize;
let l_rows: Vec<(Vec<u16>, i32, i32)> = (0..n_luma_taps)
.map(|_| {
(
u16_buf(&mut rng, len, max),
rng.range(-4, 4) * lstep as i32,
rng.range(-64, 64),
)
})
.collect();
let ltaps: Vec<UvLumaTapHbd> = l_rows
.iter()
.map(|(row, ldx, coef)| UvLumaTapHbd {
row,
ldx: *ldx,
coef: *coef,
})
.collect();
let co = 64usize;
let lo = 32usize;
let n = rng.range(1, 128) as usize;
let mut d_ref = vec![0u16; n];
let mut d_dsp = vec![0u16; n];
ns_wiener_uv_fir_run_hbd_scalar(
&mut d_ref, &c_center, co, &ctaps, &l_center, lo, <aps, lstep, n, max as i32,
);
unsafe {
f(
&mut d_dsp, &c_center, co, &ctaps, &l_center, lo, <aps, lstep, n, max as i32,
)
};
assert_eq!(
d_ref, d_dsp,
"hbd uv dispatch mismatch n={} ctaps={} ltaps={} lstep={}",
n, n_taps, n_luma_taps, lstep
);
}
}
}