use std::sync::OnceLock;
use crate::gdf_tables::{GDF_ALPHA, GDF_BIAS, GDF_INTER_ERROR, GDF_INTRA_ERROR, GDF_WEIGHT};
const GDF_PREP_COORDS: [[i8; 2]; 18] = [
[6, 0],
[5, 0],
[4, 0],
[3, 0],
[2, 1],
[2, 0],
[2, -1],
[1, 2],
[1, 1],
[1, 0],
[1, -1],
[1, -2],
[0, 6],
[0, 5],
[0, 4],
[0, 3],
[0, 2],
[0, 1],
];
const GDF_PREP_COORDS_8BPC: [[i8; 2]; 18] = [
[6, 0],
[5, 0],
[4, 0],
[3, 0],
[2, 1],
[2, 0],
[2, -1],
[1, 2],
[1, 1],
[1, 0],
[1, -1],
[1, -2],
[0, 6],
[0, 5],
[0, 4],
[0, 3],
[0, 2],
[0, 1],
];
pub(crate) type ResidualAddFn = unsafe fn(&mut [u8], &[i32], usize, i32, i32);
pub(crate) fn residual_add_row_8bpc_scalar(
dst: &mut [u8],
c: &[i32],
n: usize,
rnd: i32,
shift: i32,
) {
for (d, &coeff) in dst[..n].iter_mut().zip(&c[..n]) {
let p = *d as i32;
*d = (p + ((coeff + rnd) >> shift)).clamp(0, 255) as u8;
}
}
static RESIDUAL_ADD: OnceLock<ResidualAddFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_residual_add() -> ResidualAddFn {
*RESIDUAL_ADD.get_or_init(|| {
let mut _f = residual_add_row_8bpc_scalar as ResidualAddFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::residual_add_row_8bpc_neon as ResidualAddFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::residual_add_row_8bpc_sse41 as ResidualAddFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::residual_add_row_8bpc_avx2 as ResidualAddFn;
}
}
_f
})
}
pub(crate) type DcAddFn = unsafe fn(&mut [u8], i32, usize);
pub(crate) fn dc_add_row_8bpc_scalar(dst: &mut [u8], dc: i32, n: usize) {
for d in dst[..n].iter_mut() {
*d = (*d as i32 + dc).clamp(0, 255) as u8;
}
}
static DC_ADD: OnceLock<DcAddFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_dc_add() -> DcAddFn {
*DC_ADD.get_or_init(|| {
let mut _f = dc_add_row_8bpc_scalar as DcAddFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::dc_add_row_8bpc_neon as DcAddFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::dc_add_row_8bpc_sse41 as DcAddFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::dc_add_row_8bpc_avx2 as DcAddFn;
}
}
_f
})
}
pub(crate) type RowClipFn = unsafe fn(&mut [i32], usize, i32, i32, i32, i32);
pub(crate) fn row_clip_scalar(tmp: &mut [i32], n: usize, rnd: i32, shift: i32, min: i32, max: i32) {
for t in tmp[..n].iter_mut() {
*t = ((*t + rnd) >> shift).max(min).min(max);
}
}
static ROW_CLIP: OnceLock<RowClipFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_row_clip() -> RowClipFn {
*ROW_CLIP.get_or_init(|| {
let mut _f = row_clip_scalar as RowClipFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::row_clip_neon as RowClipFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::row_clip_sse41 as RowClipFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::row_clip_avx2 as RowClipFn;
}
}
_f
})
}
pub(crate) type CctxFn = unsafe fn(&mut [i32], &mut [i32], i32, i32, usize, i32, i32);
pub(crate) fn cctx_row_scalar(
u: &mut [i32],
v: &mut [i32],
sina: i32,
cosa: i32,
sz: usize,
min: i32,
max: i32,
) {
for (u, v) in u[..sz].iter_mut().zip(&mut v[..sz]) {
let ui = *u;
let vi = *v;
let a = ui * cosa - vi * sina;
let b = ui * sina + vi * cosa;
*u = ((a + 128 - (a < 0) as i32) >> 8).max(min).min(max);
*v = ((b + 128 - (b < 0) as i32) >> 8).max(min).min(max);
}
}
static CCTX: OnceLock<CctxFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_cctx() -> CctxFn {
*CCTX.get_or_init(|| {
let mut _f = cctx_row_scalar as CctxFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::cctx_row_neon as CctxFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::cctx_row_sse41 as CctxFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::cctx_row_avx2 as CctxFn;
}
}
_f
})
}
pub(crate) type AvgFn = unsafe fn(&mut [u8], &[i16], &[i16], usize, i32, i32);
pub(crate) fn avg_row_8bpc_scalar(
dst: &mut [u8],
t1: &[i16],
t2: &[i16],
n: usize,
rnd: i32,
sh: i32,
) {
for ((d, &a), &b) in dst[..n].iter_mut().zip(&t1[..n]).zip(&t2[..n]) {
*d = ((a as i32 + b as i32 + rnd) >> sh).clamp(0, 255) as u8;
}
}
static AVG: OnceLock<AvgFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_avg() -> AvgFn {
*AVG.get_or_init(|| {
let mut _f = avg_row_8bpc_scalar as AvgFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::avg_row_8bpc_neon as AvgFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::avg_row_8bpc_sse41 as AvgFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::avg_row_8bpc_avx2 as AvgFn;
}
}
_f
})
}
pub(crate) type WAvgFn = unsafe fn(&mut [u8], &[i16], &[i16], usize, i32, i32, i32);
#[allow(clippy::too_many_arguments)]
pub(crate) fn w_avg_row_8bpc_scalar(
dst: &mut [u8],
t1: &[i16],
t2: &[i16],
n: usize,
weight: i32,
rnd: i32,
sh: i32,
) {
for ((d, &a), &b) in dst[..n].iter_mut().zip(&t1[..n]).zip(&t2[..n]) {
*d = ((a as i32 * weight + b as i32 * (16 - weight) + rnd) >> sh).clamp(0, 255) as u8;
}
}
static W_AVG: OnceLock<WAvgFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_w_avg() -> WAvgFn {
*W_AVG.get_or_init(|| {
let mut _f = w_avg_row_8bpc_scalar as WAvgFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::w_avg_row_8bpc_neon as WAvgFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::w_avg_row_8bpc_sse41 as WAvgFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::w_avg_row_8bpc_avx2 as WAvgFn;
}
}
_f
})
}
pub(crate) type MaskFn = unsafe fn(&mut [u8], &[i16], &[i16], &[u8], usize, i32, i32);
#[allow(clippy::too_many_arguments)]
pub(crate) fn mask_row_8bpc_scalar(
dst: &mut [u8],
t1: &[i16],
t2: &[i16],
mask: &[u8],
n: usize,
rnd: i32,
sh: i32,
) {
for (((d, &a), &b), &m) in dst[..n]
.iter_mut()
.zip(&t1[..n])
.zip(&t2[..n])
.zip(&mask[..n])
{
let m = m as i32;
*d = ((a as i32 * m + b as i32 * (64 - m) + rnd) >> sh).clamp(0, 255) as u8;
}
}
static MASK: OnceLock<MaskFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_mask() -> MaskFn {
*MASK.get_or_init(|| {
let mut _f = mask_row_8bpc_scalar as MaskFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::mask_row_8bpc_neon as MaskFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::mask_row_8bpc_sse41 as MaskFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::mask_row_8bpc_avx2 as MaskFn;
}
}
_f
})
}
pub(crate) type BlendFn = unsafe fn(&mut [u8], &[u8], &[u8], usize);
pub(crate) fn blend_row_8bpc_scalar(dst: &mut [u8], tmp: &[u8], mask: &[u8], n: usize) {
for ((d, &t), &m) in dst[..n].iter_mut().zip(&tmp[..n]).zip(&mask[..n]) {
let m = m as i32;
let d0 = *d as i32;
let t = t as i32;
*d = ((d0 * (64 - m) + t * m + 32) >> 6) as u8;
}
}
static BLEND: OnceLock<BlendFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_blend() -> BlendFn {
*BLEND.get_or_init(|| {
let mut _f = blend_row_8bpc_scalar as BlendFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::blend_row_8bpc_neon as BlendFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::blend_row_8bpc_sse41 as BlendFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::blend_row_8bpc_avx2 as BlendFn;
}
}
_f
})
}
pub(crate) type MorphFn = unsafe fn(&mut [u8], i32, i32, usize);
pub(crate) fn morph_row_8bpc_scalar(dst: &mut [u8], alpha: i32, beta: i32, n: usize) {
for d in &mut dst[..n] {
*d = ((alpha * *d as i32 + beta) >> 8).clamp(0, 255) as u8;
}
}
static MORPH: OnceLock<MorphFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_morph() -> MorphFn {
*MORPH.get_or_init(|| {
let mut _f = morph_row_8bpc_scalar as MorphFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::morph_row_8bpc_neon as MorphFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::morph_row_8bpc_sse41 as MorphFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::morph_row_8bpc_avx2 as MorphFn;
}
}
_f
})
}
pub(crate) type ResidualAddHbdFn = unsafe fn(&mut [u16], &[i32], usize, i32, i32, i32);
pub(crate) fn residual_add_row_hbd_scalar(
dst: &mut [u16],
c: &[i32],
n: usize,
rnd: i32,
shift: i32,
bitdepth_max: i32,
) {
for (d, &coeff) in dst[..n].iter_mut().zip(&c[..n]) {
*d = (*d as i32 + ((coeff + rnd) >> shift)).clamp(0, bitdepth_max) as u16;
}
}
static RESIDUAL_ADD_HBD: OnceLock<ResidualAddHbdFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_residual_add_hbd() -> ResidualAddHbdFn {
*RESIDUAL_ADD_HBD.get_or_init(|| {
let mut _f = residual_add_row_hbd_scalar as ResidualAddHbdFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::residual_add_row_hbd_neon as ResidualAddHbdFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::residual_add_row_hbd_sse41 as ResidualAddHbdFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::residual_add_row_hbd_avx2 as ResidualAddHbdFn;
}
}
_f
})
}
pub(crate) type DcAddHbdFn = unsafe fn(&mut [u16], i32, usize, i32);
pub(crate) fn dc_add_row_hbd_scalar(dst: &mut [u16], dc: i32, n: usize, bitdepth_max: i32) {
for d in dst[..n].iter_mut() {
*d = (*d as i32 + dc).clamp(0, bitdepth_max) as u16;
}
}
static DC_ADD_HBD: OnceLock<DcAddHbdFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_dc_add_hbd() -> DcAddHbdFn {
*DC_ADD_HBD.get_or_init(|| {
let mut _f = dc_add_row_hbd_scalar as DcAddHbdFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::dc_add_row_hbd_neon as DcAddHbdFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::dc_add_row_hbd_sse41 as DcAddHbdFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::dc_add_row_hbd_avx2 as DcAddHbdFn;
}
}
_f
})
}
pub(crate) type AvgHbdFn = unsafe fn(&mut [u16], &[i16], &[i16], usize, i32, i32, i32);
pub(crate) fn avg_row_hbd_scalar(
dst: &mut [u16],
t1: &[i16],
t2: &[i16],
n: usize,
rnd: i32,
sh: i32,
bitdepth_max: i32,
) {
for ((d, &a), &b) in dst[..n].iter_mut().zip(&t1[..n]).zip(&t2[..n]) {
*d = ((a as i32 + b as i32 + rnd) >> sh).clamp(0, bitdepth_max) as u16;
}
}
static AVG_HBD: OnceLock<AvgHbdFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_avg_hbd() -> AvgHbdFn {
*AVG_HBD.get_or_init(|| {
let mut _f = avg_row_hbd_scalar as AvgHbdFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::avg_row_hbd_neon as AvgHbdFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::avg_row_hbd_sse41 as AvgHbdFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::avg_row_hbd_avx2 as AvgHbdFn;
}
}
_f
})
}
pub(crate) type WAvgHbdFn = unsafe fn(&mut [u16], &[i16], &[i16], usize, i32, i32, i32, i32);
#[allow(clippy::too_many_arguments)]
pub(crate) fn w_avg_row_hbd_scalar(
dst: &mut [u16],
t1: &[i16],
t2: &[i16],
n: usize,
weight: i32,
rnd: i32,
sh: i32,
bitdepth_max: i32,
) {
for ((d, &a), &b) in dst[..n].iter_mut().zip(&t1[..n]).zip(&t2[..n]) {
*d = ((a as i32 * weight + b as i32 * (16 - weight) + rnd) >> sh).clamp(0, bitdepth_max)
as u16;
}
}
static W_AVG_HBD: OnceLock<WAvgHbdFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_w_avg_hbd() -> WAvgHbdFn {
*W_AVG_HBD.get_or_init(|| {
let mut _f = w_avg_row_hbd_scalar as WAvgHbdFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::w_avg_row_hbd_neon as WAvgHbdFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::w_avg_row_hbd_sse41 as WAvgHbdFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::w_avg_row_hbd_avx2 as WAvgHbdFn;
}
}
_f
})
}
pub(crate) type MaskHbdFn = unsafe fn(&mut [u16], &[i16], &[i16], &[u8], usize, i32, i32, i32);
#[allow(clippy::too_many_arguments)]
pub(crate) fn mask_row_hbd_scalar(
dst: &mut [u16],
t1: &[i16],
t2: &[i16],
mask: &[u8],
n: usize,
rnd: i32,
sh: i32,
bitdepth_max: i32,
) {
for (((d, &a), &b), &m) in dst[..n]
.iter_mut()
.zip(&t1[..n])
.zip(&t2[..n])
.zip(&mask[..n])
{
let m = m as i32;
*d = ((a as i32 * m + b as i32 * (64 - m) + rnd) >> sh).clamp(0, bitdepth_max) as u16;
}
}
static MASK_HBD: OnceLock<MaskHbdFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_mask_hbd() -> MaskHbdFn {
*MASK_HBD.get_or_init(|| {
let mut _f = mask_row_hbd_scalar as MaskHbdFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::mask_row_hbd_neon as MaskHbdFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::mask_row_hbd_sse41 as MaskHbdFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::mask_row_hbd_avx2 as MaskHbdFn;
}
}
_f
})
}
pub(crate) type BlendHbdFn = unsafe fn(&mut [u16], &[u16], &[u8], usize);
pub(crate) fn blend_row_hbd_scalar(dst: &mut [u16], tmp: &[u16], mask: &[u8], n: usize) {
for ((d, &t), &m) in dst[..n].iter_mut().zip(&tmp[..n]).zip(&mask[..n]) {
let m = m as i32;
*d = ((*d as i32 * (64 - m) + t as i32 * m + 32) >> 6) as u16;
}
}
static BLEND_HBD: OnceLock<BlendHbdFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_blend_hbd() -> BlendHbdFn {
*BLEND_HBD.get_or_init(|| {
let mut _f = blend_row_hbd_scalar as BlendHbdFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::blend_row_hbd_neon as BlendHbdFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::blend_row_hbd_sse41 as BlendHbdFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::blend_row_hbd_avx2 as BlendHbdFn;
}
}
_f
})
}
pub(crate) type MorphHbdFn = unsafe fn(&mut [u16], i32, i32, usize, i32);
pub(crate) fn morph_row_hbd_scalar(
dst: &mut [u16],
alpha: i32,
beta: i32,
n: usize,
bitdepth_max: i32,
) {
for d in dst[..n].iter_mut() {
*d = ((alpha * *d as i32 + beta) >> 8).clamp(0, bitdepth_max) as u16;
}
}
static MORPH_HBD: OnceLock<MorphHbdFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_morph_hbd() -> MorphHbdFn {
*MORPH_HBD.get_or_init(|| {
let mut _f = morph_row_hbd_scalar as MorphHbdFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::morph_row_hbd_neon as MorphHbdFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::morph_row_hbd_sse41 as MorphHbdFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::morph_row_hbd_avx2 as MorphHbdFn;
}
}
_f
})
}
pub(crate) type GdfAddFn = unsafe fn(&mut [u8], &[i8], i32, usize);
pub(crate) fn gdf_add_run_8bpc_scalar(dst: &mut [u8], err: &[i8], scale: i32, n: usize) {
for (d, &err) in dst[..n].iter_mut().zip(&err[..n]) {
let diff = err as i32 * scale;
let mag = (diff.abs() + 8) >> 4;
let adj = if diff < 0 { -mag } else { mag };
*d = (*d as i32 + adj).clamp(0, 255) as u8;
}
}
static GDF_ADD: OnceLock<GdfAddFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_gdf_add() -> GdfAddFn {
*GDF_ADD.get_or_init(|| {
let mut _f = gdf_add_run_8bpc_scalar as GdfAddFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::gdf_add_run_8bpc_neon as GdfAddFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::gdf_add_run_8bpc_sse41 as GdfAddFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::gdf_add_run_8bpc_avx2 as GdfAddFn;
}
}
_f
})
}
#[inline]
fn gdf_bitdepth_from_max(bitdepth_max: i32) -> i32 {
if bitdepth_max <= 0xff {
8
} else if bitdepth_max <= 0x3ff {
10
} else {
12
}
}
#[inline]
pub(crate) fn gdf_add_run_hbd_scalar(
dst: &mut [u16],
err: &[i8],
scale: i32,
n: usize,
bitdepth_max: i32,
) {
let bitdepth = gdf_bitdepth_from_max(bitdepth_max);
let shift = (12 - bitdepth) as u32;
let rnd = if shift == 0 { 0 } else { 1 << (shift - 1) };
for (d, &err) in dst[..n].iter_mut().zip(&err[..n]) {
let diff = err as i32 * scale;
let mag = (diff.abs() + rnd) >> shift;
let adj = if diff < 0 { -mag } else { mag };
*d = (*d as i32 + adj).clamp(0, bitdepth_max) as u16;
}
}
pub(crate) type GdfAddHbdFn = unsafe fn(&mut [u16], &[i8], i32, usize, i32);
static GDF_ADD_HBD: OnceLock<GdfAddHbdFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_gdf_add_hbd() -> GdfAddHbdFn {
*GDF_ADD_HBD.get_or_init(|| {
let mut _f = gdf_add_run_hbd_scalar as GdfAddHbdFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::gdf_add_run_hbd_neon as GdfAddHbdFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::gdf_add_run_hbd_sse41 as GdfAddHbdFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::gdf_add_run_hbd_avx2 as GdfAddHbdFn;
}
}
_f
})
}
pub(crate) type GdfGradFn = unsafe fn(
&mut [[u16; 4]],
usize,
usize,
usize,
[&[u8]; 2],
[&[u8]; 2],
[&[u8]; 2],
usize,
i32,
u32,
);
pub(crate) type GdfGradHbdFn = unsafe fn(
&mut [[u16; 4]],
usize,
usize,
usize,
[&[u16]; 2],
[&[u16]; 2],
[&[u16]; 2],
usize,
i32,
u32,
);
#[allow(clippy::too_many_arguments)]
pub(crate) fn gdf_gradient_group_scalar(
dst: &mut [[u16; 4]],
d: usize,
base_cell: usize,
ncells: usize,
center_rows: [&[u8]; 2],
a_rows: [&[u8]; 2],
c_rows: [&[u8]; 2],
col0: usize,
dx: i32,
shift: u32,
) {
let mut acc = [0i32; 8];
for y in 0..2 {
let bcol = col0 - 1;
let acol = (bcol as i32 - dx) as usize;
let ccol = (bcol as i32 + dx) as usize;
let brow: &[u8; 8] = center_rows[y][bcol..bcol + 8].try_into().unwrap();
let arow: &[u8; 8] = a_rows[y][acol..acol + 8].try_into().unwrap();
let crow: &[u8; 8] = c_rows[y][ccol..ccol + 8].try_into().unwrap();
for (((acc, &b), &a), &c) in acc.iter_mut().zip(brow).zip(arow).zip(crow) {
let b = (b as i32) >> shift;
let a = (a as i32) >> shift;
let c = (c as i32) >> shift;
*acc += (b + b - a - c).abs();
}
}
for (cell, pair) in dst[base_cell..base_cell + ncells]
.iter_mut()
.zip(acc.as_chunks::<2>().0.iter())
{
cell[d] = (pair[0] + pair[1]) as u16;
}
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gdf_gradient_group_hbd_scalar(
dst: &mut [[u16; 4]],
d: usize,
base_cell: usize,
ncells: usize,
center_rows: [&[u16]; 2],
a_rows: [&[u16]; 2],
c_rows: [&[u16]; 2],
col0: usize,
dx: i32,
shift: u32,
) {
let mut acc = [0i32; 8];
for y in 0..2 {
let bcol = col0 - 1;
let acol = (bcol as i32 - dx) as usize;
let ccol = (bcol as i32 + dx) as usize;
let brow: &[u16; 8] = center_rows[y][bcol..bcol + 8].try_into().unwrap();
let arow: &[u16; 8] = a_rows[y][acol..acol + 8].try_into().unwrap();
let crow: &[u16; 8] = c_rows[y][ccol..ccol + 8].try_into().unwrap();
for (((acc, &b), &a), &c) in acc.iter_mut().zip(brow).zip(arow).zip(crow) {
let b = (b as i32) >> shift;
let a = (a as i32) >> shift;
let c = (c as i32) >> shift;
*acc += (b + b - a - c).abs();
}
}
for (cell, pair) in dst[base_cell..base_cell + ncells]
.iter_mut()
.zip(acc.as_chunks::<2>().0.iter())
{
cell[d] = (pair[0] + pair[1]) as u16;
}
}
static GDF_GRAD: OnceLock<GdfGradFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_gdf_grad() -> GdfGradFn {
*GDF_GRAD.get_or_init(|| {
let mut _f = gdf_gradient_group_scalar as GdfGradFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::gdf_gradient_group_neon as GdfGradFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::gdf_gradient_group_sse41 as GdfGradFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::gdf_gradient_group_avx2 as GdfGradFn;
}
}
_f
})
}
static GDF_GRAD_HBD: OnceLock<GdfGradHbdFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_gdf_grad_hbd() -> GdfGradHbdFn {
*GDF_GRAD_HBD.get_or_init(|| {
let mut _f = gdf_gradient_group_hbd_scalar as GdfGradHbdFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::gdf_gradient_group_hbd_neon as GdfGradHbdFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::gdf_gradient_group_hbd_sse41 as GdfGradHbdFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::gdf_gradient_group_hbd_avx2 as GdfGradHbdFn;
}
}
_f
})
}
#[inline]
fn gdf_prep_apply_sign(v: i32) -> i32 {
if v < 0 {
-((v.wrapping_neg() + (1 << 14)) >> 15)
} else {
(v + (1 << 14)) >> 15
}
}
#[inline]
fn gdf_prep_lookup_error(ref_dst_idx: usize, error_lut_base: usize, full_idx: usize) -> i8 {
if ref_dst_idx == 0 {
GDF_INTRA_ERROR[error_lut_base + full_idx]
} else {
GDF_INTER_ERROR[error_lut_base + full_idx]
}
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gdf_prep_pixel_8bpc(
rows: [&[u8]; 13],
col: usize,
cls: usize,
shared_vals: [i32; 3],
alpha_base: usize,
weight_base: usize,
error_lut_base: usize,
scale: i32,
ref_dst_idx: usize,
) -> i8 {
let mut idx_vals = shared_vals;
let m = rows[6][col] as i32;
for (k, &[dy, dx]) in GDF_PREP_COORDS_8BPC.iter().enumerate() {
let alpha = GDF_ALPHA[alpha_base + k * 4 + cls] as i32;
let a = rows[(6 - dy as i32) as usize][(col as i32 - dx as i32) as usize] as i32;
let b = rows[(6 + dy as i32) as usize][(col as i32 + dx as i32) as usize] as i32;
let above = ((a - m) << 2).clamp(-alpha, alpha);
let below = ((b - m) << 2).clamp(-alpha, alpha);
let v = (above + below).clamp(-512, 511);
for idx in 0..3 {
idx_vals[idx] += v * GDF_WEIGHT[weight_base + idx * 88 + k * 4 + cls] as i32;
}
}
let mut full_idx = 0usize;
for &idx_val in &idx_vals {
let v = gdf_prep_apply_sign(idx_val * scale);
let sub_idx = (v.clamp(-scale, scale - 1) + scale) as usize;
full_idx = full_idx * (scale as usize * 2) + sub_idx;
}
gdf_prep_lookup_error(ref_dst_idx, error_lut_base, full_idx)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gdf_prep_pair_8bpc_scalar(
rows: [&[u8]; 13],
col: usize,
cls: usize,
shared_vals: [i32; 3],
alpha_base: usize,
weight_base: usize,
error_lut_base: usize,
scale: i32,
ref_dst_idx: usize,
) -> [i8; 2] {
let mut out = [0i8; 2];
for (x2, out) in out.iter_mut().enumerate() {
*out = gdf_prep_pixel_8bpc(
rows,
col + x2,
cls,
shared_vals,
alpha_base,
weight_base,
error_lut_base,
scale,
ref_dst_idx,
);
}
out
}
pub(crate) type GdfPrepPair8bpcFn =
unsafe fn([&[u8]; 13], usize, usize, [i32; 3], usize, usize, usize, i32, usize) -> [i8; 2];
static GDF_PREP_PAIR_8BPC: OnceLock<GdfPrepPair8bpcFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_gdf_prep_pair_8bpc() -> GdfPrepPair8bpcFn {
*GDF_PREP_PAIR_8BPC.get_or_init(|| {
let mut _f = gdf_prep_pair_8bpc_scalar as GdfPrepPair8bpcFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::gdf_prep_pair_8bpc_neon as GdfPrepPair8bpcFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::gdf_prep_pair_8bpc_sse41 as GdfPrepPair8bpcFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::gdf_prep_pair_8bpc_avx2 as GdfPrepPair8bpcFn;
}
}
_f
})
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gdf_prep_pair_hbd_scalar(
rows: [&[u16]; 13],
col: usize,
cls: usize,
shared_vals: [i32; 3],
alpha_base: usize,
weight_base: usize,
error_lut_base: usize,
scale: i32,
down_shift: u32,
up_scale: i32,
ref_dst_idx: usize,
) -> [i8; 2] {
let mut out = [0i8; 2];
for x2 in 0..2 {
let x = col + x2;
let mut idx_vals = shared_vals;
let m = (rows[6][x] as i32) >> down_shift;
for (k, &[dy, dx]) in GDF_PREP_COORDS.iter().enumerate() {
let alpha = GDF_ALPHA[alpha_base + k * 4 + cls] as i32;
let a = (rows[(6 - dy as i32) as usize][(x as i32 - dx as i32) as usize] as i32)
>> down_shift;
let b = (rows[(6 + dy as i32) as usize][(x as i32 + dx as i32) as usize] as i32)
>> down_shift;
let above = ((a - m) * up_scale).clamp(-alpha, alpha);
let below = ((b - m) * up_scale).clamp(-alpha, alpha);
let v = (above + below).clamp(-512, 511);
for idx in 0..3 {
idx_vals[idx] += v * GDF_WEIGHT[weight_base + idx * 88 + k * 4 + cls] as i32;
}
}
let mut full_idx = 0usize;
for &idx_val in &idx_vals {
let v = gdf_prep_apply_sign(idx_val * scale);
let sub_idx = (v.clamp(-scale, scale - 1) + scale) as usize;
full_idx = full_idx * (scale as usize * 2) + sub_idx;
}
out[x2] = gdf_prep_lookup_error(ref_dst_idx, error_lut_base, full_idx);
}
out
}
pub(crate) type GdfPrepPairHbdFn = unsafe fn(
[&[u16]; 13],
usize,
usize,
[i32; 3],
usize,
usize,
usize,
i32,
u32,
i32,
usize,
) -> [i8; 2];
static GDF_PREP_PAIR_HBD: OnceLock<GdfPrepPairHbdFn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_gdf_prep_pair_hbd() -> GdfPrepPairHbdFn {
*GDF_PREP_PAIR_HBD.get_or_init(|| {
let mut _f = gdf_prep_pair_hbd_scalar as GdfPrepPairHbdFn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::gdf_prep_pair_hbd_neon as GdfPrepPairHbdFn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::gdf_prep_pair_hbd_sse41 as GdfPrepPairHbdFn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::gdf_prep_pair_hbd_avx2 as GdfPrepPairHbdFn;
}
}
_f
})
}
#[derive(Clone, Copy)]
pub(crate) struct GdfPrepRow {
pub(crate) alpha_base: usize,
pub(crate) weight_base: usize,
pub(crate) bias_idx: usize,
pub(crate) error_lut_base: usize,
pub(crate) scale: i32,
pub(crate) ref_dst_idx: usize,
pub(crate) grad_shift: i32,
}
#[inline(always)]
pub(crate) fn gdf_prep_class(
grad: [&[[u16; 4]]; 2],
hx: usize,
p: &GdfPrepRow,
) -> (usize, [i32; 3]) {
let mut grad_sums = [0i32; 4];
for (d, sum) in grad_sums.iter_mut().enumerate() {
*sum = grad[0][hx][d] as i32
+ grad[0][hx + 1][d] as i32
+ grad[1][hx][d] as i32
+ grad[1][hx + 1][d] as i32;
}
let cls = ((grad_sums[0] <= grad_sums[1]) as usize)
| (((grad_sums[2] <= grad_sums[3]) as usize) << 1);
let mut shared = GDF_BIAS[p.bias_idx].map(i32::from);
for (d, &sum) in grad_sums.iter().enumerate() {
let k = d + 18;
let alpha = GDF_ALPHA[p.alpha_base + k * 4 + cls] as i32;
let v = (sum >> p.grad_shift).min(alpha);
for (idx, acc) in shared.iter_mut().enumerate() {
*acc += v * GDF_WEIGHT[p.weight_base + idx * 88 + k * 4 + cls] as i32;
}
}
(cls, shared)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gdf_prep_row_8bpc_scalar(
rows: [&[u8]; 13],
o: usize,
w: usize,
grad: [&[[u16; 4]]; 2],
dst: &mut [i8],
p: &GdfPrepRow,
) {
let pair = resolve_gdf_prep_pair_8bpc();
let mut x1 = 0usize;
while x1 < w {
let (cls, shared) = gdf_prep_class(grad, x1 >> 1, p);
if x1 + 1 < w {
let e = unsafe {
pair(
rows,
o + x1,
cls,
shared,
p.alpha_base,
p.weight_base,
p.error_lut_base,
p.scale,
p.ref_dst_idx,
)
};
dst[x1] = e[0];
dst[x1 + 1] = e[1];
} else {
dst[x1] = gdf_prep_pixel_8bpc(
rows,
o + x1,
cls,
shared,
p.alpha_base,
p.weight_base,
p.error_lut_base,
p.scale,
p.ref_dst_idx,
);
}
x1 += 2;
}
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn gdf_prep_row_hbd_scalar(
rows: [&[u16]; 13],
o: usize,
w: usize,
grad: [&[[u16; 4]]; 2],
dst: &mut [i8],
p: &GdfPrepRow,
down_shift: u32,
up_scale: i32,
) {
let pair = resolve_gdf_prep_pair_hbd();
let mut x1 = 0usize;
while x1 < w {
let (cls, shared) = gdf_prep_class(grad, x1 >> 1, p);
let e = unsafe {
pair(
rows,
o + x1,
cls,
shared,
p.alpha_base,
p.weight_base,
p.error_lut_base,
p.scale,
down_shift,
up_scale,
p.ref_dst_idx,
)
};
dst[x1] = e[0];
if x1 + 1 < w {
dst[x1 + 1] = e[1];
}
x1 += 2;
}
}
pub(crate) type GdfPrepRow8bpcFn =
unsafe fn([&[u8]; 13], usize, usize, [&[[u16; 4]]; 2], &mut [i8], &GdfPrepRow);
pub(crate) type GdfPrepRowHbdFn =
unsafe fn([&[u16]; 13], usize, usize, [&[[u16; 4]]; 2], &mut [i8], &GdfPrepRow, u32, i32);
#[inline]
pub(crate) fn resolve_gdf_prep_row_8bpc() -> GdfPrepRow8bpcFn {
#[cfg(target_arch = "aarch64")]
{
crate::neon::gdf_prep_row_8bpc_neon as GdfPrepRow8bpcFn
}
#[cfg(not(target_arch = "aarch64"))]
{
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
if std::is_x86_feature_detected!("sse4.1") {
return crate::sse::gdf_prep_row_8bpc_sse41 as GdfPrepRow8bpcFn;
}
gdf_prep_row_8bpc_scalar as GdfPrepRow8bpcFn
}
}
#[inline]
pub(crate) fn resolve_gdf_prep_row_hbd() -> GdfPrepRowHbdFn {
#[cfg(target_arch = "aarch64")]
{
crate::neon::gdf_prep_row_hbd_neon as GdfPrepRowHbdFn
}
#[cfg(not(target_arch = "aarch64"))]
{
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
if std::is_x86_feature_detected!("sse4.1") {
return crate::sse::gdf_prep_row_hbd_sse41 as GdfPrepRowHbdFn;
}
gdf_prep_row_hbd_scalar as GdfPrepRowHbdFn
}
}
pub(crate) type CctxI16Fn = unsafe fn(&mut [i16], &mut [i16], i32, i32, usize, i32, i32);
pub(crate) fn cctx_row_i16_scalar(
u: &mut [i16],
v: &mut [i16],
sina: i32,
cosa: i32,
sz: usize,
min: i32,
max: i32,
) {
for (u, v) in u[..sz].iter_mut().zip(&mut v[..sz]) {
let ui = *u as i32;
let vi = *v as i32;
let a = ui * cosa - vi * sina;
let b = ui * sina + vi * cosa;
*u = ((a + 128 - (a < 0) as i32) >> 8).max(min).min(max) as i16;
*v = ((b + 128 - (b < 0) as i32) >> 8).max(min).min(max) as i16;
}
}
static CCTX_I16: OnceLock<CctxI16Fn> = OnceLock::new();
#[inline]
pub(crate) fn resolve_cctx_i16() -> CctxI16Fn {
*CCTX_I16.get_or_init(|| {
let mut _f = cctx_row_i16_scalar as CctxI16Fn;
#[cfg(target_arch = "aarch64")]
{
_f = crate::neon::cctx_row_i16_neon as CctxI16Fn;
}
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
{
if std::is_x86_feature_detected!("sse4.1") {
_f = crate::sse::cctx_row_i16_sse41 as CctxI16Fn;
}
}
#[cfg(all(target_arch = "x86_64", feature = "avx"))]
{
if std::is_x86_feature_detected!("avx2") {
_f = crate::avx::cctx_row_i16_avx2 as CctxI16Fn;
}
}
_f
})
}
#[cfg(test)]
mod gdf_prep_row_tests {
use super::*;
struct Rng(u64);
impl Rng {
fn r(&mut self, n: u32) -> u32 {
self.0 ^= self.0 << 13;
self.0 ^= self.0 >> 7;
self.0 ^= self.0 << 17;
(self.0 >> 16) as u32 % n
}
}
const O: usize = 8;
const STRIDE: usize = 64 + 2 * O;
fn params(rng: &mut Rng, grad_shift: i32) -> GdfPrepRow {
let ref_dst_idx = rng.r(6) as usize;
let qp_idx = rng.r(6) as usize;
let (error_lut_base, scale) = if ref_dst_idx == 0 {
(qp_idx * 4096, 8)
} else {
((ref_dst_idx - 1) * 6000 + qp_idx * 1000, 5)
};
GdfPrepRow {
alpha_base: ref_dst_idx * 528 + qp_idx * 88,
weight_base: ref_dst_idx * 1584 + qp_idx * 264,
bias_idx: ref_dst_idx * 6 + qp_idx,
error_lut_base,
scale,
ref_dst_idx,
grad_shift,
}
}
fn grads(rng: &mut Rng) -> [Vec<[u16; 4]>; 2] {
core::array::from_fn(|_| {
(0..40)
.map(|_| core::array::from_fn(|_| rng.r(2048) as u16))
.collect()
})
}
fn rows<T: Copy>(rng: &mut Rng, max: u32, cast: fn(u32) -> T) -> Vec<Vec<T>> {
(0..13)
.map(|_| {
let mut v = rng.r(max);
(0..STRIDE)
.map(|_| {
v = if rng.r(8) == 0 {
rng.r(max)
} else {
(v + rng.r(9)).saturating_sub(4).min(max - 1)
};
cast(v)
})
.collect()
})
.collect()
}
#[test]
fn row_8bpc_matches_scalar_pairs() {
let mut rng = Rng(0x5eed_1234_abcd);
for _ in 0..300 {
let w = 1 + rng.r(64) as usize;
let p = params(&mut rng, 2);
let g = grads(&mut rng);
let r = rows(&mut rng, 256, |v| v as u8);
let rows: [&[u8]; 13] = core::array::from_fn(|i| &r[i][..]);
let grad = [&g[0][..], &g[1][..]];
let mut want = vec![0i8; w];
for x1 in (0..w).step_by(2) {
let (cls, shared) = gdf_prep_class(grad, x1 >> 1, &p);
let e = gdf_prep_pair_8bpc_scalar(
rows,
O + x1,
cls,
shared,
p.alpha_base,
p.weight_base,
p.error_lut_base,
p.scale,
p.ref_dst_idx,
);
want[x1] = e[0];
if x1 + 1 < w {
want[x1 + 1] = e[1];
}
}
let mut got = vec![0i8; w];
unsafe { resolve_gdf_prep_row_8bpc()(rows, O, w, grad, &mut got, &p) };
assert_eq!(want, got, "w={w} ref={} bias={}", p.ref_dst_idx, p.bias_idx);
}
}
#[test]
fn row_hbd_matches_scalar_pairs() {
let mut rng = Rng(0xbeef_5678_1357);
for _ in 0..300 {
let bitdepth = [10u32, 12][rng.r(2) as usize];
let down_shift = if bitdepth == 12 { 2 } else { 0 };
let w = 1 + rng.r(64) as usize;
let p = params(&mut rng, 4);
let g = grads(&mut rng);
let r = rows(&mut rng, 1 << bitdepth, |v| v as u16);
let rows: [&[u16]; 13] = core::array::from_fn(|i| &r[i][..]);
let grad = [&g[0][..], &g[1][..]];
let mut want = vec![0i8; w];
for x1 in (0..w).step_by(2) {
let (cls, shared) = gdf_prep_class(grad, x1 >> 1, &p);
let e = gdf_prep_pair_hbd_scalar(
rows,
O + x1,
cls,
shared,
p.alpha_base,
p.weight_base,
p.error_lut_base,
p.scale,
down_shift,
1,
p.ref_dst_idx,
);
want[x1] = e[0];
if x1 + 1 < w {
want[x1 + 1] = e[1];
}
}
let mut got = vec![0i8; w];
unsafe { resolve_gdf_prep_row_hbd()(rows, O, w, grad, &mut got, &p, down_shift, 1) };
assert_eq!(
want, got,
"w={w} bitdepth={bitdepth} ref={} bias={}",
p.ref_dst_idx, p.bias_idx
);
}
}
}