use crate::headers::{FhRestorationPlane, PixelLayout};
use crate::intops::{iclip, iclip64to32, imax, imin};
use crate::lf_mask::{Av2Filter, Av2Restoration};
use crate::tables::PC_WIENER_LUT_TO_CLASS;
pub(crate) const LR_HAVE_LEFT: u8 = 1 << 0;
pub(crate) const LR_HAVE_RIGHT: u8 = 1 << 1;
pub(crate) const LR_HAVE_TOP: u8 = 1 << 2;
pub(crate) const LR_HAVE_BOTTOM: u8 = 1 << 3;
pub(crate) const LR_HAVE_TOP_INTEGRATED: u8 = 1 << 4;
pub(crate) const LR_HAVE_BOTTOM_INTEGRATED: u8 = 1 << 5;
pub(crate) static PC_WIENER_NORMALIZER: [u16; 4] = [3739, 3273, 3074, 7];
static MODE_WEIGHTS: [[i16; 3]; 4] = [
[-527, 15325, 321],
[26436, -17705, 17905],
[366, -147, -194],
[202, -267, -179],
];
static MODE_OFFSETS: [i16; 4] = [-547, -21565, -573, -680];
pub(crate) fn get_qval_given_tskip(qstep: i32, tskip: i32, i: usize, bitdepth_min_8: i32) -> i32 {
let shift = bitdepth_min_8.clamp(0, 30) as u32;
let tskip = tskip as i64;
let qstep = (qstep as i64 + ((1i64 << shift) >> 1)) >> shift;
let prod = (tskip * qstep + 128) >> 8;
let qval = MODE_WEIGHTS[i][0] as i64 * (tskip << 5)
+ MODE_WEIGHTS[i][1] as i64 * qstep
+ MODE_WEIGHTS[i][2] as i64 * prod;
let abs_qval = qval.abs();
let qval = if qval < 0 {
-((abs_qval + (1 << 12)) >> 13)
} else {
(abs_qval + (1 << 12)) >> 13
};
iclip64to32(255 * (MODE_OFFSETS[i] as i64 + qval), i32::MIN, i32::MAX)
}
#[inline(always)]
fn get_qval_lut_bin(qval: i32) -> i32 {
let qval = (imax(0, qval).saturating_add(1 << 13)) >> 14;
imin(qval, 255) >> 5
}
pub(crate) fn backup_row_8bpc(
dst: &mut [u8],
o: usize,
src: &[u8],
src_o: usize,
left: &[u8],
left_off: usize,
w: usize,
edge_len: usize,
edges: u8,
) {
if edges & LR_HAVE_LEFT != 0 {
for x in 0..edge_len {
dst[o - edge_len + x] = left[left_off - edge_len + x];
}
} else {
dst[o - edge_len..o].fill(src[src_o]);
}
dst[o..o + w].copy_from_slice(&src[src_o..src_o + w]);
if edges & LR_HAVE_RIGHT != 0 {
dst[o + w..o + w + edge_len].copy_from_slice(&src[src_o + w..src_o + w + edge_len]);
} else {
dst[o + w..o + w + edge_len].fill(src[src_o + w - 1]);
}
}
pub(crate) fn backup_row_lpf_8bpc(
dst: &mut [u8],
o: usize,
src: &[u8],
src_o: usize,
w: usize,
edge_len: usize,
edges: u8,
) {
if edges & LR_HAVE_LEFT != 0 {
for x in 0..edge_len {
dst[o - edge_len + x] = src[src_o - edge_len + x];
}
} else {
dst[o - edge_len..o].fill(src[src_o]);
}
dst[o..o + w].copy_from_slice(&src[src_o..src_o + w]);
if edges & LR_HAVE_RIGHT != 0 {
dst[o + w..o + w + edge_len].copy_from_slice(&src[src_o + w..src_o + w + edge_len]);
} else {
dst[o + w..o + w + edge_len].fill(src[src_o + w - 1]);
}
}
pub(crate) fn compute_gradient_row_8bpc(
exec: &crate::exec_context::ExecContext,
dst: &mut [[u16; 4]],
rows: &[&[u8]],
row_off: usize,
col_off: usize,
w: usize,
shift: u32,
) {
let offs: [[i32; 2]; 4] = [[1, 0], [0, 1], [1, 1], [-1, 1]];
let ncells = (w + 2 + 1) >> 1;
let full_groups = ncells / 4;
for g in 0..full_groups {
let base_cell = g * 4;
let col0 = col_off + base_cell * 2;
for d in 0..4 {
let dy = offs[d][0];
let dx = offs[d][1];
let center_rows = [rows[row_off - 1], rows[row_off]];
let a_rows = [
rows[(row_off as i32 - 1 - dy) as usize],
rows[(row_off as i32 - dy) as usize],
];
let c_rows = [
rows[(row_off as i32 - 1 + dy) as usize],
rows[(row_off as i32 + dy) as usize],
];
crate::filter::gdf_gradient_group_ctx(
exec,
dst,
d,
base_cell,
4,
center_rows,
a_rows,
c_rows,
col0,
dx,
shift,
);
}
}
let mut x1 = (full_groups * 4) * 2;
while x1 < w + 2 {
for d in 0..4 {
let mut grad = 0i32;
for x2 in 0..2usize {
let x = col_off + x1 + x2;
for y in 0..2 {
let dy = offs[d][0];
let dx = offs[d][1];
let ry = row_off + y;
let a = (rows[(ry as i32 - 1 - dy) as usize][(x as i32 - 1 - dx) as usize]
>> shift) as i32;
let b = (rows[ry - 1][x - 1] >> shift) as i32;
let c = (rows[(ry as i32 - 1 + dy) as usize][(x as i32 - 1 + dx) as usize]
>> shift) as i32;
grad += (b * 2 - a - c).abs();
}
}
dst[x1 >> 1][d] = grad as u16;
}
x1 += 2;
}
}
pub(crate) fn compute_gradient_row_hbd(
exec: &crate::exec_context::ExecContext,
dst: &mut [[u16; 4]],
rows: &[&[u16]],
row_off: usize,
col_off: usize,
w: usize,
shift: u32,
) {
let offs: [[i32; 2]; 4] = [[1, 0], [0, 1], [1, 1], [-1, 1]];
let ncells = (w + 2 + 1) >> 1;
let full_groups = ncells / 4;
for g in 0..full_groups {
let base_cell = g * 4;
let col0 = col_off + base_cell * 2;
for d in 0..4 {
let dy = offs[d][0];
let dx = offs[d][1];
let center_rows = [rows[row_off - 1], rows[row_off]];
let a_rows = [
rows[(row_off as i32 - 1 - dy) as usize],
rows[(row_off as i32 - dy) as usize],
];
let c_rows = [
rows[(row_off as i32 - 1 + dy) as usize],
rows[(row_off as i32 + dy) as usize],
];
crate::filter::gdf_gradient_group_hbd_ctx(
exec,
dst,
d,
base_cell,
4,
center_rows,
a_rows,
c_rows,
col0,
dx,
shift,
);
}
}
let mut x1 = (full_groups * 4) * 2;
while x1 < w + 2 {
for d in 0..4 {
let mut grad = 0i32;
for x2 in 0..2usize {
let x = col_off + x1 + x2;
for y in 0..2 {
let dy = offs[d][0];
let dx = offs[d][1];
let ry = row_off + y;
let a = (rows[(ry as i32 - 1 - dy) as usize][(x as i32 - 1 - dx) as usize]
as i32)
>> shift;
let b = (rows[ry - 1][x - 1] as i32) >> shift;
let c = (rows[(ry as i32 - 1 + dy) as usize][(x as i32 - 1 + dx) as usize]
as i32)
>> shift;
grad += (b * 2 - a - c).abs();
}
}
dst[x1 >> 1][d] = grad as u16;
}
x1 += 2;
}
}
#[inline]
fn gdf_bitdepth_from_max(bitdepth_max: i32) -> i32 {
if bitdepth_max <= 0xff {
8
} else if bitdepth_max <= 0x3ff {
10
} else {
12
}
}
pub(crate) trait PcwPx: Copy + Into<i32> {
fn simd_features(rows: &[&[Self]], row_center: usize, x0: usize) -> Option<[i32; 3]>;
}
impl PcwPx for u8 {
#[inline(always)]
#[allow(unreachable_code, unused_variables)]
fn simd_features(rows: &[&[u8]], row_center: usize, x0: usize) -> Option<[i32; 3]> {
#[cfg(target_arch = "aarch64")]
return Some(unsafe { crate::neon::pcw_features_8bpc_neon(rows, row_center, x0) });
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
if std::is_x86_feature_detected!("sse4.1") {
return Some(unsafe { crate::sse::pcw_features_8bpc_sse41(rows, row_center, x0) });
}
None
}
}
impl PcwPx for u16 {
#[inline(always)]
#[allow(unreachable_code, unused_variables)]
fn simd_features(rows: &[&[u16]], row_center: usize, x0: usize) -> Option<[i32; 3]> {
#[cfg(target_arch = "aarch64")]
return Some(unsafe { crate::neon::pcw_features_hbd_neon(rows, row_center, x0) });
#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), feature = "sse"))]
if std::is_x86_feature_detected!("sse4.1") {
return Some(unsafe { crate::sse::pcw_features_hbd_sse41(rows, row_center, x0) });
}
None
}
}
pub(crate) fn pcw_features_scalar<P: PcwPx>(
rows: &[&[P]],
row_center: usize,
x0: usize,
) -> [i32; 3] {
let mut f = [0i32; 3];
for y in row_center - 1..row_center + 5 {
let (up, mid, down) = (rows[y - 1], rows[y], rows[y + 1]);
for x in x0..x0 + 6 {
let px = |r: &[P], i: usize| -> i32 { r[i].into() };
let m2 = 2 * px(mid, x);
f[0] += (px(up, x) - m2 + px(down, x)).abs();
f[1] += (px(up, x + 1) - m2 + px(down, x - 1)).abs();
f[2] += (px(up, x - 1) - m2 + px(down, x + 1)).abs();
}
}
f
}
#[inline]
fn pcw_features<P: PcwPx>(rows: &[&[P]], row_center: usize, x0: usize) -> [i32; 3] {
let fits = x0 >= 1
&& rows[row_center - 2..row_center + 6]
.iter()
.all(|r| x0 + 9 <= r.len());
if fits && let Some(f) = P::simd_features(rows, row_center, x0) {
return f;
}
pcw_features_scalar(rows, row_center, x0)
}
pub(crate) fn get_class_lut_idx_8bpc(
rows: &[&[u8]],
row_center: usize,
col_off: usize,
noskip_mask: &[u16],
base_q: i32,
bx: usize,
by: usize,
bh: usize,
) -> i32 {
let mut f = pcw_features(rows, row_center, col_off + bx * 4 - 1);
let mut s = 0i32;
let num_pixels: [u8; 3] = [16, 4, 1];
for dy in -1i32..=1 {
for dx in -1i32..=1 {
let edge = (dy != 0) as usize + (dx != 0) as usize;
let fx = iclip((bx & 15) as i32 + dx, 0, 15) as usize;
let fy = iclip(by as i32 + dy, 0, bh as i32 - 1) as usize;
s += num_pixels[edge] as i32 * (((noskip_mask[fy] >> fx) & 1) == 0) as i32;
}
}
for i in 0..3 {
f[i] *= PC_WIENER_NORMALIZER[i] as i32;
}
s *= PC_WIENER_NORMALIZER[3] as i32;
let mut qval = get_qval_lut_bin(get_qval_given_tskip(base_q, s, 0, 0));
let mut lut_idx = qval << 9;
for i in 0..3 {
qval = get_qval_lut_bin(f[i].saturating_add(get_qval_given_tskip(base_q, s, i + 1, 0)));
lut_idx |= qval << (3 * (2 - i));
}
lut_idx
}
pub(crate) fn gdf_add_8bpc(
exec: &crate::exec_context::ExecContext,
p: &mut [u8],
stride: usize,
err: &[i8],
err_stride: usize,
w: usize,
h: usize,
scale: i32,
ll_mask: &[[u16; 4]],
) {
let bw = w >> 2;
for by in 0..h >> 2 {
let skip_mask = ll_mask[by][0];
let mut bx = 0usize;
while bx < bw {
if skip_mask & (1 << bx) != 0 {
bx += 1;
continue;
}
let bx_start = bx;
bx += 1;
while bx < bw && skip_mask & (1 << bx) == 0 {
bx += 1;
}
let x0 = bx_start << 2;
let n = (bx - bx_start) << 2;
for y in by * 4..by * 4 + 4 {
crate::filter::gdf_add_run_ctx(
exec,
&mut p[y * stride + x0..],
&err[y * err_stride + x0..],
scale,
n,
);
}
}
}
}
pub(crate) fn gdf_add_hbd(
exec: &crate::exec_context::ExecContext,
p: &mut [u16],
stride: usize,
err: &[i8],
err_stride: usize,
w: usize,
h: usize,
scale: i32,
ll_mask: &[[u16; 4]],
bitdepth_max: i32,
) {
let bw = w >> 2;
for by in 0..h >> 2 {
let skip_mask = ll_mask[by][0];
let mut bx = 0usize;
while bx < bw {
if skip_mask & (1 << bx) != 0 {
bx += 1;
continue;
}
let bx_start = bx;
bx += 1;
while bx < bw && skip_mask & (1 << bx) == 0 {
bx += 1;
}
let x0 = bx_start << 2;
let n = (bx - bx_start) << 2;
for y in by * 4..by * 4 + 4 {
crate::filter::gdf_add_run_hbd_ctx(
exec,
&mut p[y * stride + x0..],
&err[y * err_stride + x0..],
scale,
n,
bitdepth_max,
);
}
}
}
}
pub(crate) const REST_UNIT_STRIDE: usize = 76;
const ROW_ORIGIN: usize = 6;
const GRADIENT_BUF_STRIDE: usize = 33;
pub(crate) static WIENER_NS_CONFIG_UV: [[i8; 2]; 6] =
[[1, 0], [0, 1], [1, 1], [-1, 1], [2, 0], [0, 2]];
pub(crate) static WIENER_NS_CONFIG_UV_FROM_Y: [[i8; 2]; 12] = [
[1, 0],
[-1, 0],
[0, 1],
[0, -1],
[1, 1],
[-1, -1],
[-1, 1],
[1, -1],
[2, 0],
[-2, 0],
[0, 2],
[0, -2],
];
pub(crate) static PC_WIENER_CONFIG: [[i8; 2]; 12] = [
[1, 0],
[0, 1],
[2, 0],
[0, 2],
[1, 1],
[-1, 1],
[2, 1],
[2, -1],
[1, 2],
[1, -2],
[3, 0],
[0, 3],
];
pub(crate) static WIENER_NS_CONFIG_Y: [[i8; 2]; 16] = [
[1, 0],
[0, 1],
[2, 0],
[0, 2],
[1, 1],
[-1, 1],
[2, 1],
[2, -1],
[1, 2],
[1, -2],
[3, 0],
[0, 3],
[4, 0],
[0, 4],
[3, 3],
[3, -3],
];
pub(crate) fn backup_row_luma_8bpc(
dst: &mut [u8],
o: usize,
src: &[u8],
src_o: usize,
src_stride: usize,
w: usize,
edges: u8,
ss_hor: usize,
ss_ver: usize,
cfl_ds_flt: i32,
) {
if ss_ver == 0 {
backup_row_lpf_8bpc(dst, o, src, src_o, w, 4, edges);
if ss_hor != 0 && edges & LR_HAVE_RIGHT == 0 {
let fill_val = src[src_o + w - 2];
dst[o + w..o + w + 4].fill(fill_val);
}
return;
}
let src2_o = src_o + src_stride;
match cfl_ds_flt {
0 => {
let mut x = 0;
while x < w {
dst[o + x] = ((src[src_o + x] as u16
+ src[src2_o + x] as u16
+ src[src_o + x + 1] as u16
+ src[src2_o + x + 1] as u16)
>> 2) as u8;
x += 1 + ss_hor;
}
}
1 => {
for x in 0..w {
dst[o + x] = ((src[src_o + x] as u16 + src[src2_o + x] as u16) >> 1) as u8;
}
}
_ => {
dst[o..o + w].copy_from_slice(&src[src_o..src_o + w]);
}
}
if edges & LR_HAVE_LEFT != 0 {
match cfl_ds_flt {
0 => {
for i in 0..4usize {
let si = src_o - 4 + i;
let s2i = src2_o - 4 + i;
dst[o - 4 + i] = ((src[si] as u16
+ src[s2i] as u16
+ src[si + 1] as u16
+ src[s2i + 1] as u16)
>> 2) as u8;
}
}
1 => {
for i in 0..4usize {
dst[o - 4 + i] =
((src[src_o - 4 + i] as u16 + src[src2_o - 4 + i] as u16) >> 1) as u8;
}
}
_ => {
for i in 0..4usize {
dst[o - 4 + i] = src[src_o - 4 + i];
}
}
}
} else {
let fill_val = dst[o];
dst[o - 4..o].fill(fill_val);
}
if edges & LR_HAVE_RIGHT != 0 {
match cfl_ds_flt {
0 => {
for i in 0..4usize {
let si = src_o + w + i;
let s2i = src2_o + w + i;
dst[o + w + i] = ((src[si] as u16
+ src[s2i] as u16
+ src[si + 1] as u16
+ src[s2i + 1] as u16)
>> 2) as u8;
}
}
1 => {
for i in 0..4usize {
dst[o + w + i] =
((src[src_o + w + i] as u16 + src[src2_o + w + i] as u16) >> 1) as u8;
}
}
_ => {
for i in 0..4usize {
dst[o + w + i] = src[src_o + w + i];
}
}
}
} else {
let fill_val = dst[o + w - 2];
dst[o + w..o + w + 4].fill(fill_val);
}
}
pub(crate) fn ns_wiener_single_y_8bpc(
exec: &crate::exec_context::ExecContext,
p: &mut [u8],
p_off: usize,
stride: usize,
left: &[[u8; 6]],
lpf: &[u8],
lpf_off: usize,
lpf_bottom: &[u8],
lpf_bottom_off: usize,
w: usize,
h: usize,
filter: &[i8; 16],
edges: u8,
ll_mask: &[[u16; 4]],
) {
let mut row_buffers = [[0u8; REST_UNIT_STRIDE]; 9];
let mut ptrs: [usize; 9] = [0; 9];
let o = ROW_ORIGIN;
backup_row_8bpc(&mut row_buffers[4], o, &*p, p_off, &left[0], 6, w, 4, edges);
ptrs[4] = 4;
if edges & LR_HAVE_TOP_INTEGRATED != 0 {
let mut loff = lpf_off;
for i in 0..4 {
backup_row_lpf_8bpc(&mut row_buffers[i], o, lpf, loff, w, 4, edges);
loff += stride;
ptrs[i] = i;
}
} else if edges & LR_HAVE_TOP != 0 {
backup_row_lpf_8bpc(&mut row_buffers[2], o, lpf, lpf_off, w, 4, edges);
ptrs[2] = 2;
backup_row_lpf_8bpc(&mut row_buffers[3], o, lpf, lpf_off + stride, w, 4, edges);
ptrs[3] = 3;
ptrs[0] = 2;
ptrs[1] = 2;
} else {
ptrs[0] = 4;
ptrs[1] = 4;
ptrs[2] = 4;
ptrs[3] = 4;
}
backup_row_8bpc(
&mut row_buffers[5],
o,
&*p,
p_off + stride,
&left[1],
6,
w,
4,
edges,
);
ptrs[5] = 5;
backup_row_8bpc(
&mut row_buffers[6],
o,
&*p,
p_off + 2 * stride,
&left[2],
6,
w,
4,
edges,
);
ptrs[6] = 6;
backup_row_8bpc(
&mut row_buffers[7],
o,
&*p,
p_off + 3 * stride,
&left[3],
6,
w,
4,
edges,
);
ptrs[7] = 7;
let mut bak_idx: usize = 8;
for y in 0..h {
if y + 4 < h {
backup_row_8bpc(
&mut row_buffers[bak_idx],
o,
&*p,
p_off + (y + 4) * stride,
&left[y + 4],
6,
w,
4,
edges,
);
ptrs[8] = bak_idx;
} else if edges & LR_HAVE_BOTTOM_INTEGRATED != 0 {
backup_row_lpf_8bpc(
&mut row_buffers[bak_idx],
o,
&*p,
p_off + (y + 4) * stride,
w,
4,
edges,
);
ptrs[8] = bak_idx;
} else if y + 2 < h && edges & LR_HAVE_BOTTOM != 0 {
let offset_y = y + 4 - h;
backup_row_lpf_8bpc(
&mut row_buffers[bak_idx],
o,
lpf_bottom,
lpf_bottom_off + offset_y * stride,
w,
4,
edges,
);
ptrs[8] = bak_idx;
} else {
ptrs[8] = ptrs[7];
}
bak_idx += 1;
if bak_idx == 9 {
bak_idx = 0;
}
let refs: [&[u8]; 9] = core::array::from_fn(|i| &row_buffers[ptrs[i]] as &[u8]);
let taps: [crate::filter::WienerTap; 16] = core::array::from_fn(|i| {
let dy = WIENER_NS_CONFIG_Y[i][0] as i32;
let dx = WIENER_NS_CONFIG_Y[i][1] as i32;
crate::filter::WienerTap {
row_p: refs[(4 + dy) as usize],
row_m: refs[(4 - dy) as usize],
dx,
coef: filter[i] as i32,
}
});
let skip_mask = ll_mask[y >> 2][0];
let dst_row = &mut p[p_off + y * stride..];
let bw = w >> 2;
let mut bx = 0usize;
while bx < bw {
if skip_mask & (1 << bx) != 0 {
bx += 1;
continue;
}
let bx_start = bx;
bx += 1;
while bx < bw && skip_mask & (1 << bx) == 0 {
bx += 1;
}
let x0 = bx_start << 2;
let n = (bx - bx_start) << 2;
unsafe {
(exec.ns_wiener_fir)(&mut dst_row[x0..x0 + n], refs[4], o + x0, &taps, n);
}
}
for r in 0..8 {
ptrs[r] = ptrs[r + 1];
}
}
}
fn wiener_multi_8bpc(
exec: &crate::exec_context::ExecContext,
p: &mut [u8],
p_off: usize,
stride: usize,
left: &[[u8; 6]],
lpf: &[u8],
lpf_off: usize,
lpf_bottom: &[u8],
lpf_bottom_off: usize,
w: usize,
h: usize,
filters_user: Option<&[[i8; 18]]>,
filters_pretrained: Option<&[[i16; 13]]>,
subclass_lut: &[u8],
noskip_mask: &[u16],
base_q: i32,
edges: u8,
ll_mask: &[[u16; 4]],
) {
let mut classes = [0u8; 16];
let mut row_buffers = [[0u8; REST_UNIT_STRIDE]; 10];
let mut ptrs: [usize; 10] = [0; 10];
let o = ROW_ORIGIN;
backup_row_8bpc(&mut row_buffers[4], o, &*p, p_off, &left[0], 6, w, 4, edges);
ptrs[4] = 4;
if edges & LR_HAVE_TOP_INTEGRATED != 0 {
let mut loff = lpf_off;
for i in 0..4 {
backup_row_lpf_8bpc(&mut row_buffers[i], o, lpf, loff, w, 4, edges);
loff += stride;
ptrs[i] = i;
}
} else if edges & LR_HAVE_TOP != 0 {
backup_row_lpf_8bpc(&mut row_buffers[2], o, lpf, lpf_off, w, 4, edges);
ptrs[2] = 2;
backup_row_lpf_8bpc(&mut row_buffers[3], o, lpf, lpf_off + stride, w, 4, edges);
ptrs[3] = 3;
ptrs[0] = 2;
ptrs[1] = 2;
} else {
ptrs[0] = 4;
ptrs[1] = 4;
ptrs[2] = 4;
ptrs[3] = 4;
}
backup_row_8bpc(
&mut row_buffers[5],
o,
&*p,
p_off + stride,
&left[1],
6,
w,
4,
edges,
);
ptrs[5] = 5;
backup_row_8bpc(
&mut row_buffers[6],
o,
&*p,
p_off + 2 * stride,
&left[2],
6,
w,
4,
edges,
);
ptrs[6] = 6;
backup_row_8bpc(
&mut row_buffers[7],
o,
&*p,
p_off + 3 * stride,
&left[3],
6,
w,
4,
edges,
);
ptrs[7] = 7;
let mut bak_idx: usize = 8;
let bh = h >> 2;
let bw = w >> 2;
for by in 0..bh {
let by4 = by << 2;
if by + 1 < bh {
backup_row_8bpc(
&mut row_buffers[bak_idx],
o,
&*p,
p_off + (by4 + 4) * stride,
&left[by4 + 4],
6,
w,
4,
edges,
);
ptrs[8] = bak_idx;
backup_row_8bpc(
&mut row_buffers[9],
o,
&*p,
p_off + (by4 + 5) * stride,
&left[by4 + 5],
6,
w,
4,
edges,
);
ptrs[9] = 9;
} else if edges & LR_HAVE_BOTTOM_INTEGRATED != 0 {
backup_row_lpf_8bpc(
&mut row_buffers[bak_idx],
o,
&*p,
p_off + (by4 + 4) * stride,
w,
4,
edges,
);
ptrs[8] = bak_idx;
backup_row_lpf_8bpc(
&mut row_buffers[9],
o,
&*p,
p_off + (by4 + 5) * stride,
w,
4,
edges,
);
ptrs[9] = 9;
} else if edges & LR_HAVE_BOTTOM != 0 {
backup_row_lpf_8bpc(
&mut row_buffers[bak_idx],
o,
lpf_bottom,
lpf_bottom_off,
w,
4,
edges,
);
ptrs[8] = bak_idx;
backup_row_lpf_8bpc(
&mut row_buffers[9],
o,
lpf_bottom,
lpf_bottom_off + stride,
w,
4,
edges,
);
ptrs[9] = 9;
} else {
ptrs[8] = ptrs[7];
ptrs[9] = ptrs[7];
}
{
let refs: [&[u8]; 10] = core::array::from_fn(|i| &row_buffers[ptrs[i]] as &[u8]);
for bx in 0..bw {
let lut_idx = get_class_lut_idx_8bpc(&refs, 4, o, noskip_mask, base_q, bx, by, bh);
let cls = PC_WIENER_LUT_TO_CLASS[lut_idx as usize];
classes[bx] = subclass_lut[cls as usize];
}
}
for y in by4..by4 + 4 {
if y + 4 < h {
backup_row_8bpc(
&mut row_buffers[bak_idx],
o,
&*p,
p_off + (y + 4) * stride,
&left[y + 4],
6,
w,
4,
edges,
);
ptrs[8] = bak_idx;
} else if edges & LR_HAVE_BOTTOM_INTEGRATED != 0 {
backup_row_lpf_8bpc(
&mut row_buffers[bak_idx],
o,
&*p,
p_off + (y + 4) * stride,
w,
4,
edges,
);
ptrs[8] = bak_idx;
} else if y + 2 < h && edges & LR_HAVE_BOTTOM != 0 {
let offset_y = y + 4 - h;
backup_row_lpf_8bpc(
&mut row_buffers[bak_idx],
o,
lpf_bottom,
lpf_bottom_off + offset_y * stride,
w,
4,
edges,
);
ptrs[8] = bak_idx;
} else {
ptrs[8] = ptrs[7];
}
bak_idx += 1;
if bak_idx == 9 {
bak_idx = 0;
}
let refs: [&[u8]; 10] = core::array::from_fn(|i| &row_buffers[ptrs[i]] as &[u8]);
let skip_mask = ll_mask[y >> 2][0];
let dst_row = &mut p[p_off + y * stride..];
let mut bx = 0usize;
while bx < bw {
if skip_mask & (1 << bx) != 0 {
bx += 1;
continue;
}
let cls = classes[bx];
let bx_start = bx;
bx += 1;
while bx < bw && skip_mask & (1 << bx) == 0 && classes[bx] == cls {
bx += 1;
}
let x0 = bx_start << 2;
let n = (bx - bx_start) << 2;
let col0 = o + x0;
if let Some(fu) = filters_user {
let filter = &fu[cls as usize];
let taps: [crate::filter::WienerTap; 16] = core::array::from_fn(|i| {
let dy = WIENER_NS_CONFIG_Y[i][0] as i32;
let dx = WIENER_NS_CONFIG_Y[i][1] as i32;
crate::filter::WienerTap {
row_p: refs[(4 + dy) as usize],
row_m: refs[(4 - dy) as usize],
dx,
coef: filter[i] as i32,
}
});
unsafe {
(exec.ns_wiener_fir)(&mut dst_row[x0..x0 + n], refs[4], col0, &taps, n);
}
} else if let Some(fp) = filters_pretrained {
let filter = &fp[cls as usize];
let taps: [crate::filter::WienerTap; 12] = core::array::from_fn(|i| {
let dy = PC_WIENER_CONFIG[i][0] as i32;
let dx = PC_WIENER_CONFIG[i][1] as i32;
crate::filter::WienerTap {
row_p: refs[(4 + dy) as usize],
row_m: refs[(4 - dy) as usize],
dx,
coef: filter[i] as i32,
}
});
unsafe {
(exec.pc_wiener_fir)(
&mut dst_row[x0..x0 + n],
refs[4],
filter[12] as i32,
col0,
&taps,
n,
);
}
}
}
for r in 0..8 {
ptrs[r] = ptrs[r + 1];
}
}
}
}
pub(crate) fn ns_wiener_multi_8bpc(
exec: &crate::exec_context::ExecContext,
p: &mut [u8],
p_off: usize,
stride: usize,
left: &[[u8; 6]],
lpf: &[u8],
lpf_off: usize,
lpf_bottom: &[u8],
lpf_bottom_off: usize,
w: usize,
h: usize,
filters_user: &[[i8; 18]],
subclass_lut: &[u8],
noskip_mask: &[u16],
base_q: i32,
edges: u8,
ll_mask: &[[u16; 4]],
) {
wiener_multi_8bpc(
exec,
p,
p_off,
stride,
left,
lpf,
lpf_off,
lpf_bottom,
lpf_bottom_off,
w,
h,
Some(filters_user),
None,
subclass_lut,
noskip_mask,
base_q,
edges,
ll_mask,
);
}
pub(crate) fn pc_wiener_8bpc(
exec: &crate::exec_context::ExecContext,
p: &mut [u8],
p_off: usize,
stride: usize,
left: &[[u8; 6]],
lpf: &[u8],
lpf_off: usize,
lpf_bottom: &[u8],
lpf_bottom_off: usize,
w: usize,
h: usize,
filters_pretrained: &[[i16; 13]],
subclass_lut: &[u8],
noskip_mask: &[u16],
base_q: i32,
edges: u8,
ll_mask: &[[u16; 4]],
) {
wiener_multi_8bpc(
exec,
p,
p_off,
stride,
left,
lpf,
lpf_off,
lpf_bottom,
lpf_bottom_off,
w,
h,
None,
Some(filters_pretrained),
subclass_lut,
noskip_mask,
base_q,
edges,
ll_mask,
);
}
const LUMA_BUF_STRIDE: usize = REST_UNIT_STRIDE + 64;
pub(crate) fn ns_wiener_single_uv_8bpc(
exec: &crate::exec_context::ExecContext,
p: &mut [u8],
p_off: usize,
stride: usize,
left: &[[u8; 6]],
lpf: &[u8],
lpf_off: usize,
lpf_bottom: &[u8],
lpf_bottom_off: usize,
w: usize,
h: usize,
filter: &[i8; 18],
luma: &[u8],
luma_off: usize,
lstride: usize,
luma_top: &[u8],
luma_top_off: usize,
luma_bottom: &[u8],
luma_bottom_off: usize,
ss_hor: usize,
ss_ver: usize,
ds_flt: i32,
edges: u8,
ll_mask: &[[u16; 4]],
) {
let mut c_buffers = [[0u8; REST_UNIT_STRIDE]; 5];
let mut l_buffers = [[0u8; LUMA_BUF_STRIDE]; 5];
let mut c_ptrs: [usize; 5] = [0; 5];
let mut l_ptrs: [usize; 5] = [0; 5];
let o = ROW_ORIGIN;
let luma_w = w << ss_hor;
backup_row_8bpc(&mut c_buffers[2], o, &*p, p_off, &left[0], 6, w, 2, edges);
c_ptrs[2] = 2;
if edges & (LR_HAVE_TOP_INTEGRATED | LR_HAVE_TOP) != 0 {
let mut loff = lpf_off;
for i in 0..2 {
backup_row_lpf_8bpc(&mut c_buffers[i], o, lpf, loff, w, 2, edges);
c_ptrs[i] = i;
loff += stride;
}
} else {
c_ptrs[0] = 2;
c_ptrs[1] = 2;
}
backup_row_8bpc(
&mut c_buffers[3],
o,
&*p,
p_off + stride,
&left[1],
6,
w,
2,
edges,
);
c_ptrs[3] = 3;
let mut bak_idx: usize = 4;
backup_row_luma_8bpc(
&mut l_buffers[2],
o,
luma,
luma_off,
lstride,
luma_w,
edges,
ss_hor,
ss_ver,
ds_flt,
);
l_ptrs[2] = 2;
if edges & LR_HAVE_TOP_INTEGRATED != 0 {
backup_row_luma_8bpc(
&mut l_buffers[0],
o,
luma,
luma_off - (2 << ss_ver) * lstride,
lstride,
luma_w,
edges,
ss_hor,
ss_ver,
ds_flt,
);
l_ptrs[0] = 0;
backup_row_luma_8bpc(
&mut l_buffers[1],
o,
luma,
luma_off - (1 << ss_ver) * lstride,
lstride,
luma_w,
edges,
ss_hor,
ss_ver,
ds_flt,
);
l_ptrs[1] = 1;
} else if edges & LR_HAVE_TOP != 0 {
backup_row_luma_8bpc(
&mut l_buffers[0],
o,
luma_top,
luma_top_off,
0,
luma_w,
edges,
ss_hor,
ss_ver,
ds_flt,
);
l_ptrs[0] = 0;
backup_row_luma_8bpc(
&mut l_buffers[1],
o,
luma_top,
luma_top_off + if ss_ver == 0 { lstride } else { 0 },
lstride,
luma_w,
edges,
ss_hor,
ss_ver,
ds_flt,
);
l_ptrs[1] = 1;
} else {
l_ptrs[0] = 2;
l_ptrs[1] = 2;
}
backup_row_luma_8bpc(
&mut l_buffers[3],
o,
luma,
luma_off + (1 << ss_ver) * lstride,
lstride,
luma_w,
edges,
ss_hor,
ss_ver,
ds_flt,
);
l_ptrs[3] = 3;
let mut lbak_idx: usize = 4;
let mut luma_pos = luma_off;
for y in 0..h {
if y + 2 < h {
backup_row_8bpc(
&mut c_buffers[bak_idx],
o,
&*p,
p_off + (y + 2) * stride,
&left[y + 2],
6,
w,
2,
edges,
);
c_ptrs[4] = bak_idx;
} else if edges & LR_HAVE_BOTTOM_INTEGRATED != 0 {
backup_row_lpf_8bpc(
&mut c_buffers[bak_idx],
o,
&*p,
p_off + (y + 2) * stride,
w,
2,
edges,
);
c_ptrs[4] = bak_idx;
} else if edges & LR_HAVE_BOTTOM != 0 {
let offset_y = y + 2 - h;
backup_row_lpf_8bpc(
&mut c_buffers[bak_idx],
o,
lpf_bottom,
lpf_bottom_off + offset_y * stride,
w,
2,
edges,
);
c_ptrs[4] = bak_idx;
} else {
c_ptrs[4] = c_ptrs[3];
}
bak_idx += 1;
if bak_idx == 5 {
bak_idx = 0;
}
if c_ptrs[4] == c_ptrs[3] {
l_ptrs[4] = l_ptrs[3];
} else if y + 2 == h && edges & LR_HAVE_BOTTOM_INTEGRATED == 0 {
backup_row_luma_8bpc(
&mut l_buffers[lbak_idx],
o,
luma_bottom,
luma_bottom_off,
lstride,
luma_w,
edges,
ss_hor,
ss_ver,
ds_flt,
);
l_ptrs[4] = lbak_idx;
} else if y + 1 == h && edges & LR_HAVE_BOTTOM_INTEGRATED == 0 {
backup_row_luma_8bpc(
&mut l_buffers[lbak_idx],
o,
luma_bottom,
luma_bottom_off + lstride,
0,
luma_w,
edges,
ss_hor,
ss_ver,
ds_flt,
);
l_ptrs[4] = lbak_idx;
} else {
backup_row_luma_8bpc(
&mut l_buffers[lbak_idx],
o,
luma,
luma_pos + (2 << ss_ver) * lstride,
lstride,
luma_w,
edges,
ss_hor,
ss_ver,
ds_flt,
);
l_ptrs[4] = lbak_idx;
}
lbak_idx += 1;
if lbak_idx == 5 {
lbak_idx = 0;
}
let c_refs: [&[u8]; 5] = core::array::from_fn(|i| &c_buffers[c_ptrs[i]] as &[u8]);
let l_refs: [&[u8]; 5] = core::array::from_fn(|i| &l_buffers[l_ptrs[i]] as &[u8]);
let lstep = 1usize << ss_hor;
let ctaps: [crate::filter::WienerTap; 6] = core::array::from_fn(|i| {
let dy = WIENER_NS_CONFIG_UV[i][0] as i32;
let dx = WIENER_NS_CONFIG_UV[i][1] as i32;
crate::filter::WienerTap {
row_p: c_refs[(2 + dy) as usize],
row_m: c_refs[(2 - dy) as usize],
dx,
coef: filter[i] as i32,
}
});
let ltaps: [crate::filter::UvLumaTap; 12] = core::array::from_fn(|i| {
let dy = WIENER_NS_CONFIG_UV_FROM_Y[i][0] as i32;
let dx = WIENER_NS_CONFIG_UV_FROM_Y[i][1] as i32;
crate::filter::UvLumaTap {
row: l_refs[(2 + dy) as usize],
ldx: dx * lstep as i32,
coef: filter[6 + i] as i32,
}
});
let skip_mask = ll_mask[y >> 2][0];
let dst_row = &mut p[p_off + y * stride..];
let bw = w >> 2;
let mut bx = 0usize;
while bx < bw {
if skip_mask & (1 << bx) != 0 {
bx += 1;
continue;
}
let bx_start = bx;
bx += 1;
while bx < bw && skip_mask & (1 << bx) == 0 {
bx += 1;
}
let x0 = bx_start << 2;
let n = (bx - bx_start) << 2;
unsafe {
(exec.ns_wiener_uv_fir)(
&mut dst_row[x0..x0 + n],
c_refs[2],
o + x0,
&ctaps,
l_refs[2],
o + (x0 << ss_hor),
<aps,
lstep,
n,
);
}
}
for r in 0..4 {
c_ptrs[r] = c_ptrs[r + 1];
}
for r in 0..4 {
l_ptrs[r] = l_ptrs[r + 1];
}
luma_pos += lstride << ss_ver;
}
}
pub(crate) fn gdf_prep_8bpc(
exec: &crate::exec_context::ExecContext,
dst: &mut [i8],
dst_stride: usize,
p: &[u8],
p_off: usize,
stride: usize,
left: &[[u8; 6]],
lpf: &[u8],
lpf_off: usize,
lpf_bottom: &[u8],
lpf_bottom_off: usize,
w: usize,
h: usize,
ref_dst_idx: usize,
qp_idx: usize,
edges: u8,
) {
let mut row_buffers = [[0u8; REST_UNIT_STRIDE]; 13];
let mut ptrs: [usize; 13] = [0; 13];
let o = ROW_ORIGIN;
backup_row_8bpc(&mut row_buffers[6], o, p, p_off, &left[0], 6, w, 6, edges);
ptrs[6] = 6;
if edges & LR_HAVE_TOP_INTEGRATED != 0 {
for n in 0..6 {
backup_row_lpf_8bpc(
&mut row_buffers[n],
o,
lpf,
lpf_off + n * stride,
w,
6,
edges,
);
ptrs[n] = n;
}
} else if edges & LR_HAVE_TOP != 0 {
backup_row_lpf_8bpc(&mut row_buffers[4], o, lpf, lpf_off, w, 6, edges);
ptrs[4] = 4;
backup_row_lpf_8bpc(&mut row_buffers[5], o, lpf, lpf_off + stride, w, 6, edges);
ptrs[5] = 5;
ptrs[0] = 4;
ptrs[1] = 4;
ptrs[2] = 4;
ptrs[3] = 4;
} else {
for n in 0..6 {
ptrs[n] = 6;
}
}
let mut bak_idx = 7usize;
for y in 1..6 {
backup_row_8bpc(
&mut row_buffers[bak_idx],
o,
p,
p_off + y * stride,
&left[y],
6,
w,
6,
edges,
);
ptrs[bak_idx] = bak_idx;
bak_idx += 1;
}
let alpha_base = ref_dst_idx * 528 + qp_idx * 88;
let weight_base = ref_dst_idx * 1584 + qp_idx * 264;
let bias_idx = ref_dst_idx * 6 + qp_idx;
let (error_lut_base, scale) = if ref_dst_idx == 0 {
(qp_idx * 4096, 8i32)
} else {
((ref_dst_idx - 1) * 6000 + qp_idx * 1000, 5i32)
};
let prep = crate::rowops_dispatch::GdfPrepRow {
alpha_base,
weight_base,
bias_idx,
error_lut_base,
scale,
ref_dst_idx,
grad_shift: 2,
};
let mut grad = [[[0u16; 4]; GRADIENT_BUF_STRIDE]; 2];
{
let refs: [&[u8]; 13] = core::array::from_fn(|i| &row_buffers[ptrs[i]] as &[u8]);
compute_gradient_row_8bpc(exec, &mut grad[0], &refs, 6, o, w, 0);
}
let mut grad_bit: usize = 1;
for y in 0..h {
if y + 6 < h {
backup_row_8bpc(
&mut row_buffers[bak_idx],
o,
p,
p_off + (y + 6) * stride,
&left[y + 6],
6,
w,
6,
edges,
);
ptrs[12] = bak_idx;
} else if edges & LR_HAVE_BOTTOM_INTEGRATED != 0 {
backup_row_lpf_8bpc(
&mut row_buffers[bak_idx],
o,
p,
p_off + (y + 6) * stride,
w,
6,
edges,
);
ptrs[12] = bak_idx;
} else if y + 4 < h && edges & LR_HAVE_BOTTOM != 0 {
let offset_y = y + 6 - h;
backup_row_lpf_8bpc(
&mut row_buffers[bak_idx],
o,
lpf_bottom,
lpf_bottom_off + offset_y * stride,
w,
6,
edges,
);
ptrs[12] = bak_idx;
} else {
ptrs[12] = ptrs[11];
}
bak_idx += 1;
if bak_idx == 13 {
bak_idx = 0;
}
if y & 1 == 0 {
let refs: [&[u8]; 13] = core::array::from_fn(|i| &row_buffers[ptrs[i]] as &[u8]);
compute_gradient_row_8bpc(exec, &mut grad[grad_bit], &refs, 8, o, w, 0);
grad_bit ^= 1;
}
let refs: [&[u8]; 13] = core::array::from_fn(|i| &row_buffers[ptrs[i]] as &[u8]);
unsafe {
(exec.gdf_prep_row_8bpc)(
refs,
o,
w,
[&grad[0][..], &grad[1][..]],
&mut dst[y * dst_stride..],
&prep,
)
};
for r in 0..12 {
ptrs[r] = ptrs[r + 1];
}
}
}
pub(crate) fn gdf_prep_hbd(
exec: &crate::exec_context::ExecContext,
dst: &mut [i8],
dst_stride: usize,
p: &[u16],
p_off: usize,
stride: usize,
left: &[[u16; 6]],
lpf: &[u16],
lpf_off: usize,
lpf_bottom: &[u16],
lpf_bottom_off: usize,
w: usize,
h: usize,
ref_dst_idx: usize,
qp_idx: usize,
edges: u8,
bitdepth_max: i32,
) {
let bitdepth = gdf_bitdepth_from_max(bitdepth_max);
let down_shift = if bitdepth == 12 { 2 } else { 0 };
let up_shift = if bitdepth == 8 { 2 } else { 0 };
let grad_shift = 4 - up_shift;
let up_scale = 1i32 << up_shift;
let mut row_buffers = [[0u16; REST_UNIT_STRIDE]; 13];
let mut ptrs: [usize; 13] = [0; 13];
let o = ROW_ORIGIN;
backup_row_hbd(&mut row_buffers[6], o, p, p_off, &left[0], 6, w, 6, edges);
ptrs[6] = 6;
if edges & LR_HAVE_TOP_INTEGRATED != 0 {
for n in 0..6 {
backup_row_lpf_hbd(
&mut row_buffers[n],
o,
lpf,
lpf_off + n * stride,
w,
6,
edges,
);
ptrs[n] = n;
}
} else if edges & LR_HAVE_TOP != 0 {
backup_row_lpf_hbd(&mut row_buffers[4], o, lpf, lpf_off, w, 6, edges);
ptrs[4] = 4;
backup_row_lpf_hbd(&mut row_buffers[5], o, lpf, lpf_off + stride, w, 6, edges);
ptrs[5] = 5;
ptrs[0] = 4;
ptrs[1] = 4;
ptrs[2] = 4;
ptrs[3] = 4;
} else {
for n in 0..6 {
ptrs[n] = 6;
}
}
let mut bak_idx = 7usize;
for y in 1..6 {
backup_row_hbd(
&mut row_buffers[bak_idx],
o,
p,
p_off + y * stride,
&left[y],
6,
w,
6,
edges,
);
ptrs[bak_idx] = bak_idx;
bak_idx += 1;
}
let alpha_base = ref_dst_idx * 528 + qp_idx * 88;
let weight_base = ref_dst_idx * 1584 + qp_idx * 264;
let bias_idx = ref_dst_idx * 6 + qp_idx;
let (error_lut_base, scale) = if ref_dst_idx == 0 {
(qp_idx * 4096, 8i32)
} else {
((ref_dst_idx - 1) * 6000 + qp_idx * 1000, 5i32)
};
let prep = crate::rowops_dispatch::GdfPrepRow {
alpha_base,
weight_base,
bias_idx,
error_lut_base,
scale,
ref_dst_idx,
grad_shift,
};
let mut grad = [[[0u16; 4]; GRADIENT_BUF_STRIDE]; 2];
{
let refs: [&[u16]; 13] = core::array::from_fn(|i| &row_buffers[ptrs[i]] as &[u16]);
compute_gradient_row_hbd(exec, &mut grad[0], &refs, 6, o, w, down_shift);
}
let mut grad_bit: usize = 1;
for y in 0..h {
if y + 6 < h {
backup_row_hbd(
&mut row_buffers[bak_idx],
o,
p,
p_off + (y + 6) * stride,
&left[y + 6],
6,
w,
6,
edges,
);
ptrs[12] = bak_idx;
} else if edges & LR_HAVE_BOTTOM_INTEGRATED != 0 {
backup_row_lpf_hbd(
&mut row_buffers[bak_idx],
o,
p,
p_off + (y + 6) * stride,
w,
6,
edges,
);
ptrs[12] = bak_idx;
} else if y + 4 < h && edges & LR_HAVE_BOTTOM != 0 {
let offset_y = y + 6 - h;
backup_row_lpf_hbd(
&mut row_buffers[bak_idx],
o,
lpf_bottom,
lpf_bottom_off + offset_y * stride,
w,
6,
edges,
);
ptrs[12] = bak_idx;
} else {
ptrs[12] = ptrs[11];
}
bak_idx += 1;
if bak_idx == 13 {
bak_idx = 0;
}
if y & 1 == 0 {
let refs: [&[u16]; 13] = core::array::from_fn(|i| &row_buffers[ptrs[i]] as &[u16]);
compute_gradient_row_hbd(exec, &mut grad[grad_bit], &refs, 8, o, w, down_shift);
grad_bit ^= 1;
}
let refs: [&[u16]; 13] = core::array::from_fn(|i| &row_buffers[ptrs[i]] as &[u16]);
unsafe {
(exec.gdf_prep_row_hbd)(
refs,
o,
w,
[&grad[0][..], &grad[1][..]],
&mut dst[y * dst_stride..],
&prep,
down_shift,
up_scale,
)
};
for r in 0..12 {
ptrs[r] = ptrs[r + 1];
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)]
pub(crate) enum RestorationType {
#[allow(unused)]
None = 0,
PcWiener = 1,
NsWiener = 2,
#[allow(unused)]
Switchable = 3,
}
pub(crate) const LR_RESTORE_Y: i32 = 1 << 0;
pub(crate) const LR_RESTORE_U: i32 = 1 << 1;
pub(crate) const LR_RESTORE_V: i32 = 1 << 2;
pub(crate) const INLOOPFILTER_DEBLOCK: u32 = 1 << 0;
pub(crate) const INLOOPFILTER_CDEF: u32 = 1 << 1;
pub(crate) const INLOOPFILTER_CCSO: u32 = 1 << 2;
pub(crate) const INLOOPFILTER_WIENER: u32 = 1 << 3;
pub(crate) const INLOOPFILTER_GDF: u32 = 1 << 4;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)]
pub(crate) enum FirstSbInTileRow {
None = 0,
Top = 1,
Bottom = 2,
}
pub(crate) struct LrContext<'a> {
pub(crate) exec: &'a crate::exec_context::ExecContext,
pub(crate) restoration_p: &'a [FhRestorationPlane; 3],
pub(crate) gdf_qp_idx: i32,
pub(crate) gdf_scale: i32,
pub(crate) sb128: bool,
pub(crate) cfl_ds_filter_index: i32,
pub(crate) layout: PixelLayout,
pub(crate) bw: i32,
pub(crate) bh: i32,
pub(crate) sb256w: i32,
pub(crate) sbh: i32,
pub(crate) mask: &'a [Av2Filter],
pub(crate) lr_mask: &'a [Av2Restoration],
pub(crate) lr_db_line: &'a [Vec<u8>; 3],
pub(crate) lr_cdef_line: &'a [Vec<u8>; 3],
pub(crate) lf_p_luma: &'a [u8],
pub(crate) base_q: i32,
pub(crate) gdf_ref_dst_idx: i32,
pub(crate) start_of_tile_row: &'a [u8],
pub(crate) ns_subclass_lut: &'a [u8],
pub(crate) pc_subclass_lut: &'a [u8],
pub(crate) pc_filters: &'a [[i16; 13]],
pub(crate) n_tc: i32,
pub(crate) inloop_filters: u32,
pub(crate) cur_stride: [isize; 2],
pub(crate) unit_size: [u8; 2],
pub(crate) restore_planes: i32,
pub(crate) col_start_sb: &'a [u16],
pub(crate) row_start_sb: &'a [u16],
pub(crate) sbsz: i32,
}
fn lr_stripe_8bpc(
ctx: &LrContext,
p: &mut [u8],
p_off: usize,
left: &[[u8; 6]],
x: i32,
mut y: i32,
plane: i32,
w: i32,
row_h: i32,
lr: &crate::lf_mask::Av2RestorationUnit,
mut edges: u8,
first_sby_in_tile_row: FirstSbInTileRow,
tile_row_m1: i32,
) {
let chroma = (plane != 0) as i32;
let ss_ver = chroma & (ctx.layout == PixelLayout::I420) as i32;
let ss_hor = chroma & (ctx.layout != PixelLayout::I444) as i32;
let stride = ctx.cur_stride[chroma as usize];
let abs_stride = stride.unsigned_abs();
let sby = (y + if y != 0 { 8 << ss_ver } else { 0 }) >> (6 - ss_ver + ctx.sb128 as i32);
let have_tt = (ctx.n_tc > 1) as i32;
let sb256x = (x << ss_hor) >> 8;
let sb64x_idx = ((x << ss_hor) >> 6) & 3;
let mut stripe_h = imin(
(64 - 8 * (first_sby_in_tile_row != FirstSbInTileRow::None) as i32) >> ss_ver,
row_h - y,
);
let ref_dst_idx = ctx.gdf_ref_dst_idx;
let qp_idx = ctx.gdf_qp_idx;
let gdf_scale = ctx.gdf_scale;
let wiener_type: u8 = if ctx.inloop_filters & INLOOPFILTER_WIENER != 0 {
lr.restoration_type
} else {
0 };
#[derive(PartialEq)]
enum WKind {
None,
NsSingle,
NsMulti,
PcWiener,
}
let mut multi_wiener = false;
let mut noskip_mask = [0u16; 66];
let pd = &ctx.restoration_p[plane as usize].ns;
let wkind = if wiener_type == RestorationType::NsWiener as u8 {
if pd.frame_filters_on != 0 {
if pd.num_classes == 1 {
WKind::NsSingle
} else {
multi_wiener = true;
WKind::NsMulti
}
} else {
WKind::NsSingle
}
} else if wiener_type == RestorationType::PcWiener as u8 {
multi_wiener = true;
WKind::PcWiener
} else {
WKind::None
};
if multi_wiener {
let mut r = 0usize;
let mut by = y >> 2;
while by < row_h >> 2 {
let by_idx = (by & 63) as usize;
let sb256_idx = ctx.sb256w as usize * (by as usize >> 6) + sb256x as usize;
if sb256_idx < ctx.mask.len() {
let noskip_row = &ctx.mask[sb256_idx].lr_noskip_mask[by_idx];
noskip_mask[r] = noskip_row[sb64x_idx as usize];
if edges & LR_HAVE_RIGHT == 0 && w & 63 != 0 {
let shift = ((w >> 2) & 15) - 1;
let mask_val = noskip_mask[r];
let edge_bit = mask_val >> shift as u16;
noskip_mask[r] |= edge_bit << ((shift + 1) as u16);
}
}
r += 1;
by += 1;
}
}
let lstride = ctx.cur_stride[0];
let lstride_u = lstride.unsigned_abs();
let stride_u = abs_stride;
let mut gdf_err = [0i8; 64 * 64];
let mut left_idx = 0usize;
let mut noskip_offset = 0usize;
let mut cur_off = p_off;
let mut lpf_off =
(have_tt as isize * (sby as isize * (4 << ctx.sb128 as i32) - 4) * stride_u as isize
+ x as isize) as usize;
let mut llpf_off =
(have_tt as isize * (sby as isize * (4 << ctx.sb128 as i32) - 4) * lstride_u as isize
+ (x << ss_hor) as isize) as usize;
while y + stripe_h <= row_h {
edges ^= ((-(((sby + 1) != ctx.sbh) as i32 | (y + stripe_h != row_h) as i32)) as u8
^ edges)
& LR_HAVE_BOTTOM;
let inc = if (edges & (LR_HAVE_TOP | LR_HAVE_TOP_INTEGRATED | LR_HAVE_BOTTOM_INTEGRATED))
== LR_HAVE_TOP
&& y + 8 < ((ctx.bh * 4) >> ss_ver)
{
8
} else {
0
};
let sb256_idx =
ctx.sb256w as usize * ((((y << ss_ver) + inc) as usize) >> 8) + sb256x as usize;
let gdf_enabled = plane == 0
&& ctx.inloop_filters & INLOOPFILTER_GDF != 0
&& sb256_idx < ctx.mask.len()
&& ctx.mask[sb256_idx].gdf[((((y + inc) >> 4) & 12) + sb64x_idx) as usize] != 0;
let plane_u = plane as usize;
let stripe_u = stripe_h as usize;
let w_u = w as usize;
let cdef_top: Option<usize> = if (edges & (LR_HAVE_TOP | LR_HAVE_TOP_INTEGRATED))
== (LR_HAVE_TOP | LR_HAVE_TOP_INTEGRATED)
{
let cdef_off = tile_row_m1 as usize * (6 - 4 * chroma as usize) * stride_u + x as usize;
let off = cdef_off + (2 * (1 - chroma) as usize) * stride_u;
if off < ctx.lr_cdef_line[plane_u].len() {
Some(off)
} else {
None
}
} else {
None
};
if gdf_enabled {
let (top_slice, top_off): (&[u8], usize) = match cdef_top {
Some(off) => (&ctx.lr_cdef_line[plane_u], off - 2 * stride_u),
None => (&ctx.lr_db_line[plane_u], lpf_off),
};
gdf_prep_8bpc(
ctx.exec,
&mut gdf_err,
64,
&*p,
cur_off,
stride_u,
&left[left_idx..],
top_slice,
top_off,
&ctx.lr_db_line[plane_u],
lpf_off + 6 * stride_u,
w_u,
stripe_u,
ref_dst_idx as usize,
qp_idx as usize,
edges,
);
}
let y4 = (((y << ss_ver) & 255) >> 2) as usize;
let ll_rows = (stripe_h >> 2).max(1) as usize + 1;
let mut ll_mask_storage = [[0u16; 4]; 17];
let ll_mask_buf = &mut ll_mask_storage[..ll_rows.min(17)];
if sb256_idx < ctx.mask.len() {
let m = &ctx.mask[sb256_idx];
if plane == 0 || ss_hor == 0 {
let src = if plane == 0 {
&m.lossless_mask_y
} else {
&m.lossless_mask_uv
};
for (r, slot) in ll_mask_buf.iter_mut().enumerate() {
let yy = y4 + r;
if yy < 64 {
let row = &src[yy];
for k in 0..4 {
if sb64x_idx as usize + k < 4 {
slot[k] = row[sb64x_idx as usize + k];
}
}
}
}
} else {
let init_y = (y >> 2) as usize;
let end_y = ((y + stripe_h) >> 2) as usize;
let base = (y4 >> ss_ver) as usize;
for (r, yy) in (init_y..end_y).enumerate() {
let src_y = base + (yy - init_y);
if src_y < 64 && r < ll_mask_buf.len() {
let row = &m.lossless_mask_uv[src_y];
let lo = row[sb64x_idx as usize];
let hi = if sb64x_idx as usize + 1 < 4 {
row[sb64x_idx as usize + 1]
} else {
0
};
ll_mask_buf[r][0] = (lo & 0xff) | ((hi & 0xff) << 8);
}
}
}
}
if wkind != WKind::None {
let (top_slice, top_off): (&[u8], usize) = match cdef_top {
Some(off) => (&ctx.lr_cdef_line[plane_u], off),
None => (&ctx.lr_db_line[plane_u], lpf_off),
};
let db_line = &ctx.lr_db_line[plane_u];
let bottom_off = lpf_off + 6 * stride_u;
match wkind {
WKind::NsSingle if chroma == 0 => {
let mut filter = [0i8; 16];
let src = if pd.frame_filters_on != 0 {
&pd.filter[0][..16]
} else {
&lr.ns_filter[0][..16]
};
filter.copy_from_slice(src);
let (ts, to): (&[u8], usize) = if cdef_top.is_some() {
(top_slice, top_off)
} else {
(db_line, lpf_off)
};
ns_wiener_single_y_8bpc(
ctx.exec,
p,
cur_off,
stride_u,
&left[left_idx..],
ts,
to,
db_line,
bottom_off,
w_u,
stripe_u,
&filter,
edges,
ll_mask_buf,
);
}
WKind::NsSingle => {
let mut filter = [0i8; 18];
let src = if pd.frame_filters_on != 0 {
&pd.filter[0][..18]
} else {
&lr.ns_filter[0][..18]
};
filter.copy_from_slice(src);
let luma = ctx.lf_p_luma;
let luma_off = (x << ss_hor) as usize + (y << ss_ver) as usize * lstride_u;
let luma_db = &ctx.lr_db_line[0];
let (ts, to): (&[u8], usize) = if cdef_top.is_some() {
(top_slice, top_off)
} else {
(db_line, lpf_off)
};
ns_wiener_single_uv_8bpc(
ctx.exec,
p,
cur_off,
stride_u,
&left[left_idx..],
ts,
to,
db_line,
bottom_off,
w_u,
stripe_u,
&filter,
luma,
luma_off,
lstride_u,
luma_db,
llpf_off,
luma_db,
llpf_off + 6 * lstride_u,
ss_hor as usize,
ss_ver as usize,
ctx.cfl_ds_filter_index,
edges,
ll_mask_buf,
);
}
WKind::NsMulti => {
let mut filters_owned = [[0i8; 18]; 16];
let filters: &[[i8; 18]] = if pd.frame_filters_on != 0 {
&pd.filter
} else {
for (dst, src) in filters_owned.iter_mut().zip(lr.ns_filter.iter()) {
dst.copy_from_slice(&src[..18]);
}
&filters_owned
};
let (ts, to): (&[u8], usize) = if cdef_top.is_some() {
(top_slice, top_off)
} else {
(db_line, lpf_off)
};
ns_wiener_multi_8bpc(
ctx.exec,
p,
cur_off,
stride_u,
&left[left_idx..],
ts,
to,
db_line,
bottom_off,
w_u,
stripe_u,
filters,
ctx.ns_subclass_lut,
&noskip_mask[noskip_offset..],
ctx.base_q,
edges,
ll_mask_buf,
);
noskip_offset += (stripe_h >> 2) as usize;
}
WKind::PcWiener => {
let (ts, to): (&[u8], usize) = if cdef_top.is_some() {
(top_slice, top_off)
} else {
(db_line, lpf_off)
};
pc_wiener_8bpc(
ctx.exec,
p,
cur_off,
stride_u,
&left[left_idx..],
ts,
to,
db_line,
bottom_off,
w_u,
stripe_u,
ctx.pc_filters,
ctx.pc_subclass_lut,
&noskip_mask[noskip_offset..],
ctx.base_q,
edges,
ll_mask_buf,
);
noskip_offset += (stripe_h >> 2) as usize;
}
WKind::None => {}
}
}
if gdf_enabled {
gdf_add_8bpc(
ctx.exec,
&mut p[cur_off..],
stride_u,
&gdf_err,
64,
w_u,
stripe_u,
gdf_scale,
ll_mask_buf,
);
}
edges &= !(LR_HAVE_BOTTOM_INTEGRATED | LR_HAVE_TOP_INTEGRATED);
left_idx += stripe_h as usize;
cur_off += stripe_u * stride_u;
y += stripe_h;
edges |= LR_HAVE_TOP;
stripe_h = imin(64 >> ss_ver, row_h - y);
if stripe_h == 0 {
break;
}
lpf_off += 4 * stride_u;
llpf_off += 4 * lstride_u;
}
}
#[allow(clippy::too_many_arguments)]
fn lr_unit_at_8bpc(
ctx: &LrContext,
pos: i32,
w: i32,
unit_size: i32,
ss_hor: i32,
sb_idx: i32,
unit_idx_base: i32,
) -> (usize, usize) {
let shift_hor = 8 - ss_hor;
let mut start = pos & !(unit_size - 1);
let mut tc = 0usize;
while tc + 2 < ctx.col_start_sb.len()
&& start >= (ctx.col_start_sb[tc + 1] as i32 * ctx.sbsz) >> ss_hor
{
tc += 1;
}
let tile_col_start_x = (ctx.col_start_sb[tc] as i32 * ctx.sbsz) >> ss_hor;
if start > tile_col_start_x && w - start < unit_size >> 1 {
start -= unit_size;
}
(
(sb_idx + (start >> shift_hor)) as usize,
(unit_idx_base + ((start >> (shift_hor - 2)) & 3)) as usize,
)
}
#[allow(clippy::too_many_arguments)]
fn lr_chunk_8bpc(
ctx: &LrContext,
p: &mut [u8],
p_off: usize,
left: &[[u8; 6]],
x: i32,
y: i32,
plane: i32,
width: i32,
row_h: i32,
edges: u8,
first_sby_in_tile_row: FirstSbInTileRow,
tile_row_m1: i32,
first_unit: (usize, usize),
w: i32,
unit_size: i32,
ss_hor: i32,
sb_idx: i32,
unit_idx_base: i32,
) {
type Px = u8;
let plane_u = plane as usize;
let unit_ok =
|(sb, u): (usize, usize)| sb < ctx.lr_mask.len() && u < ctx.lr_mask[sb].lr[plane_u].len();
let mut segs: [(i32, (usize, usize)); 4] = [(0, first_unit); 4];
let mut n_segs = 1usize;
let mut off = unit_size;
while unit_size < width && off < width && n_segs < 4 {
let unit = lr_unit_at_8bpc(ctx, x + off, w, unit_size, ss_hor, sb_idx, unit_idx_base);
if unit != segs[n_segs - 1].1 {
segs[n_segs] = (off, unit);
n_segs += 1;
}
off += unit_size;
}
if n_segs == 1 {
if unit_ok(first_unit) {
let lr_unit = &ctx.lr_mask[first_unit.0].lr[plane_u][first_unit.1];
lr_stripe_8bpc(
ctx,
p,
p_off,
left,
x,
y,
plane,
width,
row_h,
lr_unit,
edges,
first_sby_in_tile_row,
tile_row_m1,
);
}
return;
}
let abs_stride = ctx.cur_stride[(plane != 0) as usize].unsigned_abs();
let n = if plane != 0 { 2 } else { 6 };
let rows = (row_h - y).max(0) as usize;
let mut borders = [[[Px::default(); 6]; 264]; 3];
for (k, border) in borders.iter_mut().enumerate().take(n_segs - 1) {
let col = p_off + segs[k + 1].0 as usize;
for (row, b) in border.iter_mut().enumerate().take(rows) {
let at = col + row * abs_stride;
if at >= n && at <= p.len() {
b[6 - n..].copy_from_slice(&p[at - n..at]);
}
}
}
for k in 0..n_segs {
let (seg_x, unit) = segs[k];
let seg_w = if k + 1 < n_segs { segs[k + 1].0 } else { width } - seg_x;
let mut seg_edges = edges;
if k > 0 {
seg_edges |= LR_HAVE_LEFT;
}
if k + 1 < n_segs {
seg_edges |= LR_HAVE_RIGHT;
}
if !unit_ok(unit) {
continue;
}
let seg_left: &[[Px; 6]] = if k == 0 { left } else { &borders[k - 1] };
let lr_unit = &ctx.lr_mask[unit.0].lr[plane_u][unit.1];
lr_stripe_8bpc(
ctx,
p,
p_off + seg_x as usize,
seg_left,
x + seg_x,
y,
plane,
seg_w,
row_h,
lr_unit,
seg_edges,
first_sby_in_tile_row,
tile_row_m1,
);
}
}
fn lr_sbrow(
ctx: &LrContext,
p: &mut [u8],
p_off: usize,
y: i32,
w: i32,
h: i32,
row_h: i32,
plane: i32,
first_sby_in_tile_row: FirstSbInTileRow,
tile_row_m1: i32,
) {
let chroma = (plane != 0) as i32;
let ss_ver = chroma & (ctx.layout == PixelLayout::I420) as i32;
let ss_hor = chroma & (ctx.layout != PixelLayout::I444) as i32;
let p_stride = ctx.cur_stride[chroma as usize];
let abs_stride = p_stride.unsigned_abs();
let unit_size_log2 = ctx.unit_size[chroma.min(1) as usize] as i32;
let unit_size = 1 << unit_size_log2;
let half_unit_size = unit_size >> 1;
let row_y = y + ((8 >> ss_ver) * (y != 0) as i32);
let shift_hor = 8 - ss_hor;
let mut pre_lr_border = [[[0u8; 6]; 264]; 2];
let probe = if first_sby_in_tile_row == FirstSbInTileRow::Top {
y
} else {
row_y
};
let tile_row_px = |i: usize| (ctx.row_start_sb[i] as i32 * ctx.sbsz) >> ss_ver;
let mut tr = 0usize;
while tr + 2 < ctx.row_start_sb.len() && probe >= tile_row_px(tr + 1) {
tr += 1;
}
let tile_top = tile_row_px(tr);
let tile_bot = imin(tile_row_px(tr + 1), h);
let mut aligned_unit_pos = tile_top + ((probe - tile_top) & !(unit_size - 1));
if aligned_unit_pos > tile_top && aligned_unit_pos + half_unit_size > tile_bot {
aligned_unit_pos -= unit_size;
}
aligned_unit_pos <<= ss_ver;
let sb_idx = (aligned_unit_pos >> 8) * ctx.sb256w;
let unit_idx_base = ((aligned_unit_pos >> 6) & 0x3) << 2;
let mut edges: u8 = if y > 0 { LR_HAVE_TOP } else { 0 } | LR_HAVE_RIGHT;
if first_sby_in_tile_row == FirstSbInTileRow::Top {
edges |= LR_HAVE_BOTTOM_INTEGRATED;
}
if first_sby_in_tile_row == FirstSbInTileRow::Bottom && y > 0 {
edges |= LR_HAVE_TOP_INTEGRATED;
}
let mut lr_idx_0 = sb_idx as usize;
let mut lr_unit_0 = unit_idx_base as usize;
let mut bit = 0usize;
let mut x = 0i32;
let mut cur_p_off = p_off;
let mut tile_col = 0usize;
while x + 64 < w {
let next_x = x + 64;
let mut next_iter_lru_start_x = next_x & !(unit_size - 1);
while tile_col + 2 < ctx.col_start_sb.len()
&& next_iter_lru_start_x >= (ctx.col_start_sb[tile_col + 1] as i32 * ctx.sbsz) >> ss_hor
{
tile_col += 1;
}
let tile_col_start_x = (ctx.col_start_sb[tile_col] as i32 * ctx.sbsz) >> ss_hor;
if next_iter_lru_start_x > tile_col_start_x && w - next_iter_lru_start_x < half_unit_size {
next_iter_lru_start_x -= unit_size;
}
let next_u_idx = unit_idx_base + ((next_iter_lru_start_x >> (shift_hor - 2)) & 3);
let next_sb_idx = sb_idx + (next_iter_lru_start_x >> shift_hor);
let lr_sb = lr_idx_0;
let lr_u = lr_unit_0;
let n = if plane != 0 { 2 } else { 6 };
let border_src_off = cur_p_off + 64;
if border_src_off + n <= p.len() {
for row in 0..(row_h - y) as usize {
let src_row = border_src_off + row * abs_stride;
if src_row >= n && src_row < p.len() {
let start = src_row - n;
for i in 0..n.min(6) {
pre_lr_border[bit][row][6 - n + i] = p[start + i];
}
}
}
}
lr_chunk_8bpc(
ctx,
p,
cur_p_off,
&pre_lr_border[1 - bit],
x,
y,
plane,
64,
row_h,
edges,
first_sby_in_tile_row,
tile_row_m1,
(lr_sb, lr_u),
w,
unit_size,
ss_hor,
sb_idx,
unit_idx_base,
);
lr_idx_0 = next_sb_idx as usize;
lr_unit_0 = next_u_idx as usize;
x = next_x;
cur_p_off += 64;
edges |= LR_HAVE_LEFT;
bit ^= 1;
}
edges &= !LR_HAVE_RIGHT;
let end_w = w - x;
let lr_sb = lr_idx_0;
let lr_u = lr_unit_0;
lr_chunk_8bpc(
ctx,
p,
cur_p_off,
&pre_lr_border[1 - bit],
x,
y,
plane,
end_w,
row_h,
edges,
first_sby_in_tile_row,
tile_row_m1,
(lr_sb, lr_u),
w,
unit_size,
ss_hor,
sb_idx,
unit_idx_base,
);
}
pub(crate) fn lr_sbrow_8bpc(ctx: &LrContext, dst: &mut [&mut [u8]; 3], sby: i32) {
if ctx.inloop_filters & INLOOPFILTER_WIENER == 0 {
return;
}
let dst_stride = ctx.cur_stride;
let mut restore_planes = ctx.restore_planes;
if dst[1].is_empty() || dst[2].is_empty() {
restore_planes &= LR_RESTORE_Y;
}
let not_last = (sby + 1 < ctx.sbh) as i32;
let start_tile_val = if (sby as usize) < ctx.start_of_tile_row.len() {
ctx.start_of_tile_row[sby as usize]
} else {
0
};
let tile_row = (start_tile_val >> 1) as i32;
let first_sby_in_tile_row_flag = start_tile_val & 1;
if restore_planes & (LR_RESTORE_U | LR_RESTORE_V) != 0 {
let ss_ver = (ctx.layout == PixelLayout::I420) as i32;
let ss_hor = (ctx.layout != PixelLayout::I444) as i32;
let h = (ctx.bh * 4) >> ss_ver;
let w = (ctx.bw * 4) >> ss_hor;
let next_row_y = (sby + 1) << ((6 - ss_ver) + ctx.sb128 as i32);
let row_h = imin(next_row_y - (8 >> ss_ver) * not_last, h);
let offset_uv = (8 * (sby != 0) as i32) >> ss_ver;
let mut y_stripe = (sby << ((6 - ss_ver) + ctx.sb128 as i32)) - offset_uv;
if sby != 0 && first_sby_in_tile_row_flag != 0 {
let first_top = if first_sby_in_tile_row_flag != 0 {
FirstSbInTileRow::Top
} else {
FirstSbInTileRow::None
};
if restore_planes & LR_RESTORE_U != 0 {
let abs_stride_uv = dst_stride[1].unsigned_abs();
lr_sbrow(
ctx,
dst[1],
(y_stripe.max(0) as usize) * abs_stride_uv,
y_stripe,
w,
h,
y_stripe + (8 >> ss_ver),
1,
first_top,
tile_row - 1,
);
}
if restore_planes & LR_RESTORE_V != 0 {
let abs_stride_uv = dst_stride[1].unsigned_abs();
lr_sbrow(
ctx,
dst[2],
(y_stripe.max(0) as usize) * abs_stride_uv,
y_stripe,
w,
h,
y_stripe + (8 >> ss_ver),
2,
first_top,
tile_row - 1,
);
}
y_stripe += 8 >> ss_ver;
}
let first_bot = if first_sby_in_tile_row_flag != 0 {
FirstSbInTileRow::Bottom
} else {
FirstSbInTileRow::None
};
if restore_planes & LR_RESTORE_U != 0 {
let abs_stride_uv = dst_stride[1].unsigned_abs();
lr_sbrow(
ctx,
dst[1],
(y_stripe.max(0) as usize) * abs_stride_uv,
y_stripe,
w,
h,
row_h,
1,
first_bot,
tile_row - 1,
);
}
if restore_planes & LR_RESTORE_V != 0 {
let abs_stride_uv = dst_stride[1].unsigned_abs();
lr_sbrow(
ctx,
dst[2],
(y_stripe.max(0) as usize) * abs_stride_uv,
y_stripe,
w,
h,
row_h,
2,
first_bot,
tile_row - 1,
);
}
}
if restore_planes & LR_RESTORE_Y != 0 {
let h = ctx.bh * 4;
let w = ctx.bw * 4;
let next_row_y = (sby + 1) << (6 + ctx.sb128 as i32);
let row_h = imin(next_row_y - 8 * not_last, h);
let offset_y = 8 * (sby != 0) as i32;
let mut y_stripe = (sby << (6 + ctx.sb128 as i32)) - offset_y;
if sby != 0 && first_sby_in_tile_row_flag != 0 {
let abs_stride_y = dst_stride[0].unsigned_abs();
lr_sbrow(
ctx,
dst[0],
(y_stripe.max(0) as usize) * abs_stride_y,
y_stripe,
w,
h,
y_stripe + 8,
0,
FirstSbInTileRow::Top,
tile_row - 1,
);
y_stripe += 8;
}
let abs_stride_y = dst_stride[0].unsigned_abs();
lr_sbrow(
ctx,
dst[0],
(y_stripe.max(0) as usize) * abs_stride_y,
y_stripe,
w,
h,
row_h,
0,
if first_sby_in_tile_row_flag != 0 {
FirstSbInTileRow::Bottom
} else {
FirstSbInTileRow::None
},
tile_row - 1,
);
}
}
pub(crate) const EMPTY_LR_DB_LINE_HBD: [Vec<u16>; 3] = [Vec::new(), Vec::new(), Vec::new()];
pub(crate) fn backup_row_hbd(
dst: &mut [u16],
o: usize,
src: &[u16],
src_o: usize,
left: &[u16],
left_off: usize,
w: usize,
edge_len: usize,
edges: u8,
) {
if edges & LR_HAVE_LEFT != 0 {
for x in 0..edge_len {
dst[o - edge_len + x] = left[left_off - edge_len + x];
}
} else {
dst[o - edge_len..o].fill(src[src_o]);
}
dst[o..o + w].copy_from_slice(&src[src_o..src_o + w]);
if edges & LR_HAVE_RIGHT != 0 {
dst[o + w..o + w + edge_len].copy_from_slice(&src[src_o + w..src_o + w + edge_len]);
} else {
dst[o + w..o + w + edge_len].fill(src[src_o + w - 1]);
}
}
pub(crate) fn backup_row_lpf_hbd(
dst: &mut [u16],
o: usize,
src: &[u16],
src_o: usize,
w: usize,
edge_len: usize,
edges: u8,
) {
if edges & LR_HAVE_LEFT != 0 {
for x in 0..edge_len {
dst[o - edge_len + x] = src[src_o - edge_len + x];
}
} else {
dst[o - edge_len..o].fill(src[src_o]);
}
dst[o..o + w].copy_from_slice(&src[src_o..src_o + w]);
if edges & LR_HAVE_RIGHT != 0 {
dst[o + w..o + w + edge_len].copy_from_slice(&src[src_o + w..src_o + w + edge_len]);
} else {
dst[o + w..o + w + edge_len].fill(src[src_o + w - 1]);
}
}
pub(crate) fn backup_row_luma_hbd(
dst: &mut [u16],
o: usize,
src: &[u16],
src_o: usize,
src_stride: usize,
w: usize,
edges: u8,
ss_hor: usize,
ss_ver: usize,
cfl_ds_flt: i32,
) {
if ss_ver == 0 {
backup_row_lpf_hbd(dst, o, src, src_o, w, 4, edges);
if ss_hor != 0 && edges & LR_HAVE_RIGHT == 0 {
let fill_val = src[src_o + w - 2];
dst[o + w..o + w + 4].fill(fill_val);
}
return;
}
let src2_o = src_o + src_stride;
match cfl_ds_flt {
0 => {
let mut x = 0;
while x < w {
dst[o + x] = ((src[src_o + x] as u32
+ src[src2_o + x] as u32
+ src[src_o + x + 1] as u32
+ src[src2_o + x + 1] as u32)
>> 2) as u16;
x += 1 + ss_hor;
}
}
1 => {
for x in 0..w {
dst[o + x] = ((src[src_o + x] as u32 + src[src2_o + x] as u32) >> 1) as u16;
}
}
_ => {
dst[o..o + w].copy_from_slice(&src[src_o..src_o + w]);
}
}
if edges & LR_HAVE_LEFT != 0 {
match cfl_ds_flt {
0 => {
for i in 0..4usize {
let si = src_o - 4 + i;
let s2i = src2_o - 4 + i;
dst[o - 4 + i] = ((src[si] as u32
+ src[s2i] as u32
+ src[si + 1] as u32
+ src[s2i + 1] as u32)
>> 2) as u16;
}
}
1 => {
for i in 0..4usize {
dst[o - 4 + i] =
((src[src_o - 4 + i] as u32 + src[src2_o - 4 + i] as u32) >> 1) as u16;
}
}
_ => {
for i in 0..4usize {
dst[o - 4 + i] = src[src_o - 4 + i];
}
}
}
} else {
let fill_val = dst[o];
dst[o - 4..o].fill(fill_val);
}
if edges & LR_HAVE_RIGHT != 0 {
match cfl_ds_flt {
0 => {
for i in 0..4usize {
let si = src_o + w + i;
let s2i = src2_o + w + i;
dst[o + w + i] = ((src[si] as u32
+ src[s2i] as u32
+ src[si + 1] as u32
+ src[s2i + 1] as u32)
>> 2) as u16;
}
}
1 => {
for i in 0..4usize {
dst[o + w + i] =
((src[src_o + w + i] as u32 + src[src2_o + w + i] as u32) >> 1) as u16;
}
}
_ => {
for i in 0..4usize {
dst[o + w + i] = src[src_o + w + i];
}
}
}
} else {
let fill_val = dst[o + w - 2];
dst[o + w..o + w + 4].fill(fill_val);
}
}
pub(crate) fn get_class_lut_idx_hbd(
rows: &[&[u16]],
row_center: usize,
col_off: usize,
noskip_mask: &[u16],
base_q: i32,
bx: usize,
by: usize,
bh: usize,
bitdepth_min_8: i32,
) -> i32 {
let mut f = pcw_features(rows, row_center, col_off + bx * 4 - 1);
let mut s = 0i32;
let num_pixels: [u8; 3] = [16, 4, 1];
for dy in -1i32..=1 {
for dx in -1i32..=1 {
let edge = (dy != 0) as usize + (dx != 0) as usize;
let fx = iclip((bx & 15) as i32 + dx, 0, 15) as usize;
let fy = iclip(by as i32 + dy, 0, bh as i32 - 1) as usize;
s += num_pixels[edge] as i32 * (((noskip_mask[fy] >> fx) & 1) == 0) as i32;
}
}
let rnd = (1i64 << bitdepth_min_8) >> 1;
for i in 0..3 {
f[i] = ((f[i] as i64 * PC_WIENER_NORMALIZER[i] as i64 + rnd) >> bitdepth_min_8) as i32;
}
s *= PC_WIENER_NORMALIZER[3] as i32;
let mut qval = get_qval_lut_bin(get_qval_given_tskip(base_q, s, 0, bitdepth_min_8));
let mut lut_idx = qval << 9;
for i in 0..3 {
qval = get_qval_lut_bin(f[i].saturating_add(get_qval_given_tskip(
base_q,
s,
i + 1,
bitdepth_min_8,
)));
lut_idx |= qval << (3 * (2 - i));
}
lut_idx
}
pub(crate) fn ns_wiener_single_y_hbd(
exec: &crate::exec_context::ExecContext,
p: &mut [u16],
p_off: usize,
stride: usize,
left: &[[u16; 6]],
lpf: &[u16],
lpf_off: usize,
lpf_bottom: &[u16],
lpf_bottom_off: usize,
w: usize,
h: usize,
filter: &[i8; 16],
edges: u8,
ll_mask: &[[u16; 4]],
bitdepth_max: i32,
) {
let mut row_buffers = [[0u16; REST_UNIT_STRIDE]; 9];
let mut ptrs: [usize; 9] = [0; 9];
let o = ROW_ORIGIN;
backup_row_hbd(&mut row_buffers[4], o, &*p, p_off, &left[0], 6, w, 4, edges);
ptrs[4] = 4;
if edges & LR_HAVE_TOP_INTEGRATED != 0 {
let mut loff = lpf_off;
for i in 0..4 {
backup_row_lpf_hbd(&mut row_buffers[i], o, lpf, loff, w, 4, edges);
loff += stride;
ptrs[i] = i;
}
} else if edges & LR_HAVE_TOP != 0 {
backup_row_lpf_hbd(&mut row_buffers[2], o, lpf, lpf_off, w, 4, edges);
ptrs[2] = 2;
backup_row_lpf_hbd(&mut row_buffers[3], o, lpf, lpf_off + stride, w, 4, edges);
ptrs[3] = 3;
ptrs[0] = 2;
ptrs[1] = 2;
} else {
ptrs[0] = 4;
ptrs[1] = 4;
ptrs[2] = 4;
ptrs[3] = 4;
}
backup_row_hbd(
&mut row_buffers[5],
o,
&*p,
p_off + stride,
&left[1],
6,
w,
4,
edges,
);
ptrs[5] = 5;
backup_row_hbd(
&mut row_buffers[6],
o,
&*p,
p_off + 2 * stride,
&left[2],
6,
w,
4,
edges,
);
ptrs[6] = 6;
backup_row_hbd(
&mut row_buffers[7],
o,
&*p,
p_off + 3 * stride,
&left[3],
6,
w,
4,
edges,
);
ptrs[7] = 7;
let mut bak_idx: usize = 8;
for y in 0..h {
if y + 4 < h {
backup_row_hbd(
&mut row_buffers[bak_idx],
o,
&*p,
p_off + (y + 4) * stride,
&left[y + 4],
6,
w,
4,
edges,
);
ptrs[8] = bak_idx;
} else if edges & LR_HAVE_BOTTOM_INTEGRATED != 0 {
backup_row_lpf_hbd(
&mut row_buffers[bak_idx],
o,
&*p,
p_off + (y + 4) * stride,
w,
4,
edges,
);
ptrs[8] = bak_idx;
} else if y + 2 < h && edges & LR_HAVE_BOTTOM != 0 {
let offset_y = y + 4 - h;
backup_row_lpf_hbd(
&mut row_buffers[bak_idx],
o,
lpf_bottom,
lpf_bottom_off + offset_y * stride,
w,
4,
edges,
);
ptrs[8] = bak_idx;
} else {
ptrs[8] = ptrs[7];
}
bak_idx += 1;
if bak_idx == 9 {
bak_idx = 0;
}
let refs: [&[u16]; 9] = core::array::from_fn(|i| &row_buffers[ptrs[i]] as &[u16]);
let taps: [crate::filter::WienerTapHbd; 16] = core::array::from_fn(|i| {
let dy = WIENER_NS_CONFIG_Y[i][0] as i32;
let dx = WIENER_NS_CONFIG_Y[i][1] as i32;
crate::filter::WienerTapHbd {
row_p: refs[(4 + dy) as usize],
row_m: refs[(4 - dy) as usize],
dx,
coef: filter[i] as i32,
}
});
let skip_mask = ll_mask[y >> 2][0];
let dst_row = &mut p[p_off + y * stride..];
let bw = w >> 2;
let mut bx = 0usize;
while bx < bw {
if skip_mask & (1 << bx) != 0 {
bx += 1;
continue;
}
let bx_start = bx;
bx += 1;
while bx < bw && skip_mask & (1 << bx) == 0 {
bx += 1;
}
let x0 = bx_start << 2;
let n = (bx - bx_start) << 2;
unsafe {
(exec.ns_wiener_fir_hbd)(
&mut dst_row[x0..x0 + n],
refs[4],
o + x0,
&taps,
n,
bitdepth_max,
);
}
}
for r in 0..8 {
ptrs[r] = ptrs[r + 1];
}
}
}
fn wiener_multi_hbd(
exec: &crate::exec_context::ExecContext,
p: &mut [u16],
p_off: usize,
stride: usize,
left: &[[u16; 6]],
lpf: &[u16],
lpf_off: usize,
lpf_bottom: &[u16],
lpf_bottom_off: usize,
w: usize,
h: usize,
filters_user: Option<&[[i8; 18]]>,
filters_pretrained: Option<&[[i16; 13]]>,
subclass_lut: &[u8],
noskip_mask: &[u16],
base_q: i32,
edges: u8,
ll_mask: &[[u16; 4]],
bitdepth_min_8: i32,
bitdepth_max: i32,
) {
let mut classes = [0u8; 16];
let mut row_buffers = [[0u16; REST_UNIT_STRIDE]; 10];
let mut ptrs: [usize; 10] = [0; 10];
let o = ROW_ORIGIN;
backup_row_hbd(&mut row_buffers[4], o, &*p, p_off, &left[0], 6, w, 4, edges);
ptrs[4] = 4;
if edges & LR_HAVE_TOP_INTEGRATED != 0 {
let mut loff = lpf_off;
for i in 0..4 {
backup_row_lpf_hbd(&mut row_buffers[i], o, lpf, loff, w, 4, edges);
loff += stride;
ptrs[i] = i;
}
} else if edges & LR_HAVE_TOP != 0 {
backup_row_lpf_hbd(&mut row_buffers[2], o, lpf, lpf_off, w, 4, edges);
ptrs[2] = 2;
backup_row_lpf_hbd(&mut row_buffers[3], o, lpf, lpf_off + stride, w, 4, edges);
ptrs[3] = 3;
ptrs[0] = 2;
ptrs[1] = 2;
} else {
ptrs[0] = 4;
ptrs[1] = 4;
ptrs[2] = 4;
ptrs[3] = 4;
}
backup_row_hbd(
&mut row_buffers[5],
o,
&*p,
p_off + stride,
&left[1],
6,
w,
4,
edges,
);
ptrs[5] = 5;
backup_row_hbd(
&mut row_buffers[6],
o,
&*p,
p_off + 2 * stride,
&left[2],
6,
w,
4,
edges,
);
ptrs[6] = 6;
backup_row_hbd(
&mut row_buffers[7],
o,
&*p,
p_off + 3 * stride,
&left[3],
6,
w,
4,
edges,
);
ptrs[7] = 7;
let mut bak_idx: usize = 8;
let bh = h >> 2;
let bw = w >> 2;
for by in 0..bh {
let by4 = by << 2;
if by + 1 < bh {
backup_row_hbd(
&mut row_buffers[bak_idx],
o,
&*p,
p_off + (by4 + 4) * stride,
&left[by4 + 4],
6,
w,
4,
edges,
);
ptrs[8] = bak_idx;
backup_row_hbd(
&mut row_buffers[9],
o,
&*p,
p_off + (by4 + 5) * stride,
&left[by4 + 5],
6,
w,
4,
edges,
);
ptrs[9] = 9;
} else if edges & LR_HAVE_BOTTOM_INTEGRATED != 0 {
backup_row_lpf_hbd(
&mut row_buffers[bak_idx],
o,
&*p,
p_off + (by4 + 4) * stride,
w,
4,
edges,
);
ptrs[8] = bak_idx;
backup_row_lpf_hbd(
&mut row_buffers[9],
o,
&*p,
p_off + (by4 + 5) * stride,
w,
4,
edges,
);
ptrs[9] = 9;
} else if edges & LR_HAVE_BOTTOM != 0 {
backup_row_lpf_hbd(
&mut row_buffers[bak_idx],
o,
lpf_bottom,
lpf_bottom_off,
w,
4,
edges,
);
ptrs[8] = bak_idx;
backup_row_lpf_hbd(
&mut row_buffers[9],
o,
lpf_bottom,
lpf_bottom_off + stride,
w,
4,
edges,
);
ptrs[9] = 9;
} else {
ptrs[8] = ptrs[7];
ptrs[9] = ptrs[7];
}
{
let refs: [&[u16]; 10] = core::array::from_fn(|i| &row_buffers[ptrs[i]] as &[u16]);
for bx in 0..bw {
let lut_idx = get_class_lut_idx_hbd(
&refs,
4,
o,
noskip_mask,
base_q,
bx,
by,
bh,
bitdepth_min_8,
);
let cls = PC_WIENER_LUT_TO_CLASS[lut_idx as usize];
classes[bx] = subclass_lut[cls as usize];
}
}
for y in by4..by4 + 4 {
if y + 4 < h {
backup_row_hbd(
&mut row_buffers[bak_idx],
o,
&*p,
p_off + (y + 4) * stride,
&left[y + 4],
6,
w,
4,
edges,
);
ptrs[8] = bak_idx;
} else if edges & LR_HAVE_BOTTOM_INTEGRATED != 0 {
backup_row_lpf_hbd(
&mut row_buffers[bak_idx],
o,
&*p,
p_off + (y + 4) * stride,
w,
4,
edges,
);
ptrs[8] = bak_idx;
} else if y + 2 < h && edges & LR_HAVE_BOTTOM != 0 {
let offset_y = y + 4 - h;
backup_row_lpf_hbd(
&mut row_buffers[bak_idx],
o,
lpf_bottom,
lpf_bottom_off + offset_y * stride,
w,
4,
edges,
);
ptrs[8] = bak_idx;
} else {
ptrs[8] = ptrs[7];
}
bak_idx += 1;
if bak_idx == 9 {
bak_idx = 0;
}
let refs: [&[u16]; 10] = core::array::from_fn(|i| &row_buffers[ptrs[i]] as &[u16]);
let skip_mask = ll_mask[y >> 2][0];
let dst_row = &mut p[p_off + y * stride..];
let mut bx = 0usize;
while bx < bw {
if skip_mask & (1 << bx) != 0 {
bx += 1;
continue;
}
let cls = classes[bx];
let bx_start = bx;
bx += 1;
while bx < bw && skip_mask & (1 << bx) == 0 && classes[bx] == cls {
bx += 1;
}
let x0 = bx_start << 2;
let n = (bx - bx_start) << 2;
let col0 = o + x0;
if let Some(fu) = filters_user {
let filter = &fu[cls as usize];
let taps: [crate::filter::WienerTapHbd; 16] = core::array::from_fn(|i| {
let dy = WIENER_NS_CONFIG_Y[i][0] as i32;
let dx = WIENER_NS_CONFIG_Y[i][1] as i32;
crate::filter::WienerTapHbd {
row_p: refs[(4 + dy) as usize],
row_m: refs[(4 - dy) as usize],
dx,
coef: filter[i] as i32,
}
});
unsafe {
(exec.ns_wiener_fir_hbd)(
&mut dst_row[x0..x0 + n],
refs[4],
col0,
&taps,
n,
bitdepth_max,
);
}
} else if let Some(fp) = filters_pretrained {
let filter = &fp[cls as usize];
let taps: [crate::filter::WienerTapHbd; 12] = core::array::from_fn(|i| {
let dy = PC_WIENER_CONFIG[i][0] as i32;
let dx = PC_WIENER_CONFIG[i][1] as i32;
crate::filter::WienerTapHbd {
row_p: refs[(4 + dy) as usize],
row_m: refs[(4 - dy) as usize],
dx,
coef: filter[i] as i32,
}
});
unsafe {
(exec.pc_wiener_fir_hbd)(
&mut dst_row[x0..x0 + n],
refs[4],
filter[12] as i32,
col0,
&taps,
n,
bitdepth_max,
);
}
}
}
for r in 0..8 {
ptrs[r] = ptrs[r + 1];
}
}
}
}
pub(crate) fn ns_wiener_multi_hbd(
exec: &crate::exec_context::ExecContext,
p: &mut [u16],
p_off: usize,
stride: usize,
left: &[[u16; 6]],
lpf: &[u16],
lpf_off: usize,
lpf_bottom: &[u16],
lpf_bottom_off: usize,
w: usize,
h: usize,
filters_user: &[[i8; 18]],
subclass_lut: &[u8],
noskip_mask: &[u16],
base_q: i32,
edges: u8,
ll_mask: &[[u16; 4]],
bitdepth_min_8: i32,
bitdepth_max: i32,
) {
wiener_multi_hbd(
exec,
p,
p_off,
stride,
left,
lpf,
lpf_off,
lpf_bottom,
lpf_bottom_off,
w,
h,
Some(filters_user),
None,
subclass_lut,
noskip_mask,
base_q,
edges,
ll_mask,
bitdepth_min_8,
bitdepth_max,
);
}
pub(crate) fn pc_wiener_hbd(
exec: &crate::exec_context::ExecContext,
p: &mut [u16],
p_off: usize,
stride: usize,
left: &[[u16; 6]],
lpf: &[u16],
lpf_off: usize,
lpf_bottom: &[u16],
lpf_bottom_off: usize,
w: usize,
h: usize,
filters_pretrained: &[[i16; 13]],
subclass_lut: &[u8],
noskip_mask: &[u16],
base_q: i32,
edges: u8,
ll_mask: &[[u16; 4]],
bitdepth_min_8: i32,
bitdepth_max: i32,
) {
wiener_multi_hbd(
exec,
p,
p_off,
stride,
left,
lpf,
lpf_off,
lpf_bottom,
lpf_bottom_off,
w,
h,
None,
Some(filters_pretrained),
subclass_lut,
noskip_mask,
base_q,
edges,
ll_mask,
bitdepth_min_8,
bitdepth_max,
);
}
pub(crate) fn ns_wiener_single_uv_hbd(
exec: &crate::exec_context::ExecContext,
p: &mut [u16],
p_off: usize,
stride: usize,
left: &[[u16; 6]],
lpf: &[u16],
lpf_off: usize,
lpf_bottom: &[u16],
lpf_bottom_off: usize,
w: usize,
h: usize,
filter: &[i8; 18],
luma: &[u16],
luma_off: usize,
lstride: usize,
luma_top: &[u16],
luma_top_off: usize,
luma_bottom: &[u16],
luma_bottom_off: usize,
ss_hor: usize,
ss_ver: usize,
ds_flt: i32,
edges: u8,
ll_mask: &[[u16; 4]],
bitdepth_max: i32,
) {
let mut c_buffers = [[0u16; REST_UNIT_STRIDE]; 5];
let mut l_buffers = [[0u16; LUMA_BUF_STRIDE]; 5];
let mut c_ptrs: [usize; 5] = [0; 5];
let mut l_ptrs: [usize; 5] = [0; 5];
let o = ROW_ORIGIN;
let luma_w = w << ss_hor;
backup_row_hbd(&mut c_buffers[2], o, &*p, p_off, &left[0], 6, w, 2, edges);
c_ptrs[2] = 2;
if edges & (LR_HAVE_TOP_INTEGRATED | LR_HAVE_TOP) != 0 {
let mut loff = lpf_off;
for i in 0..2 {
backup_row_lpf_hbd(&mut c_buffers[i], o, lpf, loff, w, 2, edges);
c_ptrs[i] = i;
loff += stride;
}
} else {
c_ptrs[0] = 2;
c_ptrs[1] = 2;
}
backup_row_hbd(
&mut c_buffers[3],
o,
&*p,
p_off + stride,
&left[1],
6,
w,
2,
edges,
);
c_ptrs[3] = 3;
let mut bak_idx: usize = 4;
backup_row_luma_hbd(
&mut l_buffers[2],
o,
luma,
luma_off,
lstride,
luma_w,
edges,
ss_hor,
ss_ver,
ds_flt,
);
l_ptrs[2] = 2;
if edges & LR_HAVE_TOP_INTEGRATED != 0 {
backup_row_luma_hbd(
&mut l_buffers[0],
o,
luma,
luma_off - (2 << ss_ver) * lstride,
lstride,
luma_w,
edges,
ss_hor,
ss_ver,
ds_flt,
);
l_ptrs[0] = 0;
backup_row_luma_hbd(
&mut l_buffers[1],
o,
luma,
luma_off - (1 << ss_ver) * lstride,
lstride,
luma_w,
edges,
ss_hor,
ss_ver,
ds_flt,
);
l_ptrs[1] = 1;
} else if edges & LR_HAVE_TOP != 0 {
backup_row_luma_hbd(
&mut l_buffers[0],
o,
luma_top,
luma_top_off,
0,
luma_w,
edges,
ss_hor,
ss_ver,
ds_flt,
);
l_ptrs[0] = 0;
backup_row_luma_hbd(
&mut l_buffers[1],
o,
luma_top,
luma_top_off + if ss_ver == 0 { lstride } else { 0 },
lstride,
luma_w,
edges,
ss_hor,
ss_ver,
ds_flt,
);
l_ptrs[1] = 1;
} else {
l_ptrs[0] = 2;
l_ptrs[1] = 2;
}
backup_row_luma_hbd(
&mut l_buffers[3],
o,
luma,
luma_off + (1 << ss_ver) * lstride,
lstride,
luma_w,
edges,
ss_hor,
ss_ver,
ds_flt,
);
l_ptrs[3] = 3;
let mut lbak_idx: usize = 4;
let mut luma_pos = luma_off;
for y in 0..h {
if y + 2 < h {
backup_row_hbd(
&mut c_buffers[bak_idx],
o,
&*p,
p_off + (y + 2) * stride,
&left[y + 2],
6,
w,
2,
edges,
);
c_ptrs[4] = bak_idx;
} else if edges & LR_HAVE_BOTTOM_INTEGRATED != 0 {
backup_row_lpf_hbd(
&mut c_buffers[bak_idx],
o,
&*p,
p_off + (y + 2) * stride,
w,
2,
edges,
);
c_ptrs[4] = bak_idx;
} else if edges & LR_HAVE_BOTTOM != 0 {
let offset_y = y + 2 - h;
backup_row_lpf_hbd(
&mut c_buffers[bak_idx],
o,
lpf_bottom,
lpf_bottom_off + offset_y * stride,
w,
2,
edges,
);
c_ptrs[4] = bak_idx;
} else {
c_ptrs[4] = c_ptrs[3];
}
bak_idx += 1;
if bak_idx == 5 {
bak_idx = 0;
}
if c_ptrs[4] == c_ptrs[3] {
l_ptrs[4] = l_ptrs[3];
} else if y + 2 == h && edges & LR_HAVE_BOTTOM_INTEGRATED == 0 {
backup_row_luma_hbd(
&mut l_buffers[lbak_idx],
o,
luma_bottom,
luma_bottom_off,
lstride,
luma_w,
edges,
ss_hor,
ss_ver,
ds_flt,
);
l_ptrs[4] = lbak_idx;
} else if y + 1 == h && edges & LR_HAVE_BOTTOM_INTEGRATED == 0 {
backup_row_luma_hbd(
&mut l_buffers[lbak_idx],
o,
luma_bottom,
luma_bottom_off + lstride,
0,
luma_w,
edges,
ss_hor,
ss_ver,
ds_flt,
);
l_ptrs[4] = lbak_idx;
} else {
backup_row_luma_hbd(
&mut l_buffers[lbak_idx],
o,
luma,
luma_pos + (2 << ss_ver) * lstride,
lstride,
luma_w,
edges,
ss_hor,
ss_ver,
ds_flt,
);
l_ptrs[4] = lbak_idx;
}
lbak_idx += 1;
if lbak_idx == 5 {
lbak_idx = 0;
}
let c_refs: [&[u16]; 5] = core::array::from_fn(|i| &c_buffers[c_ptrs[i]] as &[u16]);
let l_refs: [&[u16]; 5] = core::array::from_fn(|i| &l_buffers[l_ptrs[i]] as &[u16]);
let lstep = 1usize << ss_hor;
let ctaps: [crate::filter::WienerTapHbd; 6] = core::array::from_fn(|i| {
let dy = WIENER_NS_CONFIG_UV[i][0] as i32;
let dx = WIENER_NS_CONFIG_UV[i][1] as i32;
crate::filter::WienerTapHbd {
row_p: c_refs[(2 + dy) as usize],
row_m: c_refs[(2 - dy) as usize],
dx,
coef: filter[i] as i32,
}
});
let ltaps: [crate::filter::UvLumaTapHbd; 12] = core::array::from_fn(|i| {
let dy = WIENER_NS_CONFIG_UV_FROM_Y[i][0] as i32;
let dx = WIENER_NS_CONFIG_UV_FROM_Y[i][1] as i32;
crate::filter::UvLumaTapHbd {
row: l_refs[(2 + dy) as usize],
ldx: dx * lstep as i32,
coef: filter[6 + i] as i32,
}
});
let skip_mask = ll_mask[y >> 2][0];
let dst_row = &mut p[p_off + y * stride..];
let bw = w >> 2;
let mut bx = 0usize;
while bx < bw {
if skip_mask & (1 << bx) != 0 {
bx += 1;
continue;
}
let bx_start = bx;
bx += 1;
while bx < bw && skip_mask & (1 << bx) == 0 {
bx += 1;
}
let x0 = bx_start << 2;
let n = (bx - bx_start) << 2;
unsafe {
(exec.ns_wiener_uv_fir_hbd)(
&mut dst_row[x0..x0 + n],
c_refs[2],
o + x0,
&ctaps,
l_refs[2],
o + (x0 << ss_hor),
<aps,
lstep,
n,
bitdepth_max,
);
}
}
for r in 0..4 {
c_ptrs[r] = c_ptrs[r + 1];
}
for r in 0..4 {
l_ptrs[r] = l_ptrs[r + 1];
}
luma_pos += lstride << ss_ver;
}
}
pub(crate) struct LrContextHbd<'a> {
pub(crate) exec: &'a crate::exec_context::ExecContext,
pub(crate) restoration_p: &'a [FhRestorationPlane; 3],
pub(crate) gdf_qp_idx: i32,
pub(crate) gdf_scale: i32,
pub(crate) gdf_ref_dst_idx: i32,
pub(crate) sb128: bool,
pub(crate) cfl_ds_filter_index: i32,
pub(crate) layout: PixelLayout,
pub(crate) bw: i32,
pub(crate) bh: i32,
pub(crate) sb256w: i32,
pub(crate) sbh: i32,
pub(crate) mask: &'a [Av2Filter],
pub(crate) lr_mask: &'a [Av2Restoration],
pub(crate) lr_db_line: &'a [Vec<u16>; 3],
pub(crate) lr_cdef_line: &'a [Vec<u16>; 3],
pub(crate) lf_p_luma: &'a [u16],
pub(crate) base_q: i32,
pub(crate) start_of_tile_row: &'a [u8],
pub(crate) ns_subclass_lut: &'a [u8],
pub(crate) pc_subclass_lut: &'a [u8],
pub(crate) pc_filters: &'a [[i16; 13]],
pub(crate) n_tc: i32,
pub(crate) inloop_filters: u32,
pub(crate) cur_stride: [isize; 2],
pub(crate) unit_size: [u8; 2],
pub(crate) restore_planes: i32,
pub(crate) bitdepth_min_8: i32,
pub(crate) bitdepth_max: i32,
pub(crate) col_start_sb: &'a [u16],
pub(crate) row_start_sb: &'a [u16],
pub(crate) sbsz: i32,
}
fn lr_stripe_hbd(
ctx: &LrContextHbd,
p: &mut [u16],
p_off: usize,
left: &[[u16; 6]],
x: i32,
mut y: i32,
plane: i32,
w: i32,
row_h: i32,
lr: &crate::lf_mask::Av2RestorationUnit,
mut edges: u8,
first_sby_in_tile_row: FirstSbInTileRow,
tile_row_m1: i32,
) {
let chroma = (plane != 0) as i32;
let ss_ver = chroma & (ctx.layout == PixelLayout::I420) as i32;
let ss_hor = chroma & (ctx.layout != PixelLayout::I444) as i32;
let stride = ctx.cur_stride[chroma as usize];
let abs_stride = stride.unsigned_abs();
let sby = (y + if y != 0 { 8 << ss_ver } else { 0 }) >> (6 - ss_ver + ctx.sb128 as i32);
let have_tt = (ctx.n_tc > 1) as i32;
let sb256x = (x << ss_hor) >> 8;
let sb64x_idx = ((x << ss_hor) >> 6) & 3;
let mut stripe_h = imin(
(64 - 8 * (first_sby_in_tile_row != FirstSbInTileRow::None) as i32) >> ss_ver,
row_h - y,
);
let ref_dst_idx = ctx.gdf_ref_dst_idx;
let qp_idx = ctx.gdf_qp_idx;
let gdf_scale = ctx.gdf_scale;
let wiener_type: u8 = if ctx.inloop_filters & INLOOPFILTER_WIENER != 0 {
lr.restoration_type
} else {
0 };
#[derive(PartialEq)]
enum WKind {
None,
NsSingle,
NsMulti,
PcWiener,
}
let mut multi_wiener = false;
let mut noskip_mask = [0u16; 66];
let pd = &ctx.restoration_p[plane as usize].ns;
let wkind = if wiener_type == RestorationType::NsWiener as u8 {
if pd.frame_filters_on != 0 {
if pd.num_classes == 1 {
WKind::NsSingle
} else {
multi_wiener = true;
WKind::NsMulti
}
} else {
WKind::NsSingle
}
} else if wiener_type == RestorationType::PcWiener as u8 {
multi_wiener = true;
WKind::PcWiener
} else {
WKind::None
};
if multi_wiener {
let mut r = 0usize;
let mut by = y >> 2;
while by < row_h >> 2 {
let by_idx = (by & 63) as usize;
let sb256_idx = ctx.sb256w as usize * (by as usize >> 6) + sb256x as usize;
if sb256_idx < ctx.mask.len() {
let noskip_row = &ctx.mask[sb256_idx].lr_noskip_mask[by_idx];
noskip_mask[r] = noskip_row[sb64x_idx as usize];
if edges & LR_HAVE_RIGHT == 0 && w & 63 != 0 {
let shift = ((w >> 2) & 15) - 1;
let mask_val = noskip_mask[r];
let edge_bit = mask_val >> shift as u16;
noskip_mask[r] |= edge_bit << ((shift + 1) as u16);
}
}
r += 1;
by += 1;
}
}
let lstride = ctx.cur_stride[0];
let lstride_u = lstride.unsigned_abs();
let stride_u = abs_stride;
let mut gdf_err = [0i8; 64 * 64];
let mut left_idx = 0usize;
let mut noskip_offset = 0usize;
let mut cur_off = p_off;
let mut lpf_off =
(have_tt as isize * (sby as isize * (4 << ctx.sb128 as i32) - 4) * stride_u as isize
+ x as isize) as usize;
let mut llpf_off =
(have_tt as isize * (sby as isize * (4 << ctx.sb128 as i32) - 4) * lstride_u as isize
+ (x << ss_hor) as isize) as usize;
while y + stripe_h <= row_h {
edges ^= ((-(((sby + 1) != ctx.sbh) as i32 | (y + stripe_h != row_h) as i32)) as u8
^ edges)
& LR_HAVE_BOTTOM;
let inc = if (edges & (LR_HAVE_TOP | LR_HAVE_TOP_INTEGRATED | LR_HAVE_BOTTOM_INTEGRATED))
== LR_HAVE_TOP
&& y + 8 < ((ctx.bh * 4) >> ss_ver)
{
8
} else {
0
};
let sb256_idx =
ctx.sb256w as usize * ((((y << ss_ver) + inc) as usize) >> 8) + sb256x as usize;
let gdf_enabled = plane == 0
&& ctx.inloop_filters & INLOOPFILTER_GDF != 0
&& sb256_idx < ctx.mask.len()
&& ctx.mask[sb256_idx].gdf[((((y + inc) >> 4) & 12) + sb64x_idx) as usize] != 0;
let plane_u = plane as usize;
let stripe_u = stripe_h as usize;
let w_u = w as usize;
let cdef_top: Option<usize> = if (edges & (LR_HAVE_TOP | LR_HAVE_TOP_INTEGRATED))
== (LR_HAVE_TOP | LR_HAVE_TOP_INTEGRATED)
{
let cdef_off = tile_row_m1 as usize * (6 - 4 * chroma as usize) * stride_u + x as usize;
let off = cdef_off + (2 * (1 - chroma) as usize) * stride_u;
if off < ctx.lr_cdef_line[plane_u].len() {
Some(off)
} else {
None
}
} else {
None
};
if gdf_enabled {
let (top_slice, top_off): (&[u16], usize) = match cdef_top {
Some(off) => (&ctx.lr_cdef_line[plane_u], off - 2 * stride_u),
None => (&ctx.lr_db_line[plane_u], lpf_off),
};
gdf_prep_hbd(
ctx.exec,
&mut gdf_err,
64,
&*p,
cur_off,
stride_u,
&left[left_idx..],
top_slice,
top_off,
&ctx.lr_db_line[plane_u],
lpf_off + 6 * stride_u,
w_u,
stripe_u,
ref_dst_idx as usize,
qp_idx as usize,
edges,
ctx.bitdepth_max,
);
}
let y4 = (((y << ss_ver) & 255) >> 2) as usize;
let ll_rows = (stripe_h >> 2).max(1) as usize + 1;
let mut ll_mask_storage = [[0u16; 4]; 17];
let ll_mask_buf = &mut ll_mask_storage[..ll_rows.min(17)];
if sb256_idx < ctx.mask.len() {
let m = &ctx.mask[sb256_idx];
if plane == 0 || ss_hor == 0 {
let src = if plane == 0 {
&m.lossless_mask_y
} else {
&m.lossless_mask_uv
};
for (r, slot) in ll_mask_buf.iter_mut().enumerate() {
let yy = y4 + r;
if yy < 64 {
let row = &src[yy];
for k in 0..4 {
if sb64x_idx as usize + k < 4 {
slot[k] = row[sb64x_idx as usize + k];
}
}
}
}
} else {
let init_y = (y >> 2) as usize;
let end_y = ((y + stripe_h) >> 2) as usize;
let base = (y4 >> ss_ver) as usize;
for (r, yy) in (init_y..end_y).enumerate() {
let src_y = base + (yy - init_y);
if src_y < 64 && r < ll_mask_buf.len() {
let row = &m.lossless_mask_uv[src_y];
let lo = row[sb64x_idx as usize];
let hi = if sb64x_idx as usize + 1 < 4 {
row[sb64x_idx as usize + 1]
} else {
0
};
ll_mask_buf[r][0] = (lo & 0xff) | ((hi & 0xff) << 8);
}
}
}
}
if wkind != WKind::None {
let (top_slice, top_off): (&[u16], usize) = match cdef_top {
Some(off) => (&ctx.lr_cdef_line[plane_u], off),
None => (&ctx.lr_db_line[plane_u], lpf_off),
};
let db_line = &ctx.lr_db_line[plane_u];
let bottom_off = lpf_off + 6 * stride_u;
match wkind {
WKind::NsSingle if chroma == 0 => {
let mut filter = [0i8; 16];
let src = if pd.frame_filters_on != 0 {
&pd.filter[0][..16]
} else {
&lr.ns_filter[0][..16]
};
filter.copy_from_slice(src);
let (ts, to): (&[u16], usize) = if cdef_top.is_some() {
(top_slice, top_off)
} else {
(db_line, lpf_off)
};
ns_wiener_single_y_hbd(
ctx.exec,
p,
cur_off,
stride_u,
&left[left_idx..],
ts,
to,
db_line,
bottom_off,
w_u,
stripe_u,
&filter,
edges,
ll_mask_buf,
ctx.bitdepth_max,
);
}
WKind::NsSingle => {
let mut filter = [0i8; 18];
let src = if pd.frame_filters_on != 0 {
&pd.filter[0][..18]
} else {
&lr.ns_filter[0][..18]
};
filter.copy_from_slice(src);
let luma = ctx.lf_p_luma;
let luma_off = (x << ss_hor) as usize + (y << ss_ver) as usize * lstride_u;
let luma_db = &ctx.lr_db_line[0];
let (ts, to): (&[u16], usize) = if cdef_top.is_some() {
(top_slice, top_off)
} else {
(db_line, lpf_off)
};
ns_wiener_single_uv_hbd(
ctx.exec,
p,
cur_off,
stride_u,
&left[left_idx..],
ts,
to,
db_line,
bottom_off,
w_u,
stripe_u,
&filter,
luma,
luma_off,
lstride_u,
luma_db,
llpf_off,
luma_db,
llpf_off + 6 * lstride_u,
ss_hor as usize,
ss_ver as usize,
ctx.cfl_ds_filter_index,
edges,
ll_mask_buf,
ctx.bitdepth_max,
);
}
WKind::NsMulti => {
let mut filters_owned = [[0i8; 18]; 16];
let filters: &[[i8; 18]] = if pd.frame_filters_on != 0 {
&pd.filter
} else {
for (dst, src) in filters_owned.iter_mut().zip(lr.ns_filter.iter()) {
dst.copy_from_slice(&src[..18]);
}
&filters_owned
};
let (ts, to): (&[u16], usize) = if cdef_top.is_some() {
(top_slice, top_off)
} else {
(db_line, lpf_off)
};
ns_wiener_multi_hbd(
ctx.exec,
p,
cur_off,
stride_u,
&left[left_idx..],
ts,
to,
db_line,
bottom_off,
w_u,
stripe_u,
filters,
ctx.ns_subclass_lut,
&noskip_mask[noskip_offset..],
ctx.base_q,
edges,
ll_mask_buf,
ctx.bitdepth_min_8,
ctx.bitdepth_max,
);
noskip_offset += (stripe_h >> 2) as usize;
}
WKind::PcWiener => {
let (ts, to): (&[u16], usize) = if cdef_top.is_some() {
(top_slice, top_off)
} else {
(db_line, lpf_off)
};
pc_wiener_hbd(
ctx.exec,
p,
cur_off,
stride_u,
&left[left_idx..],
ts,
to,
db_line,
bottom_off,
w_u,
stripe_u,
ctx.pc_filters,
ctx.pc_subclass_lut,
&noskip_mask[noskip_offset..],
ctx.base_q,
edges,
ll_mask_buf,
ctx.bitdepth_min_8,
ctx.bitdepth_max,
);
noskip_offset += (stripe_h >> 2) as usize;
}
WKind::None => {}
}
}
if gdf_enabled {
gdf_add_hbd(
ctx.exec,
&mut p[cur_off..],
stride_u,
&gdf_err,
64,
w_u,
stripe_u,
gdf_scale,
ll_mask_buf,
ctx.bitdepth_max,
);
}
edges &= !(LR_HAVE_BOTTOM_INTEGRATED | LR_HAVE_TOP_INTEGRATED);
left_idx += stripe_h as usize;
cur_off += stripe_u * stride_u;
y += stripe_h;
edges |= LR_HAVE_TOP;
stripe_h = imin(64 >> ss_ver, row_h - y);
if stripe_h == 0 {
break;
}
lpf_off += 4 * stride_u;
llpf_off += 4 * lstride_u;
}
}
#[allow(clippy::too_many_arguments)]
fn lr_unit_at_hbd(
ctx: &LrContextHbd,
pos: i32,
w: i32,
unit_size: i32,
ss_hor: i32,
sb_idx: i32,
unit_idx_base: i32,
) -> (usize, usize) {
let shift_hor = 8 - ss_hor;
let mut start = pos & !(unit_size - 1);
let mut tc = 0usize;
while tc + 2 < ctx.col_start_sb.len()
&& start >= (ctx.col_start_sb[tc + 1] as i32 * ctx.sbsz) >> ss_hor
{
tc += 1;
}
let tile_col_start_x = (ctx.col_start_sb[tc] as i32 * ctx.sbsz) >> ss_hor;
if start > tile_col_start_x && w - start < unit_size >> 1 {
start -= unit_size;
}
(
(sb_idx + (start >> shift_hor)) as usize,
(unit_idx_base + ((start >> (shift_hor - 2)) & 3)) as usize,
)
}
#[allow(clippy::too_many_arguments)]
fn lr_chunk_hbd(
ctx: &LrContextHbd,
p: &mut [u16],
p_off: usize,
left: &[[u16; 6]],
x: i32,
y: i32,
plane: i32,
width: i32,
row_h: i32,
edges: u8,
first_sby_in_tile_row: FirstSbInTileRow,
tile_row_m1: i32,
first_unit: (usize, usize),
w: i32,
unit_size: i32,
ss_hor: i32,
sb_idx: i32,
unit_idx_base: i32,
) {
type Px = u16;
let plane_u = plane as usize;
let unit_ok =
|(sb, u): (usize, usize)| sb < ctx.lr_mask.len() && u < ctx.lr_mask[sb].lr[plane_u].len();
let mut segs: [(i32, (usize, usize)); 4] = [(0, first_unit); 4];
let mut n_segs = 1usize;
let mut off = unit_size;
while unit_size < width && off < width && n_segs < 4 {
let unit = lr_unit_at_hbd(ctx, x + off, w, unit_size, ss_hor, sb_idx, unit_idx_base);
if unit != segs[n_segs - 1].1 {
segs[n_segs] = (off, unit);
n_segs += 1;
}
off += unit_size;
}
if n_segs == 1 {
if unit_ok(first_unit) {
let lr_unit = &ctx.lr_mask[first_unit.0].lr[plane_u][first_unit.1];
lr_stripe_hbd(
ctx,
p,
p_off,
left,
x,
y,
plane,
width,
row_h,
lr_unit,
edges,
first_sby_in_tile_row,
tile_row_m1,
);
}
return;
}
let abs_stride = ctx.cur_stride[(plane != 0) as usize].unsigned_abs();
let n = if plane != 0 { 2 } else { 6 };
let rows = (row_h - y).max(0) as usize;
let mut borders = [[[Px::default(); 6]; 264]; 3];
for (k, border) in borders.iter_mut().enumerate().take(n_segs - 1) {
let col = p_off + segs[k + 1].0 as usize;
for (row, b) in border.iter_mut().enumerate().take(rows) {
let at = col + row * abs_stride;
if at >= n && at <= p.len() {
b[6 - n..].copy_from_slice(&p[at - n..at]);
}
}
}
for k in 0..n_segs {
let (seg_x, unit) = segs[k];
let seg_w = if k + 1 < n_segs { segs[k + 1].0 } else { width } - seg_x;
let mut seg_edges = edges;
if k > 0 {
seg_edges |= LR_HAVE_LEFT;
}
if k + 1 < n_segs {
seg_edges |= LR_HAVE_RIGHT;
}
if !unit_ok(unit) {
continue;
}
let seg_left: &[[Px; 6]] = if k == 0 { left } else { &borders[k - 1] };
let lr_unit = &ctx.lr_mask[unit.0].lr[plane_u][unit.1];
lr_stripe_hbd(
ctx,
p,
p_off + seg_x as usize,
seg_left,
x + seg_x,
y,
plane,
seg_w,
row_h,
lr_unit,
seg_edges,
first_sby_in_tile_row,
tile_row_m1,
);
}
}
fn lr_sbrow_hbd_inner(
ctx: &LrContextHbd,
p: &mut [u16],
p_off: usize,
y: i32,
w: i32,
h: i32,
row_h: i32,
plane: i32,
first_sby_in_tile_row: FirstSbInTileRow,
tile_row_m1: i32,
) {
let chroma = (plane != 0) as i32;
let ss_ver = chroma & (ctx.layout == PixelLayout::I420) as i32;
let ss_hor = chroma & (ctx.layout != PixelLayout::I444) as i32;
let p_stride = ctx.cur_stride[chroma as usize];
let abs_stride = p_stride.unsigned_abs();
let unit_size_log2 = ctx.unit_size[chroma.min(1) as usize] as i32;
let unit_size = 1 << unit_size_log2;
let half_unit_size = unit_size >> 1;
let row_y = y + ((8 >> ss_ver) * (y != 0) as i32);
let shift_hor = 8 - ss_hor;
let mut pre_lr_border = [[[0u16; 6]; 264]; 2];
let probe = if first_sby_in_tile_row == FirstSbInTileRow::Top {
y
} else {
row_y
};
let tile_row_px = |i: usize| (ctx.row_start_sb[i] as i32 * ctx.sbsz) >> ss_ver;
let mut tr = 0usize;
while tr + 2 < ctx.row_start_sb.len() && probe >= tile_row_px(tr + 1) {
tr += 1;
}
let tile_top = tile_row_px(tr);
let tile_bot = imin(tile_row_px(tr + 1), h);
let mut aligned_unit_pos = tile_top + ((probe - tile_top) & !(unit_size - 1));
if aligned_unit_pos > tile_top && aligned_unit_pos + half_unit_size > tile_bot {
aligned_unit_pos -= unit_size;
}
aligned_unit_pos <<= ss_ver;
let sb_idx = (aligned_unit_pos >> 8) * ctx.sb256w;
let unit_idx_base = ((aligned_unit_pos >> 6) & 0x3) << 2;
let mut edges: u8 = if y > 0 { LR_HAVE_TOP } else { 0 } | LR_HAVE_RIGHT;
if first_sby_in_tile_row == FirstSbInTileRow::Top {
edges |= LR_HAVE_BOTTOM_INTEGRATED;
}
if first_sby_in_tile_row == FirstSbInTileRow::Bottom && y > 0 {
edges |= LR_HAVE_TOP_INTEGRATED;
}
let mut lr_idx_0 = sb_idx as usize;
let mut lr_unit_0 = unit_idx_base as usize;
let mut bit = 0usize;
let mut x = 0i32;
let mut cur_p_off = p_off;
let mut tile_col = 0usize;
while x + 64 < w {
let next_x = x + 64;
let mut next_iter_lru_start_x = next_x & !(unit_size - 1);
while tile_col + 2 < ctx.col_start_sb.len()
&& next_iter_lru_start_x >= (ctx.col_start_sb[tile_col + 1] as i32 * ctx.sbsz) >> ss_hor
{
tile_col += 1;
}
let tile_col_start_x = (ctx.col_start_sb[tile_col] as i32 * ctx.sbsz) >> ss_hor;
if next_iter_lru_start_x > tile_col_start_x && w - next_iter_lru_start_x < half_unit_size {
next_iter_lru_start_x -= unit_size;
}
let next_u_idx = unit_idx_base + ((next_iter_lru_start_x >> (shift_hor - 2)) & 3);
let next_sb_idx = sb_idx + (next_iter_lru_start_x >> shift_hor);
let n = if plane != 0 { 2 } else { 6 };
let border_src_off = cur_p_off + 64;
if border_src_off + n <= p.len() {
for row in 0..(row_h - y) as usize {
let src_row = border_src_off + row * abs_stride;
if src_row >= n && src_row < p.len() {
let start = src_row - n;
for i in 0..n.min(6) {
pre_lr_border[bit][row][6 - n + i] = p[start + i];
}
}
}
}
let lr_sb = lr_idx_0;
let lr_u = lr_unit_0;
lr_chunk_hbd(
ctx,
p,
cur_p_off,
&pre_lr_border[1 - bit],
x,
y,
plane,
64,
row_h,
edges,
first_sby_in_tile_row,
tile_row_m1,
(lr_sb, lr_u),
w,
unit_size,
ss_hor,
sb_idx,
unit_idx_base,
);
lr_idx_0 = next_sb_idx as usize;
lr_unit_0 = next_u_idx as usize;
x = next_x;
cur_p_off += 64;
edges |= LR_HAVE_LEFT;
bit ^= 1;
}
edges &= !LR_HAVE_RIGHT;
let end_w = w - x;
let lr_sb = lr_idx_0;
let lr_u = lr_unit_0;
lr_chunk_hbd(
ctx,
p,
cur_p_off,
&pre_lr_border[1 - bit],
x,
y,
plane,
end_w,
row_h,
edges,
first_sby_in_tile_row,
tile_row_m1,
(lr_sb, lr_u),
w,
unit_size,
ss_hor,
sb_idx,
unit_idx_base,
);
}
pub(crate) fn lr_sbrow_hbd(ctx: &LrContextHbd, dst: &mut [&mut [u16]; 3], sby: i32) {
let dst_stride = ctx.cur_stride;
let mut restore_planes = ctx.restore_planes;
if dst[1].is_empty() || dst[2].is_empty() {
restore_planes &= LR_RESTORE_Y;
}
let not_last = (sby + 1 < ctx.sbh) as i32;
let start_tile_val = if (sby as usize) < ctx.start_of_tile_row.len() {
ctx.start_of_tile_row[sby as usize]
} else {
0
};
let tile_row = (start_tile_val >> 1) as i32;
let first_sby_in_tile_row_flag = start_tile_val & 1;
if restore_planes & (LR_RESTORE_U | LR_RESTORE_V) != 0 {
let ss_ver = (ctx.layout == PixelLayout::I420) as i32;
let ss_hor = (ctx.layout != PixelLayout::I444) as i32;
let h = (ctx.bh * 4) >> ss_ver;
let w = (ctx.bw * 4) >> ss_hor;
let next_row_y = (sby + 1) << ((6 - ss_ver) + ctx.sb128 as i32);
let row_h = imin(next_row_y - (8 >> ss_ver) * not_last, h);
let offset_uv = (8 * (sby != 0) as i32) >> ss_ver;
let mut y_stripe = (sby << ((6 - ss_ver) + ctx.sb128 as i32)) - offset_uv;
if sby != 0 && first_sby_in_tile_row_flag != 0 {
let first_top = if first_sby_in_tile_row_flag != 0 {
FirstSbInTileRow::Top
} else {
FirstSbInTileRow::None
};
if restore_planes & LR_RESTORE_U != 0 {
let abs_stride_uv = dst_stride[1].unsigned_abs();
lr_sbrow_hbd_inner(
ctx,
dst[1],
(y_stripe.max(0) as usize) * abs_stride_uv,
y_stripe,
w,
h,
y_stripe + (8 >> ss_ver),
1,
first_top,
tile_row - 1,
);
}
if restore_planes & LR_RESTORE_V != 0 {
let abs_stride_uv = dst_stride[1].unsigned_abs();
lr_sbrow_hbd_inner(
ctx,
dst[2],
(y_stripe.max(0) as usize) * abs_stride_uv,
y_stripe,
w,
h,
y_stripe + (8 >> ss_ver),
2,
first_top,
tile_row - 1,
);
}
y_stripe += 8 >> ss_ver;
}
let first_bot = if first_sby_in_tile_row_flag != 0 {
FirstSbInTileRow::Bottom
} else {
FirstSbInTileRow::None
};
if restore_planes & LR_RESTORE_U != 0 {
let abs_stride_uv = dst_stride[1].unsigned_abs();
lr_sbrow_hbd_inner(
ctx,
dst[1],
(y_stripe.max(0) as usize) * abs_stride_uv,
y_stripe,
w,
h,
row_h,
1,
first_bot,
tile_row - 1,
);
}
if restore_planes & LR_RESTORE_V != 0 {
let abs_stride_uv = dst_stride[1].unsigned_abs();
lr_sbrow_hbd_inner(
ctx,
dst[2],
(y_stripe.max(0) as usize) * abs_stride_uv,
y_stripe,
w,
h,
row_h,
2,
first_bot,
tile_row - 1,
);
}
}
if restore_planes & LR_RESTORE_Y != 0 {
let h = ctx.bh * 4;
let w = ctx.bw * 4;
let next_row_y = (sby + 1) << (6 + ctx.sb128 as i32);
let row_h = imin(next_row_y - 8 * not_last, h);
let offset_y = 8 * (sby != 0) as i32;
let mut y_stripe = (sby << (6 + ctx.sb128 as i32)) - offset_y;
if sby != 0 && first_sby_in_tile_row_flag != 0 {
let abs_stride_y = dst_stride[0].unsigned_abs();
lr_sbrow_hbd_inner(
ctx,
dst[0],
(y_stripe.max(0) as usize) * abs_stride_y,
y_stripe,
w,
h,
y_stripe + 8,
0,
FirstSbInTileRow::Top,
tile_row - 1,
);
y_stripe += 8;
}
let abs_stride_y = dst_stride[0].unsigned_abs();
lr_sbrow_hbd_inner(
ctx,
dst[0],
(y_stripe.max(0) as usize) * abs_stride_y,
y_stripe,
w,
h,
row_h,
0,
if first_sby_in_tile_row_flag != 0 {
FirstSbInTileRow::Bottom
} else {
FirstSbInTileRow::None
},
tile_row - 1,
);
}
}
#[cfg(test)]
mod avm_looprestoration_shape_tests {
use super::*;
#[test]
fn restoration_type_ids_match_avm_lr_syntax() {
assert_eq!(RestorationType::None as u8, 0);
assert_eq!(RestorationType::PcWiener as u8, 1);
assert_eq!(RestorationType::NsWiener as u8, 2);
assert_eq!(RestorationType::Switchable as u8, 3);
assert_eq!(
crate::headers::RestorationType::None as u8,
RestorationType::None as u8
);
assert_eq!(
crate::headers::RestorationType::PcWiener as u8,
RestorationType::PcWiener as u8,
);
assert_eq!(
crate::headers::RestorationType::NsWiener as u8,
RestorationType::NsWiener as u8,
);
assert_eq!(
crate::headers::RestorationType::Switchable as u8,
RestorationType::Switchable as u8,
);
}
#[test]
fn wiener_nonsep_and_pc_tap_shapes_match_avm_symmetric_form() {
assert_eq!(
WIENER_NS_CONFIG_Y,
[
[1, 0],
[0, 1],
[2, 0],
[0, 2],
[1, 1],
[-1, 1],
[2, 1],
[2, -1],
[1, 2],
[1, -2],
[3, 0],
[0, 3],
[4, 0],
[0, 4],
[3, 3],
[3, -3],
]
);
assert_eq!(
WIENER_NS_CONFIG_UV,
[[1, 0], [0, 1], [1, 1], [-1, 1], [2, 0], [0, 2]]
);
assert_eq!(
WIENER_NS_CONFIG_UV_FROM_Y,
[
[1, 0],
[-1, 0],
[0, 1],
[0, -1],
[1, 1],
[-1, -1],
[-1, 1],
[1, -1],
[2, 0],
[-2, 0],
[0, 2],
[0, -2],
]
);
assert_eq!(&PC_WIENER_CONFIG[..], &WIENER_NS_CONFIG_Y[..12]);
}
}
#[cfg(test)]
mod pcw_feature_tests {
use super::*;
#[test]
fn pcw_features_simd_matches_scalar() {
let mut s = 0xfeed_u64;
let mut r = move |max: u32| {
s = s
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
((s >> 33) as u32) % (max + 1)
};
for &bits in &[8u32, 10, 12] {
let max = (1 << bits) - 1;
let len = 80;
let data: Vec<Vec<u16>> = (0..10)
.map(|_| (0..len).map(|_| r(max) as u16).collect())
.collect();
let rows16: Vec<&[u16]> = data.iter().map(|v| &v[..]).collect();
let data8: Vec<Vec<u8>> = data
.iter()
.map(|v| v.iter().map(|&x| x as u8).collect())
.collect();
let rows8: Vec<&[u8]> = data8.iter().map(|v| &v[..]).collect();
for x0 in 1..len - 6 {
if bits == 8 {
assert_eq!(
pcw_features(&rows8, 4, x0),
pcw_features_scalar(&rows8, 4, x0),
"8b x0={x0}"
);
}
assert_eq!(
pcw_features(&rows16, 4, x0),
pcw_features_scalar(&rows16, 4, x0),
"{bits}b x0={x0}"
);
}
}
}
}