use crate::dip_tables::DIP_WEIGHTS;
use crate::intops::ulog2;
use crate::levels::{ANGLE_IBP_FLAG, ANGLE_MULTI_MRL_FLAG};
use crate::tables::SM_WEIGHTS;
use std::arch::x86_64::*;
#[inline(always)]
fn load_u16x16(a: &[u16; 16]) -> __m256i {
unsafe { _mm256_loadu_si256(a.as_ptr() as *const __m256i) }
}
#[inline(always)]
fn load_u16x8(a: &[u16; 8]) -> __m128i {
unsafe { _mm_loadu_si128(a.as_ptr().cast()) }
}
#[inline(always)]
fn store_u16x16(a: &mut [u16; 16], v: __m256i) {
unsafe { _mm256_storeu_si256(a.as_mut_ptr() as *mut __m256i, v) };
}
#[inline(always)]
fn splat_u16x64_avx2(a: &mut [u16; 64], v: __m256i) {
let (chunks, _) = a.as_chunks_mut::<16>();
store_u16x16(&mut chunks[0], v);
store_u16x16(&mut chunks[1], v);
store_u16x16(&mut chunks[2], v);
store_u16x16(&mut chunks[3], v);
}
#[inline(always)]
fn splat_row_u16_avx2_with_v(row: &mut [u16], v: u16, vv: __m256i, vv8: __m128i) {
let (chunks64, rem64) = row.as_chunks_mut::<64>();
for c in chunks64.iter_mut() {
splat_u16x64_avx2(c, vv);
}
let (chunks16, rem16) = rem64.as_chunks_mut::<16>();
for c in chunks16.iter_mut() {
store_u16x16(c, vv);
}
let (chunks8, rem) = rem16.as_chunks_mut::<8>();
for c in chunks8.iter_mut() {
store_u16x8(c, vv8);
}
for d in rem.iter_mut() {
*d = v;
}
}
#[inline(always)]
fn store_u16x8(a: &mut [u16; 8], v: __m128i) {
unsafe { _mm_storeu_si128(a.as_mut_ptr().cast(), v) };
}
#[inline]
#[target_feature(enable = "avx2")]
fn splat_row_u16_avx2(row: &mut [u16], v: u16) {
let vv = _mm256_set1_epi16(v as i16);
let vv8 = _mm256_castsi256_si128(vv);
splat_row_u16_avx2_with_v(row, v, vv, vv8);
}
#[inline]
#[target_feature(enable = "avx2")]
fn splat_h_rows4_u16_avx2(dst: &mut [u16], stride: usize, off: usize, w: usize, v: [u16; 4]) {
let block = &mut dst[off..off + 3 * stride + w];
let (row0, rest) = block.split_at_mut(stride);
let (row1, rest) = rest.split_at_mut(stride);
let (row2, row3) = rest.split_at_mut(stride);
let row0 = &mut row0[..w];
let row1 = &mut row1[..w];
let row2 = &mut row2[..w];
let row3 = &mut row3[..w];
let v0 = _mm256_set1_epi16(v[0] as i16);
let v1 = _mm256_set1_epi16(v[1] as i16);
let v2 = _mm256_set1_epi16(v[2] as i16);
let v3 = _mm256_set1_epi16(v[3] as i16);
let (r0_64, r0_rem64) = row0.as_chunks_mut::<64>();
let (r1_64, r1_rem64) = row1.as_chunks_mut::<64>();
let (r2_64, r2_rem64) = row2.as_chunks_mut::<64>();
let (r3_64, r3_rem64) = row3.as_chunks_mut::<64>();
for (((r0, r1), r2), r3) in r0_64
.iter_mut()
.zip(r1_64.iter_mut())
.zip(r2_64.iter_mut())
.zip(r3_64.iter_mut())
{
splat_u16x64_avx2(r0, v0);
splat_u16x64_avx2(r1, v1);
splat_u16x64_avx2(r2, v2);
splat_u16x64_avx2(r3, v3);
}
let (r0_16, r0_rem16) = r0_rem64.as_chunks_mut::<16>();
let (r1_16, r1_rem16) = r1_rem64.as_chunks_mut::<16>();
let (r2_16, r2_rem16) = r2_rem64.as_chunks_mut::<16>();
let (r3_16, r3_rem16) = r3_rem64.as_chunks_mut::<16>();
for (((r0, r1), r2), r3) in r0_16
.iter_mut()
.zip(r1_16.iter_mut())
.zip(r2_16.iter_mut())
.zip(r3_16.iter_mut())
{
store_u16x16(r0, v0);
store_u16x16(r1, v1);
store_u16x16(r2, v2);
store_u16x16(r3, v3);
}
let v0_8 = _mm256_castsi256_si128(v0);
let v1_8 = _mm256_castsi256_si128(v1);
let v2_8 = _mm256_castsi256_si128(v2);
let v3_8 = _mm256_castsi256_si128(v3);
let (r0_8, r0_rem) = r0_rem16.as_chunks_mut::<8>();
let (r1_8, r1_rem) = r1_rem16.as_chunks_mut::<8>();
let (r2_8, r2_rem) = r2_rem16.as_chunks_mut::<8>();
let (r3_8, r3_rem) = r3_rem16.as_chunks_mut::<8>();
for (((r0, r1), r2), r3) in r0_8
.iter_mut()
.zip(r1_8.iter_mut())
.zip(r2_8.iter_mut())
.zip(r3_8.iter_mut())
{
store_u16x8(r0, v0_8);
store_u16x8(r1, v1_8);
store_u16x8(r2, v2_8);
store_u16x8(r3, v3_8);
}
for d in r0_rem.iter_mut() {
*d = v[0];
}
for d in r1_rem.iter_mut() {
*d = v[1];
}
for d in r2_rem.iter_mut() {
*d = v[2];
}
for d in r3_rem.iter_mut() {
*d = v[3];
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avg_pred_hbd_avx2(dst: &mut [u16], stride: usize, tmp: &[u16], w: usize, h: usize) {
for y in 0..h {
let dst_row = &mut dst[y * stride..y * stride + w];
let tmp_row = &tmp[y * 64..y * 64 + w];
let (d16, drem) = dst_row.as_chunks_mut::<16>();
for (d, t) in d16.iter_mut().zip(tmp_row.as_chunks::<16>().0.iter()) {
store_u16x16(d, _mm256_avg_epu16(load_u16x16(d), load_u16x16(t)));
}
let base = d16.len() * 16;
for (i, d) in drem.iter_mut().enumerate() {
*d = ((*d as u32 + tmp_row[base + i] as u32 + 1) >> 1) as u16;
}
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn ibp_blend_hbd_avx2(
dst: &mut [u16],
stride: usize,
tmp: &[u16],
w: usize,
h: usize,
inv: bool,
weights: &[[u8; 16]; 16],
bitdepth_max: u16,
) {
let x_shift = w >> 5;
let y_shift = h >> 5;
let c128 = _mm256_set1_epi32(128);
let c64 = _mm256_set1_epi32(64);
let mut wrow = [0u16; 128];
for y in 0..h {
let wy = y >> y_shift;
for (x, weight) in wrow[..w].iter_mut().enumerate() {
let wx = x >> x_shift;
*weight = (if inv {
weights[wx][wy]
} else {
weights[wy][wx]
}) as u16;
}
let dst_row = &mut dst[y * stride..y * stride + w];
let tmp_row = &tmp[y * 64..y * 64 + w];
let (dst8, dst_tail) = dst_row.as_chunks_mut::<8>();
let (tmp8, tmp_tail) = tmp_row.as_chunks::<8>();
let (w8, w_tail) = wrow[..w].as_chunks::<8>();
for ((d, t), weight) in dst8.iter_mut().zip(tmp8.iter()).zip(w8.iter()) {
let wv = load_u16x8_i32_slice(weight);
let dv = load_u16x8_i32_slice(d);
let tv = load_u16x8_i32_slice(t);
let acc = _mm256_add_epi32(
_mm256_add_epi32(
_mm256_mullo_epi32(tv, _mm256_sub_epi32(c128, wv)),
_mm256_mullo_epi32(dv, wv),
),
c64,
);
store_i32x8_u16_max(d, _mm256_srli_epi32::<7>(acc), bitdepth_max);
}
for ((d, &t), &weight) in dst_tail.iter_mut().zip(tmp_tail.iter()).zip(w_tail.iter()) {
let weight = weight as u32;
let t = t as u32;
let d0 = *d as u32;
*d = ((t * (128 - weight) + d0 * weight + 64) >> 7).min(bitdepth_max as u32) as u16;
}
}
}
#[inline(always)]
const fn shuffle(z: u32, y: u32, x: u32, w: u32) -> i32 {
((z << 6) | (y << 4) | (x << 2) | w) as i32
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn _mm_hsumv_epi32(v: __m128i) -> __m128i {
let mut hi = _mm_shuffle_epi32::<{ shuffle(0, 0, 3, 2) }>(v);
let mut v = _mm_add_epi32(v, hi);
hi = _mm_shuffle_epi32::<{ shuffle(0, 0, 0, 1) }>(v);
v = _mm_add_epi32(v, hi);
v
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn _mm256_hsum_epi32(v: __m256i) -> u32 {
let lo = _mm_hsumv_epi32(_mm256_castsi256_si128(v));
let hi = _mm_hsumv_epi32(_mm256_extracti128_si256::<1>(v));
_mm_cvtsi128_si32(_mm_add_epi32(lo, hi)) as u32
}
#[target_feature(enable = "avx2")]
fn sum_u16_avx2(s: &[u16]) -> u32 {
let mut acc = _mm256_setzero_si256();
let (chunks, rem) = s.as_chunks::<16>();
for c in chunks.iter() {
let v = load_u16x16(c);
acc = _mm256_add_epi32(acc, _mm256_cvtepu16_epi32(_mm256_castsi256_si128(v)));
acc = _mm256_add_epi32(acc, _mm256_cvtepu16_epi32(_mm256_extracti128_si256::<1>(v)));
}
let mut total = _mm256_hsum_epi32(acc);
for &v in rem {
total += v as u32;
}
total
}
#[target_feature(enable = "avx2")]
fn splat_fill_avx2(dst: &mut [u16], stride: usize, off: usize, w: usize, h: usize, dc: u16) {
let vv = _mm256_set1_epi16(dc as i16);
let vv8 = _mm256_castsi256_si128(vv);
let mut p = off;
let mut rows = h;
while rows >= 4 {
splat_row_u16_avx2_with_v(&mut dst[p..p + w], dc, vv, vv8);
p += stride;
splat_row_u16_avx2_with_v(&mut dst[p..p + w], dc, vv, vv8);
p += stride;
splat_row_u16_avx2_with_v(&mut dst[p..p + w], dc, vv, vv8);
p += stride;
splat_row_u16_avx2_with_v(&mut dst[p..p + w], dc, vv, vv8);
p += stride;
rows -= 4;
}
for _ in 0..rows {
splat_row_u16_avx2_with_v(&mut dst[p..p + w], dc, vv, vv8);
p += stride;
}
}
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn dip_load_u16x4_i32_avx2(src: *const u16) -> __m128i {
unsafe { _mm_cvtepu16_epi32(_mm_loadl_epi64(src.cast())) }
}
#[inline]
#[target_feature(enable = "avx2")]
fn dip_dot_hbd_avx2(inp: &[i32; 11], weights: &[u16; 11]) -> i32 {
let inp8 = unsafe { _mm256_loadu_si256(inp.as_ptr().cast()) };
let w8 = _mm256_cvtepu16_epi32(unsafe { _mm_loadu_si128(weights.as_ptr().cast()) });
let mut s = _mm256_hsum_epi32(_mm256_mullo_epi32(inp8, w8)) as i32;
s += weights[8] as i32 * inp[8];
s += weights[9] as i32 * inp[9];
s += weights[10] as i32 * inp[10];
s
}
#[inline]
#[target_feature(enable = "avx2")]
fn dip_dot2_block4_hbd_avx2(
mut acc: __m128i,
inp: &[i32; 11],
w0: &[u16; 11],
w1: &[u16; 11],
base: usize,
) -> __m128i {
let r0 = unsafe { dip_load_u16x4_i32_avx2(w0.as_ptr().add(base)) };
let r1 = unsafe { dip_load_u16x4_i32_avx2(w1.as_ptr().add(base)) };
let z = _mm_setzero_si128();
let t01 = _mm_unpacklo_epi32(r0, r1);
let t23 = _mm_unpackhi_epi32(r0, r1);
let c0 = _mm_unpacklo_epi64(t01, z);
let c1 = _mm_unpackhi_epi64(t01, z);
let c2 = _mm_unpacklo_epi64(t23, z);
let c3 = _mm_unpackhi_epi64(t23, z);
acc = _mm_add_epi32(acc, _mm_mullo_epi32(c0, _mm_set1_epi32(inp[base])));
acc = _mm_add_epi32(acc, _mm_mullo_epi32(c1, _mm_set1_epi32(inp[base + 1])));
acc = _mm_add_epi32(acc, _mm_mullo_epi32(c2, _mm_set1_epi32(inp[base + 2])));
_mm_add_epi32(acc, _mm_mullo_epi32(c3, _mm_set1_epi32(inp[base + 3])))
}
#[inline]
#[target_feature(enable = "avx2")]
fn dip_dot2_hbd_avx2(inp: &[i32; 11], w0: &[u16; 11], w1: &[u16; 11]) -> __m128i {
let mut acc = _mm_setzero_si128();
acc = dip_dot2_block4_hbd_avx2(acc, inp, w0, w1, 0);
acc = dip_dot2_block4_hbd_avx2(acc, inp, w0, w1, 4);
for i in 8..11 {
let w = _mm_setr_epi32(w0[i] as i32, w1[i] as i32, 0, 0);
acc = _mm_add_epi32(acc, _mm_mullo_epi32(_mm_set1_epi32(inp[i]), w));
}
acc
}
#[inline]
#[target_feature(enable = "avx2")]
fn dip_dot4_block4_hbd_avx2(
mut acc: __m128i,
inp: &[i32; 11],
w0: &[u16; 11],
w1: &[u16; 11],
w2: &[u16; 11],
w3: &[u16; 11],
base: usize,
) -> __m128i {
let r0 = unsafe { dip_load_u16x4_i32_avx2(w0.as_ptr().add(base)) };
let r1 = unsafe { dip_load_u16x4_i32_avx2(w1.as_ptr().add(base)) };
let r2 = unsafe { dip_load_u16x4_i32_avx2(w2.as_ptr().add(base)) };
let r3 = unsafe { dip_load_u16x4_i32_avx2(w3.as_ptr().add(base)) };
let t01 = _mm_unpacklo_epi32(r0, r1);
let t23 = _mm_unpacklo_epi32(r2, r3);
let c0 = _mm_unpacklo_epi64(t01, t23);
let c1 = _mm_unpackhi_epi64(t01, t23);
let t01 = _mm_unpackhi_epi32(r0, r1);
let t23 = _mm_unpackhi_epi32(r2, r3);
let c2 = _mm_unpacklo_epi64(t01, t23);
let c3 = _mm_unpackhi_epi64(t01, t23);
acc = _mm_add_epi32(acc, _mm_mullo_epi32(c0, _mm_set1_epi32(inp[base])));
acc = _mm_add_epi32(acc, _mm_mullo_epi32(c1, _mm_set1_epi32(inp[base + 1])));
acc = _mm_add_epi32(acc, _mm_mullo_epi32(c2, _mm_set1_epi32(inp[base + 2])));
_mm_add_epi32(acc, _mm_mullo_epi32(c3, _mm_set1_epi32(inp[base + 3])))
}
#[inline]
#[target_feature(enable = "avx2")]
fn dip_dot4_hbd_avx2(
inp: &[i32; 11],
w0: &[u16; 11],
w1: &[u16; 11],
w2: &[u16; 11],
w3: &[u16; 11],
) -> __m128i {
let mut acc = _mm_setzero_si128();
acc = dip_dot4_block4_hbd_avx2(acc, inp, w0, w1, w2, w3, 0);
acc = dip_dot4_block4_hbd_avx2(acc, inp, w0, w1, w2, w3, 4);
for i in 8..11 {
let w = _mm_setr_epi32(w0[i] as i32, w1[i] as i32, w2[i] as i32, w3[i] as i32);
acc = _mm_add_epi32(acc, _mm_mullo_epi32(_mm_set1_epi32(inp[i]), w));
}
acc
}
#[inline]
#[target_feature(enable = "avx2")]
fn dip_finish_u16x4_avx2(s: __m128i, in_sum: i32, bitdepth_max: i32) -> __m128i {
let v = _mm_sub_epi32(
_mm_srai_epi32::<12>(_mm_add_epi32(s, _mm_set1_epi32(2048))),
_mm_set1_epi32(in_sum),
);
let v = _mm_min_epi32(
_mm_max_epi32(v, _mm_setzero_si128()),
_mm_set1_epi32(bitdepth_max),
);
_mm_packus_epi32(v, _mm_setzero_si128())
}
#[inline]
#[target_feature(enable = "avx2")]
fn store_dip_u16x4_avx2<const STEP_X: usize>(dst_row: &mut [u16], x: usize, v: __m128i) {
if STEP_X == 1 {
unsafe { _mm_storel_epi64(dst_row.as_mut_ptr().add(x).cast(), v) };
} else {
dst_row[x] = _mm_extract_epi16::<0>(v) as u16;
dst_row[x + STEP_X] = _mm_extract_epi16::<1>(v) as u16;
dst_row[x + STEP_X * 2] = _mm_extract_epi16::<2>(v) as u16;
dst_row[x + STEP_X * 3] = _mm_extract_epi16::<3>(v) as u16;
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn store_dip_u16x2_avx2<const STEP_X: usize>(dst_row: &mut [u16], x: usize, v: __m128i) {
if STEP_X == 1 {
unsafe { _mm_store_ss(dst_row.as_mut_ptr().add(x).cast(), _mm_castsi128_ps(v)) };
} else {
dst_row[x] = _mm_extract_epi16::<0>(v) as u16;
dst_row[x + STEP_X] = _mm_extract_epi16::<1>(v) as u16;
}
}
#[allow(clippy::too_many_arguments)]
#[inline]
#[target_feature(enable = "avx2")]
fn dip_emit_hbd_avx2<const STEP_X: usize>(
dst: &mut [u16],
stride: usize,
width: usize,
trans: bool,
dw: usize,
dh: usize,
step_y: usize,
grid_h: usize,
grid_w: usize,
inp: &[i32; 11],
weights: &[[u16; 11]; 64],
in_sum: i32,
bitdepth_max: i32,
) {
let mut y = step_y - 1;
for gy in 0..grid_h {
let iy = gy * dh;
let mut gx = 0usize;
let mut x = STEP_X - 1;
let dst_row = &mut dst[y * stride..y * stride + width];
while gx + 4 <= grid_w {
let ix0 = gx * dw;
let ix1 = ix0 + dw;
let ix2 = ix1 + dw;
let ix3 = ix2 + dw;
let idx0 = if trans { ix0 * 8 + iy } else { iy * 8 + ix0 };
let idx1 = if trans { ix1 * 8 + iy } else { iy * 8 + ix1 };
let idx2 = if trans { ix2 * 8 + iy } else { iy * 8 + ix2 };
let idx3 = if trans { ix3 * 8 + iy } else { iy * 8 + ix3 };
let s = dip_finish_u16x4_avx2(
dip_dot4_hbd_avx2(
inp,
&weights[idx0],
&weights[idx1],
&weights[idx2],
&weights[idx3],
),
in_sum,
bitdepth_max,
);
store_dip_u16x4_avx2::<STEP_X>(dst_row, x, s);
gx += 4;
x += STEP_X * 4;
}
if gx + 2 <= grid_w {
let ix0 = gx * dw;
let ix1 = ix0 + dw;
let idx0 = if trans { ix0 * 8 + iy } else { iy * 8 + ix0 };
let idx1 = if trans { ix1 * 8 + iy } else { iy * 8 + ix1 };
let s = dip_finish_u16x4_avx2(
dip_dot2_hbd_avx2(inp, &weights[idx0], &weights[idx1]),
in_sum,
bitdepth_max,
);
store_dip_u16x2_avx2::<STEP_X>(dst_row, x, s);
gx += 2;
x += STEP_X * 2;
}
while gx < grid_w {
let ix = gx * dw;
let idx = if trans { ix * 8 + iy } else { iy * 8 + ix };
let s = dip_dot_hbd_avx2(inp, &weights[idx]);
dst_row[x] = (((s + 2048) >> 12) - in_sum).clamp(0, bitdepth_max) as u16;
gx += 1;
x += STEP_X;
}
y += step_y;
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn dip_vertical_interp_hbd_avx2(
dst: &mut [u16],
stride: usize,
tl: &[u16],
o: usize,
w: usize,
step_y: usize,
uhl2: i32,
grid_h: usize,
) {
if step_y <= 1 {
return;
}
let mut p0_buf = [0u16; 128];
let mut p1_buf = [0u16; 128];
for gy in 0..grid_h {
let base_y = gy * step_y;
let sparse_y = base_y + step_y - 1;
if gy == 0 {
p0_buf[..w].copy_from_slice(&tl[o + 1..o + 1 + w]);
} else {
let prev = (base_y - 1) * stride;
p0_buf[..w].copy_from_slice(&dst[prev..prev + w]);
}
let p1_off = sparse_y * stride;
p1_buf[..w].copy_from_slice(&dst[p1_off..p1_off + w]);
let p0 = &p0_buf[..w];
let p1 = &p1_buf[..w];
for z in 0..step_y - 1 {
let z1 = (z + 1) as i16;
let w0 = _mm256_set1_epi16((step_y as i16) - z1);
let w1 = _mm256_set1_epi16(z1);
let sh = _mm_cvtsi32_si128(uhl2);
let row_off = (base_y + z) * stride;
let row = &mut dst[row_off..row_off + w];
let (row16, row_rem16) = row.as_chunks_mut::<16>();
let (p0_16, p0_rem16) = p0.as_chunks::<16>();
let (p1_16, p1_rem16) = p1.as_chunks::<16>();
for ((d, a), b) in row16.iter_mut().zip(p0_16.iter()).zip(p1_16.iter()) {
let r = _mm256_srl_epi16(
_mm256_add_epi16(
_mm256_mullo_epi16(load_u16x16(a), w0),
_mm256_mullo_epi16(load_u16x16(b), w1),
),
sh,
);
store_u16x16(d, r);
}
let (row8, row_tail) = row_rem16.as_chunks_mut::<8>();
let (p0_8, p0_tail) = p0_rem16.as_chunks::<8>();
let (p1_8, p1_tail) = p1_rem16.as_chunks::<8>();
for ((d, a), b) in row8.iter_mut().zip(p0_8.iter()).zip(p1_8.iter()) {
let w0_128 = _mm_set1_epi16((step_y as i16) - z1);
let w1_128 = _mm_set1_epi16(z1);
let r = _mm_srl_epi16(
_mm_add_epi16(
_mm_mullo_epi16(load_u16x8(a), w0_128),
_mm_mullo_epi16(load_u16x8(b), w1_128),
),
sh,
);
store_u16x8(d, r);
}
let tail_w0 = step_y as i32 - z as i32 - 1;
let tail_w1 = z as i32 + 1;
for ((d, &a), &b) in row_tail.iter_mut().zip(p0_tail.iter()).zip(p1_tail.iter()) {
*d = ((a as i32 * tail_w0 + b as i32 * tail_w1) >> uhl2) as u16;
}
}
}
}
#[target_feature(enable = "avx2")]
pub(crate) fn ipred_v_hbd_avx2(
dst: &mut [u16],
stride: usize,
tl: &[u16],
o: usize,
w: usize,
h: usize,
angle: i32,
bitdepth_max: u16,
) {
if w < 16 {
crate::sse::ipred_v_hbd_sse41(dst, stride, tl, o, w, h, angle, bitdepth_max);
return;
}
if angle & ANGLE_MULTI_MRL_FLAG != 0 {
let e_stride = (w + h) * 2 + 1;
let top1 = &tl[o + 1..o + 1 + w];
let top2 = &tl[o + 1 + e_stride..o + 1 + e_stride + w];
let (chunks, rem) = dst[..w].as_chunks_mut::<16>();
for ((d, a), b) in chunks
.iter_mut()
.zip(top1.as_chunks::<16>().0.iter())
.zip(top2.as_chunks::<16>().0.iter())
{
store_u16x16(d, _mm256_avg_epu16(load_u16x16(a), load_u16x16(b)));
}
let base = chunks.len() * 16;
for (i, d) in rem.iter_mut().enumerate() {
let x = base + i;
*d = ((top1[x] as u32 + top2[x] as u32 + 1) >> 1) as u16;
}
} else {
dst[..w].copy_from_slice(&tl[o + 1..o + 1 + w]);
}
if h > 1 {
let (first, rest) = dst.split_at_mut(stride);
let src = &first[..w];
for row in rest.chunks_mut(stride).take(h - 1) {
row[..w].copy_from_slice(src);
}
}
}
#[inline]
fn h_left_u16(tl: &[u16], o: usize, e_stride: usize, y: usize, mrl: bool) -> u16 {
if mrl {
((tl[o - 1 - y] as u32 + tl[o + e_stride - 1 - y] as u32 + 1) >> 1) as u16
} else {
tl[o - 1 - y]
}
}
#[target_feature(enable = "avx2")]
pub(crate) fn ipred_h_hbd_avx2(
dst: &mut [u16],
stride: usize,
tl: &[u16],
o: usize,
w: usize,
h: usize,
angle: i32,
bitdepth_max: u16,
) {
if w < 8 {
crate::sse::ipred_h_hbd_sse41(dst, stride, tl, o, w, h, angle, bitdepth_max);
return;
}
let e_stride = (w + h) * 2 + 1;
let mrl = angle & ANGLE_MULTI_MRL_FLAG != 0;
let mut y = 0usize;
let mut off = 0usize;
while y + 3 < h {
let v = [
h_left_u16(tl, o, e_stride, y, mrl),
h_left_u16(tl, o, e_stride, y + 1, mrl),
h_left_u16(tl, o, e_stride, y + 2, mrl),
h_left_u16(tl, o, e_stride, y + 3, mrl),
];
splat_h_rows4_u16_avx2(dst, stride, off, w, v);
y += 4;
off += stride * 4;
}
while y < h {
let v = h_left_u16(tl, o, e_stride, y, mrl);
splat_row_u16_avx2(&mut dst[off..off + w], v);
y += 1;
off += stride;
}
}
#[target_feature(enable = "avx2")]
pub(crate) fn ipred_dc_128_hbd_avx2(
dst: &mut [u16],
stride: usize,
w: usize,
h: usize,
bitdepth_max: u16,
) {
splat_fill_avx2(dst, stride, 0, w, h, (bitdepth_max + 1) >> 1);
}
#[target_feature(enable = "avx2")]
pub(crate) fn ipred_dc_top_hbd_avx2(
dst: &mut [u16],
stride: usize,
tl: &[u16],
o: usize,
w: usize,
h: usize,
angle: i32,
bitdepth_max: u16,
) {
if angle & ANGLE_IBP_FLAG != 0 {
crate::sse::ipred_dc_top_hbd_sse41(dst, stride, tl, o, w, h, angle, bitdepth_max);
return;
}
let dc = (((w >> 1) as u32 + sum_u16_avx2(&tl[o + 1..o + 1 + w]))
>> (w as u32).trailing_zeros()) as u16;
splat_fill_avx2(dst, stride, 0, w, h, dc);
}
#[target_feature(enable = "avx2")]
pub(crate) fn ipred_dc_left_hbd_avx2(
dst: &mut [u16],
stride: usize,
tl: &[u16],
o: usize,
w: usize,
h: usize,
angle: i32,
bitdepth_max: u16,
) {
if angle & ANGLE_IBP_FLAG != 0 {
crate::sse::ipred_dc_left_hbd_sse41(dst, stride, tl, o, w, h, angle, bitdepth_max);
return;
}
let dc =
(((h >> 1) as u32 + sum_u16_avx2(&tl[o - h..o])) >> (h as u32).trailing_zeros()) as u16;
splat_fill_avx2(dst, stride, 0, w, h, dc);
}
#[target_feature(enable = "avx2")]
pub(crate) fn ipred_dc_hbd_avx2(
dst: &mut [u16],
stride: usize,
tl: &[u16],
o: usize,
w: usize,
h: usize,
angle: i32,
bitdepth_max: u16,
) {
if angle & ANGLE_IBP_FLAG != 0 {
crate::sse::ipred_dc_hbd_sse41(dst, stride, tl, o, w, h, angle, bitdepth_max);
return;
}
let n = (w + h) as u32;
let sum = sum_u16_avx2(&tl[o + 1..o + 1 + w]) + sum_u16_avx2(&tl[o - h..o]);
let dc = if n & (n - 1) == 0 {
(sum + w as u32) >> n.trailing_zeros()
} else {
crate::ipred::fast_div32_dc(sum, n).min(bitdepth_max as u32)
} as u16;
splat_fill_avx2(dst, stride, 0, w, h, dc);
}
#[inline]
#[target_feature(enable = "avx2")]
fn sra_i32x8(v: __m256i, shift: i32) -> __m256i {
_mm256_sra_epi32(v, _mm_cvtsi32_si128(shift))
}
#[inline]
#[target_feature(enable = "avx2")]
fn dist8_hbd(base: i32) -> __m256i {
_mm256_sub_epi32(_mm256_set1_epi32(base), setr_i32x8(0, 1, 2, 3, 4, 5, 6, 7))
}
#[inline]
#[target_feature(enable = "avx2")]
fn weights8_hbd(w: &[u8]) -> __m256i {
debug_assert!(w.len() >= 8);
setr_i32x8(
w[0] as i32,
w[1] as i32,
w[2] as i32,
w[3] as i32,
w[4] as i32,
w[5] as i32,
w[6] as i32,
w[7] as i32,
)
}
#[inline]
#[target_feature(enable = "avx2")]
fn abs_i32x8(v: __m256i) -> __m256i {
let m = _mm256_srai_epi32::<31>(v);
_mm256_sub_epi32(_mm256_xor_si256(v, m), m)
}
#[inline]
#[target_feature(enable = "avx2")]
fn le_i32x8(a: __m256i, b: __m256i) -> __m256i {
_mm256_andnot_si256(_mm256_cmpgt_epi32(a, b), _mm256_cmpeq_epi32(a, a))
}
#[target_feature(enable = "avx2")]
pub(crate) fn ipred_smooth_v_hbd_avx2(
dst: &mut [u16],
stride: usize,
tl: &[u16],
o: usize,
w: usize,
h: usize,
bitdepth_max: u16,
) {
if w < 8 {
crate::sse::ipred_smooth_v_hbd_sse41(dst, stride, tl, o, w, h, bitdepth_max);
return;
}
let bhl2 = ulog2(h as u32) as i32;
let rnd = _mm256_set1_epi32((h >> 1) as i32);
let scale = (w * h >= 64) as usize + (w * h > 512) as usize;
let weights = &SM_WEIGHTS[scale];
let bottom_s = tl[o - h - 1] as i32;
let bottom = _mm256_set1_epi32(bottom_s);
let add32 = _mm256_set1_epi32(32);
let top_src = &tl[o + 1..o + 1 + w];
let mut y = 0usize;
let mut off = 0usize;
while y + 3 < h {
let (row0_s, rest) = dst[off..].split_at_mut(stride);
let (row1_s, rest) = rest.split_at_mut(stride);
let (row2_s, row3_s) = rest.split_at_mut(stride);
let row0 = &mut row0_s[..w];
let row1 = &mut row1_s[..w];
let row2 = &mut row2_s[..w];
let row3 = &mut row3_s[..w];
let off_y0 = _mm256_set1_epi32((h - 1 - y) as i32);
let off_y1 = _mm256_set1_epi32((h - 2 - y) as i32);
let off_y2 = _mm256_set1_epi32((h - 3 - y) as i32);
let off_y3 = _mm256_set1_epi32((h - 4 - y) as i32);
let w_ver0 = _mm256_set1_epi32(weights[y] as i32);
let w_ver1 = _mm256_set1_epi32(weights[y + 1] as i32);
let w_ver2 = _mm256_set1_epi32(weights[y + 2] as i32);
let w_ver3 = _mm256_set1_epi32(weights[y + 3] as i32);
let (chunks0, rem0) = row0.as_chunks_mut::<8>();
let (chunks1, rem1) = row1.as_chunks_mut::<8>();
let (chunks2, rem2) = row2.as_chunks_mut::<8>();
let (chunks3, rem3) = row3.as_chunks_mut::<8>();
for ((((d0, d1), d2), d3), t) in chunks0
.iter_mut()
.zip(chunks1.iter_mut())
.zip(chunks2.iter_mut())
.zip(chunks3.iter_mut())
.zip(top_src.as_chunks::<8>().0.iter())
{
let above = load_u16x8_i32_slice(t);
let pred0 = _mm256_add_epi32(
bottom,
sra_i32x8(
_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, bottom), off_y0),
rnd,
),
bhl2,
),
);
let out0 = _mm256_add_epi32(
pred0,
_mm256_srai_epi32::<6>(_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, pred0), w_ver0),
add32,
)),
);
store_i32x8_u16_max(d0, out0, bitdepth_max);
let pred1 = _mm256_add_epi32(
bottom,
sra_i32x8(
_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, bottom), off_y1),
rnd,
),
bhl2,
),
);
let out1 = _mm256_add_epi32(
pred1,
_mm256_srai_epi32::<6>(_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, pred1), w_ver1),
add32,
)),
);
store_i32x8_u16_max(d1, out1, bitdepth_max);
let pred2 = _mm256_add_epi32(
bottom,
sra_i32x8(
_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, bottom), off_y2),
rnd,
),
bhl2,
),
);
let out2 = _mm256_add_epi32(
pred2,
_mm256_srai_epi32::<6>(_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, pred2), w_ver2),
add32,
)),
);
store_i32x8_u16_max(d2, out2, bitdepth_max);
let pred3 = _mm256_add_epi32(
bottom,
sra_i32x8(
_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, bottom), off_y3),
rnd,
),
bhl2,
),
);
let out3 = _mm256_add_epi32(
pred3,
_mm256_srai_epi32::<6>(_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, pred3), w_ver3),
add32,
)),
);
store_i32x8_u16_max(d3, out3, bitdepth_max);
}
let base = chunks0.len() * 8;
for (i, (((d0, d1), d2), d3)) in rem0
.iter_mut()
.zip(rem1.iter_mut())
.zip(rem2.iter_mut())
.zip(rem3.iter_mut())
.enumerate()
{
let x = base + i;
let above = tl[o + 1 + x] as i32;
let pred0 = bottom_s
+ (((above - bottom_s) * (h as i32 - 1 - y as i32) + (h >> 1) as i32) >> bhl2);
let pred1 = bottom_s
+ (((above - bottom_s) * (h as i32 - 2 - y as i32) + (h >> 1) as i32) >> bhl2);
let pred2 = bottom_s
+ (((above - bottom_s) * (h as i32 - 3 - y as i32) + (h >> 1) as i32) >> bhl2);
let pred3 = bottom_s
+ (((above - bottom_s) * (h as i32 - 4 - y as i32) + (h >> 1) as i32) >> bhl2);
*d0 = (pred0 + (((above - pred0) * weights[y] as i32 + 32) >> 6))
.clamp(0, bitdepth_max as i32) as u16;
*d1 = (pred1 + (((above - pred1) * weights[y + 1] as i32 + 32) >> 6))
.clamp(0, bitdepth_max as i32) as u16;
*d2 = (pred2 + (((above - pred2) * weights[y + 2] as i32 + 32) >> 6))
.clamp(0, bitdepth_max as i32) as u16;
*d3 = (pred3 + (((above - pred3) * weights[y + 3] as i32 + 32) >> 6))
.clamp(0, bitdepth_max as i32) as u16;
}
y += 4;
off += stride * 4;
}
while y + 1 < h {
let (head, tail) = dst[off..].split_at_mut(stride);
let row0 = &mut head[..w];
let row1 = &mut tail[..w];
let off_y0 = _mm256_set1_epi32((h - 1 - y) as i32);
let off_y1 = _mm256_set1_epi32((h - 2 - y) as i32);
let w_ver0 = _mm256_set1_epi32(weights[y] as i32);
let w_ver1 = _mm256_set1_epi32(weights[y + 1] as i32);
let (chunks0, rem0) = row0.as_chunks_mut::<8>();
let (chunks1, rem1) = row1.as_chunks_mut::<8>();
for ((d0, d1), t) in chunks0
.iter_mut()
.zip(chunks1.iter_mut())
.zip(top_src.as_chunks::<8>().0.iter())
{
let above = load_u16x8_i32_slice(t);
let pred0 = _mm256_add_epi32(
bottom,
sra_i32x8(
_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, bottom), off_y0),
rnd,
),
bhl2,
),
);
let out0 = _mm256_add_epi32(
pred0,
_mm256_srai_epi32::<6>(_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, pred0), w_ver0),
add32,
)),
);
store_i32x8_u16_max(d0, out0, bitdepth_max);
let pred1 = _mm256_add_epi32(
bottom,
sra_i32x8(
_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, bottom), off_y1),
rnd,
),
bhl2,
),
);
let out1 = _mm256_add_epi32(
pred1,
_mm256_srai_epi32::<6>(_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, pred1), w_ver1),
add32,
)),
);
store_i32x8_u16_max(d1, out1, bitdepth_max);
}
let base = chunks0.len() * 8;
for (i, (d0, d1)) in rem0.iter_mut().zip(rem1.iter_mut()).enumerate() {
let x = base + i;
let above = tl[o + 1 + x] as i32;
let pred0 = bottom_s
+ (((above - bottom_s) * (h as i32 - 1 - y as i32) + (h >> 1) as i32) >> bhl2);
let pred1 = bottom_s
+ (((above - bottom_s) * (h as i32 - 2 - y as i32) + (h >> 1) as i32) >> bhl2);
*d0 = (pred0 + (((above - pred0) * weights[y] as i32 + 32) >> 6))
.clamp(0, bitdepth_max as i32) as u16;
*d1 = (pred1 + (((above - pred1) * weights[y + 1] as i32 + 32) >> 6))
.clamp(0, bitdepth_max as i32) as u16;
}
y += 2;
off += stride * 2;
}
if y < h {
let off_y = _mm256_set1_epi32((h - 1 - y) as i32);
let w_ver = _mm256_set1_epi32(weights[y] as i32);
let row = &mut dst[off..off + w];
let (chunks, rem) = row.as_chunks_mut::<8>();
for (d, t) in chunks.iter_mut().zip(top_src.as_chunks::<8>().0.iter()) {
let above = load_u16x8_i32_slice(t);
let pred = _mm256_add_epi32(
bottom,
sra_i32x8(
_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, bottom), off_y),
rnd,
),
bhl2,
),
);
let out = _mm256_add_epi32(
pred,
_mm256_srai_epi32::<6>(_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, pred), w_ver),
add32,
)),
);
store_i32x8_u16_max(d, out, bitdepth_max);
}
let base = chunks.len() * 8;
for (i, d) in rem.iter_mut().enumerate() {
let x = base + i;
let above = tl[o + 1 + x] as i32;
let pred = bottom_s
+ (((above - bottom_s) * (h as i32 - 1 - y as i32) + (h >> 1) as i32) >> bhl2);
*d = (pred + (((above - pred) * weights[y] as i32 + 32) >> 6))
.clamp(0, bitdepth_max as i32) as u16;
}
}
}
#[target_feature(enable = "avx2")]
pub(crate) fn ipred_smooth_h_hbd_avx2(
dst: &mut [u16],
stride: usize,
tl: &[u16],
o: usize,
w: usize,
h: usize,
bitdepth_max: u16,
) {
if w < 8 {
crate::sse::ipred_smooth_h_hbd_sse41(dst, stride, tl, o, w, h, bitdepth_max);
return;
}
let bwl2 = ulog2(w as u32) as i32;
let rnd = _mm256_set1_epi32((w >> 1) as i32);
let scale = (w * h >= 64) as usize + (w * h > 512) as usize;
let weights = &SM_WEIGHTS[scale];
let right_s = tl[o + w + 1] as i32;
let right = _mm256_set1_epi32(right_s);
let add32 = _mm256_set1_epi32(32);
let mut y = 0usize;
let mut off = 0usize;
while y + 1 < h {
let (head, tail) = dst[off..].split_at_mut(stride);
let row0 = &mut head[..w];
let row1 = &mut tail[..w];
let left0_s = tl[o - 1 - y] as i32;
let left1_s = tl[o - 2 - y] as i32;
let left0 = _mm256_set1_epi32(left0_s);
let left1 = _mm256_set1_epi32(left1_s);
let diff0 = _mm256_set1_epi32(left0_s - right_s);
let diff1 = _mm256_set1_epi32(left1_s - right_s);
let (chunks0, rem0) = row0.as_chunks_mut::<8>();
let (chunks1, rem1) = row1.as_chunks_mut::<8>();
for (ci, ((d0, d1), wx)) in chunks0
.iter_mut()
.zip(chunks1.iter_mut())
.zip(weights[..w].as_chunks::<8>().0.iter())
.enumerate()
{
let x0 = (w - 1 - ci * 8) as i32;
let dist = dist8_hbd(x0);
let wx = weights8_hbd(wx);
let pred0 = _mm256_add_epi32(
right,
sra_i32x8(_mm256_add_epi32(_mm256_mullo_epi32(diff0, dist), rnd), bwl2),
);
let out0 = _mm256_add_epi32(
pred0,
_mm256_srai_epi32::<6>(_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(left0, pred0), wx),
add32,
)),
);
store_i32x8_u16_max(d0, out0, bitdepth_max);
let pred1 = _mm256_add_epi32(
right,
sra_i32x8(_mm256_add_epi32(_mm256_mullo_epi32(diff1, dist), rnd), bwl2),
);
let out1 = _mm256_add_epi32(
pred1,
_mm256_srai_epi32::<6>(_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(left1, pred1), wx),
add32,
)),
);
store_i32x8_u16_max(d1, out1, bitdepth_max);
}
let base = chunks0.len() * 8;
for (i, (d0, d1)) in rem0.iter_mut().zip(rem1.iter_mut()).enumerate() {
let x = base + i;
let pred0 = right_s
+ (((left0_s - right_s) * (w as i32 - 1 - x as i32) + (w >> 1) as i32) >> bwl2);
let pred1 = right_s
+ (((left1_s - right_s) * (w as i32 - 1 - x as i32) + (w >> 1) as i32) >> bwl2);
*d0 = (pred0 + (((left0_s - pred0) * weights[x] as i32 + 32) >> 6))
.clamp(0, bitdepth_max as i32) as u16;
*d1 = (pred1 + (((left1_s - pred1) * weights[x] as i32 + 32) >> 6))
.clamp(0, bitdepth_max as i32) as u16;
}
y += 2;
off += stride * 2;
}
if y < h {
let left_s = tl[o - 1 - y] as i32;
let left = _mm256_set1_epi32(left_s);
let diff = _mm256_set1_epi32(left_s - right_s);
let row = &mut dst[off..off + w];
let (chunks, rem) = row.as_chunks_mut::<8>();
for (ci, (d, wx)) in chunks
.iter_mut()
.zip(weights[..w].as_chunks::<8>().0.iter())
.enumerate()
{
let x0 = (w - 1 - ci * 8) as i32;
let pred = _mm256_add_epi32(
right,
sra_i32x8(
_mm256_add_epi32(_mm256_mullo_epi32(diff, dist8_hbd(x0)), rnd),
bwl2,
),
);
let out = _mm256_add_epi32(
pred,
_mm256_srai_epi32::<6>(_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(left, pred), weights8_hbd(wx)),
add32,
)),
);
store_i32x8_u16_max(d, out, bitdepth_max);
}
let base = chunks.len() * 8;
for (i, d) in rem.iter_mut().enumerate() {
let x = base + i;
let pred = right_s
+ (((left_s - right_s) * (w as i32 - 1 - x as i32) + (w >> 1) as i32) >> bwl2);
*d = (pred + (((left_s - pred) * weights[x] as i32 + 32) >> 6))
.clamp(0, bitdepth_max as i32) as u16;
}
}
}
#[target_feature(enable = "avx2")]
pub(crate) fn ipred_smooth_hbd_avx2(
dst: &mut [u16],
stride: usize,
tl: &[u16],
o: usize,
w: usize,
h: usize,
bitdepth_max: u16,
) {
if w < 8 {
crate::sse::ipred_smooth_hbd_sse41(dst, stride, tl, o, w, h, bitdepth_max);
return;
}
let bwl2 = ulog2(w as u32) as i32;
let bhl2 = ulog2(h as u32) as i32;
let rnd_ver = _mm256_set1_epi32((h >> 1) as i32);
let rnd_hor = _mm256_set1_epi32((w >> 1) as i32);
let scale = (w * h >= 64) as usize + (w * h > 512) as usize;
let weights = &SM_WEIGHTS[scale];
let right_s = tl[o + w + 1] as i32;
let bottom_s = tl[o - h - 1] as i32;
let right = _mm256_set1_epi32(right_s);
let bottom = _mm256_set1_epi32(bottom_s);
let add32 = _mm256_set1_epi32(32);
let one = _mm256_set1_epi32(1);
let top_src = &tl[o + 1..o + 1 + w];
let mut y = 0usize;
let mut off = 0usize;
while y + 1 < h {
let (head, tail) = dst[off..].split_at_mut(stride);
let row0 = &mut head[..w];
let row1 = &mut tail[..w];
let left0_s = tl[o - 1 - y] as i32;
let left1_s = tl[o - 2 - y] as i32;
let left0 = _mm256_set1_epi32(left0_s);
let left1 = _mm256_set1_epi32(left1_s);
let diff_hor0 = _mm256_set1_epi32(left0_s - right_s);
let diff_hor1 = _mm256_set1_epi32(left1_s - right_s);
let off_ver0 = _mm256_set1_epi32((h - 1 - y) as i32);
let off_ver1 = _mm256_set1_epi32((h - 2 - y) as i32);
let w_ver0 = _mm256_set1_epi32(weights[y] as i32);
let w_ver1 = _mm256_set1_epi32(weights[y + 1] as i32);
let (chunks0, rem0) = row0.as_chunks_mut::<8>();
let (chunks1, rem1) = row1.as_chunks_mut::<8>();
for (ci, (((d0, d1), t), wx)) in chunks0
.iter_mut()
.zip(chunks1.iter_mut())
.zip(top_src.as_chunks::<8>().0.iter())
.zip(weights[..w].as_chunks::<8>().0.iter())
.enumerate()
{
let above = load_u16x8_i32_slice(t);
let wx = weights8_hbd(wx);
let x0 = (w - 1 - ci * 8) as i32;
let dist = dist8_hbd(x0);
let pv0 = _mm256_add_epi32(
bottom,
sra_i32x8(
_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, bottom), off_ver0),
rnd_ver,
),
bhl2,
),
);
let ph0 = _mm256_add_epi32(
right,
sra_i32x8(
_mm256_add_epi32(_mm256_mullo_epi32(diff_hor0, dist), rnd_hor),
bwl2,
),
);
let pv0 = _mm256_add_epi32(
pv0,
_mm256_srai_epi32::<6>(_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, pv0), w_ver0),
add32,
)),
);
let ph0 = _mm256_add_epi32(
ph0,
_mm256_srai_epi32::<6>(_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(left0, ph0), wx),
add32,
)),
);
store_i32x8_u16_max(
d0,
_mm256_srai_epi32::<1>(_mm256_add_epi32(_mm256_add_epi32(pv0, ph0), one)),
bitdepth_max,
);
let pv1 = _mm256_add_epi32(
bottom,
sra_i32x8(
_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, bottom), off_ver1),
rnd_ver,
),
bhl2,
),
);
let ph1 = _mm256_add_epi32(
right,
sra_i32x8(
_mm256_add_epi32(_mm256_mullo_epi32(diff_hor1, dist), rnd_hor),
bwl2,
),
);
let pv1 = _mm256_add_epi32(
pv1,
_mm256_srai_epi32::<6>(_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, pv1), w_ver1),
add32,
)),
);
let ph1 = _mm256_add_epi32(
ph1,
_mm256_srai_epi32::<6>(_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(left1, ph1), wx),
add32,
)),
);
store_i32x8_u16_max(
d1,
_mm256_srai_epi32::<1>(_mm256_add_epi32(_mm256_add_epi32(pv1, ph1), one)),
bitdepth_max,
);
}
let base = chunks0.len() * 8;
for (i, (d0, d1)) in rem0.iter_mut().zip(rem1.iter_mut()).enumerate() {
let x = base + i;
let above = tl[o + 1 + x] as i32;
let mut pv0 = bottom_s
+ (((above - bottom_s) * (h as i32 - 1 - y as i32) + (h >> 1) as i32) >> bhl2);
let mut ph0 = right_s
+ (((left0_s - right_s) * (w as i32 - 1 - x as i32) + (w >> 1) as i32) >> bwl2);
pv0 += ((above - pv0) * weights[y] as i32 + 32) >> 6;
ph0 += ((left0_s - ph0) * weights[x] as i32 + 32) >> 6;
*d0 = ((pv0 + ph0 + 1) >> 1).clamp(0, bitdepth_max as i32) as u16;
let mut pv1 = bottom_s
+ (((above - bottom_s) * (h as i32 - 2 - y as i32) + (h >> 1) as i32) >> bhl2);
let mut ph1 = right_s
+ (((left1_s - right_s) * (w as i32 - 1 - x as i32) + (w >> 1) as i32) >> bwl2);
pv1 += ((above - pv1) * weights[y + 1] as i32 + 32) >> 6;
ph1 += ((left1_s - ph1) * weights[x] as i32 + 32) >> 6;
*d1 = ((pv1 + ph1 + 1) >> 1).clamp(0, bitdepth_max as i32) as u16;
}
y += 2;
off += stride * 2;
}
if y < h {
let left_s = tl[o - 1 - y] as i32;
let left = _mm256_set1_epi32(left_s);
let diff_hor = _mm256_set1_epi32(left_s - right_s);
let off_ver = _mm256_set1_epi32((h - 1 - y) as i32);
let w_ver = _mm256_set1_epi32(weights[y] as i32);
let row = &mut dst[off..off + w];
let (chunks, rem) = row.as_chunks_mut::<8>();
for (ci, ((d, t), wx)) in chunks
.iter_mut()
.zip(top_src.as_chunks::<8>().0.iter())
.zip(weights[..w].as_chunks::<8>().0.iter())
.enumerate()
{
let above = load_u16x8_i32_slice(t);
let pv = _mm256_add_epi32(
bottom,
sra_i32x8(
_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, bottom), off_ver),
rnd_ver,
),
bhl2,
),
);
let x0 = (w - 1 - ci * 8) as i32;
let ph = _mm256_add_epi32(
right,
sra_i32x8(
_mm256_add_epi32(_mm256_mullo_epi32(diff_hor, dist8_hbd(x0)), rnd_hor),
bwl2,
),
);
let pv = _mm256_add_epi32(
pv,
_mm256_srai_epi32::<6>(_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(above, pv), w_ver),
add32,
)),
);
let ph = _mm256_add_epi32(
ph,
_mm256_srai_epi32::<6>(_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_sub_epi32(left, ph), weights8_hbd(wx)),
add32,
)),
);
store_i32x8_u16_max(
d,
_mm256_srai_epi32::<1>(_mm256_add_epi32(_mm256_add_epi32(pv, ph), one)),
bitdepth_max,
);
}
let base = chunks.len() * 8;
for (i, d) in rem.iter_mut().enumerate() {
let x = base + i;
let above = tl[o + 1 + x] as i32;
let mut pv = bottom_s
+ (((above - bottom_s) * (h as i32 - 1 - y as i32) + (h >> 1) as i32) >> bhl2);
let mut ph = right_s
+ (((left_s - right_s) * (w as i32 - 1 - x as i32) + (w >> 1) as i32) >> bwl2);
pv += ((above - pv) * weights[y] as i32 + 32) >> 6;
ph += ((left_s - ph) * weights[x] as i32 + 32) >> 6;
*d = ((pv + ph + 1) >> 1).clamp(0, bitdepth_max as i32) as u16;
}
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn paeth_i32x8_avx2(left: __m256i, top: __m256i, topleft: __m256i) -> __m256i {
let base = _mm256_sub_epi32(_mm256_add_epi32(left, top), topleft);
let ld = abs_i32x8(_mm256_sub_epi32(left, base));
let td = abs_i32x8(_mm256_sub_epi32(top, base));
let tld = abs_i32x8(_mm256_sub_epi32(topleft, base));
let left_mask = _mm256_and_si256(le_i32x8(ld, td), le_i32x8(ld, tld));
let top_mask = le_i32x8(td, tld);
let inner = _mm256_blendv_epi8(topleft, top, top_mask);
_mm256_blendv_epi8(inner, left, left_mask)
}
#[target_feature(enable = "avx2")]
pub(crate) fn ipred_paeth_hbd_avx2(
dst: &mut [u16],
stride: usize,
tl: &[u16],
o: usize,
w: usize,
h: usize,
bitdepth_max: u16,
) {
if w < 8 {
crate::sse::ipred_paeth_hbd_sse41(dst, stride, tl, o, w, h, bitdepth_max);
return;
}
let topleft_s = tl[o] as i32;
let topleft = _mm256_set1_epi32(topleft_s);
let top_src = &tl[o + 1..o + 1 + w];
let mut y = 0usize;
let mut off = 0usize;
while y + 3 < h {
let (row0_s, rest) = dst[off..].split_at_mut(stride);
let (row1_s, rest) = rest.split_at_mut(stride);
let (row2_s, row3_s) = rest.split_at_mut(stride);
let row0 = &mut row0_s[..w];
let row1 = &mut row1_s[..w];
let row2 = &mut row2_s[..w];
let row3 = &mut row3_s[..w];
let left0_s = tl[o - 1 - y] as i32;
let left1_s = tl[o - 2 - y] as i32;
let left2_s = tl[o - 3 - y] as i32;
let left3_s = tl[o - 4 - y] as i32;
let left0 = _mm256_set1_epi32(left0_s);
let left1 = _mm256_set1_epi32(left1_s);
let left2 = _mm256_set1_epi32(left2_s);
let left3 = _mm256_set1_epi32(left3_s);
let (chunks0, rem0) = row0.as_chunks_mut::<8>();
let (chunks1, rem1) = row1.as_chunks_mut::<8>();
let (chunks2, rem2) = row2.as_chunks_mut::<8>();
let (chunks3, rem3) = row3.as_chunks_mut::<8>();
for ((((d0, d1), d2), d3), t) in chunks0
.iter_mut()
.zip(chunks1.iter_mut())
.zip(chunks2.iter_mut())
.zip(chunks3.iter_mut())
.zip(top_src.as_chunks::<8>().0.iter())
{
let top = load_u16x8_i32_slice(t);
store_i32x8_u16_max(d0, paeth_i32x8_avx2(left0, top, topleft), bitdepth_max);
store_i32x8_u16_max(d1, paeth_i32x8_avx2(left1, top, topleft), bitdepth_max);
store_i32x8_u16_max(d2, paeth_i32x8_avx2(left2, top, topleft), bitdepth_max);
store_i32x8_u16_max(d3, paeth_i32x8_avx2(left3, top, topleft), bitdepth_max);
}
let base_x = chunks0.len() * 8;
for (i, (((d0, d1), d2), d3)) in rem0
.iter_mut()
.zip(rem1.iter_mut())
.zip(rem2.iter_mut())
.zip(rem3.iter_mut())
.enumerate()
{
let top_s = tl[o + 1 + base_x + i] as i32;
let base0 = left0_s + top_s - topleft_s;
let ld0 = (left0_s - base0).abs();
let td0 = (top_s - base0).abs();
let tld0 = (topleft_s - base0).abs();
*d0 = if ld0 <= td0 && ld0 <= tld0 {
left0_s
} else if td0 <= tld0 {
top_s
} else {
topleft_s
} as u16;
let base1 = left1_s + top_s - topleft_s;
let ld1 = (left1_s - base1).abs();
let td1 = (top_s - base1).abs();
let tld1 = (topleft_s - base1).abs();
*d1 = if ld1 <= td1 && ld1 <= tld1 {
left1_s
} else if td1 <= tld1 {
top_s
} else {
topleft_s
} as u16;
let base2 = left2_s + top_s - topleft_s;
let ld2 = (left2_s - base2).abs();
let td2 = (top_s - base2).abs();
let tld2 = (topleft_s - base2).abs();
*d2 = if ld2 <= td2 && ld2 <= tld2 {
left2_s
} else if td2 <= tld2 {
top_s
} else {
topleft_s
} as u16;
let base3 = left3_s + top_s - topleft_s;
let ld3 = (left3_s - base3).abs();
let td3 = (top_s - base3).abs();
let tld3 = (topleft_s - base3).abs();
*d3 = if ld3 <= td3 && ld3 <= tld3 {
left3_s
} else if td3 <= tld3 {
top_s
} else {
topleft_s
} as u16;
}
y += 4;
off += stride * 4;
}
while y + 1 < h {
let (head, tail) = dst[off..].split_at_mut(stride);
let row0 = &mut head[..w];
let row1 = &mut tail[..w];
let left0_s = tl[o - 1 - y] as i32;
let left1_s = tl[o - 2 - y] as i32;
let left0 = _mm256_set1_epi32(left0_s);
let left1 = _mm256_set1_epi32(left1_s);
let (chunks0, rem0) = row0.as_chunks_mut::<8>();
let (chunks1, rem1) = row1.as_chunks_mut::<8>();
for ((d0, d1), t) in chunks0
.iter_mut()
.zip(chunks1.iter_mut())
.zip(top_src.as_chunks::<8>().0.iter())
{
let top = load_u16x8_i32_slice(t);
store_i32x8_u16_max(d0, paeth_i32x8_avx2(left0, top, topleft), bitdepth_max);
store_i32x8_u16_max(d1, paeth_i32x8_avx2(left1, top, topleft), bitdepth_max);
}
let base_x = chunks0.len() * 8;
for (i, (d0, d1)) in rem0.iter_mut().zip(rem1.iter_mut()).enumerate() {
let top_s = tl[o + 1 + base_x + i] as i32;
let base0 = left0_s + top_s - topleft_s;
let ld0 = (left0_s - base0).abs();
let td0 = (top_s - base0).abs();
let tld0 = (topleft_s - base0).abs();
*d0 = if ld0 <= td0 && ld0 <= tld0 {
left0_s
} else if td0 <= tld0 {
top_s
} else {
topleft_s
} as u16;
let base1 = left1_s + top_s - topleft_s;
let ld1 = (left1_s - base1).abs();
let td1 = (top_s - base1).abs();
let tld1 = (topleft_s - base1).abs();
*d1 = if ld1 <= td1 && ld1 <= tld1 {
left1_s
} else if td1 <= tld1 {
top_s
} else {
topleft_s
} as u16;
}
y += 2;
off += stride * 2;
}
if y < h {
let left_s = tl[o - 1 - y] as i32;
let left = _mm256_set1_epi32(left_s);
let row = &mut dst[off..off + w];
let (chunks, rem) = row.as_chunks_mut::<8>();
for (d, t) in chunks.iter_mut().zip(top_src.as_chunks::<8>().0.iter()) {
let top = load_u16x8_i32_slice(t);
store_i32x8_u16_max(d, paeth_i32x8_avx2(left, top, topleft), bitdepth_max);
}
let base_x = chunks.len() * 8;
for (i, d) in rem.iter_mut().enumerate() {
let top_s = tl[o + 1 + base_x + i] as i32;
let base = left_s + top_s - topleft_s;
let ld = (left_s - base).abs();
let td = (top_s - base).abs();
let tld = (topleft_s - base).abs();
*d = if ld <= td && ld <= tld {
left_s
} else if td <= tld {
top_s
} else {
topleft_s
} as u16;
}
}
}
#[inline(always)]
fn load_u16x8_i32_slice(s: &[u16]) -> __m256i {
debug_assert!(s.len() >= 8);
unsafe { _mm256_cvtepu16_epi32(_mm_loadu_si128(s.as_ptr() as *const __m128i)) }
}
#[inline(always)]
fn store_i32x8_u16_max(a: &mut [u16], v: __m256i, bitdepth_max: u16) {
debug_assert!(a.len() >= 8);
let v = unsafe {
let zero = _mm256_setzero_si256();
let maxv = _mm256_set1_epi32(bitdepth_max as i32);
_mm256_min_epi32(_mm256_max_epi32(v, zero), maxv)
};
let packed =
unsafe { _mm_packus_epi32(_mm256_castsi256_si128(v), _mm256_extracti128_si256::<1>(v)) };
unsafe { _mm_storeu_si128(a.as_mut_ptr() as *mut __m128i, packed) };
}
#[inline]
#[target_feature(enable = "avx2")]
fn dr_filter8_hbd_avx2(
f: &crate::ipred::DrFilter4Tap,
bitdepth_max: u16,
a0: __m256i,
a1: __m256i,
a2: __m256i,
a3: __m256i,
) -> __m256i {
let acc = _mm256_add_epi32(
_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_set1_epi32(f.a as i32), a0),
_mm256_mullo_epi32(_mm256_set1_epi32(f.b as i32), a1),
),
_mm256_add_epi32(
_mm256_mullo_epi32(_mm256_set1_epi32(f.c as i32), a2),
_mm256_mullo_epi32(_mm256_set1_epi32(f.d as i32), a3),
),
);
let v = _mm256_srai_epi32::<7>(_mm256_add_epi32(acc, _mm256_set1_epi32(64)));
_mm256_min_epi32(
_mm256_max_epi32(v, _mm256_setzero_si256()),
_mm256_set1_epi32(bitdepth_max as i32),
)
}
#[inline]
#[target_feature(enable = "avx2")]
fn z1_luma_row_hbd_avx2(
filt: &[u16],
top_off: usize,
base0: i32,
max_base_x: i32,
fill: u16,
f: &crate::ipred::DrFilter4Tap,
dst_row: &mut [u16],
w: usize,
bitdepth_max: u16,
) {
let n_filter = ((max_base_x - base0 + 1).max(0) as usize).min(w);
let base_const = (top_off as i32 + base0) as usize;
let (body, fill_tail) = dst_row[..w].split_at_mut(n_filter);
let (chunks16, rem16) = body.as_chunks_mut::<16>();
for (ci, d) in chunks16.iter_mut().enumerate() {
let bi = base_const + ci * 16;
let (d0, d1) = d.split_at_mut(8);
let v = dr_filter8_hbd_avx2(
f,
bitdepth_max,
load_u16x8_i32_slice(&filt[bi - 1..]),
load_u16x8_i32_slice(&filt[bi..]),
load_u16x8_i32_slice(&filt[bi + 1..]),
load_u16x8_i32_slice(&filt[bi + 2..]),
);
store_i32x8_u16_max(d0, v, bitdepth_max);
let bi = bi + 8;
let v = dr_filter8_hbd_avx2(
f,
bitdepth_max,
load_u16x8_i32_slice(&filt[bi - 1..]),
load_u16x8_i32_slice(&filt[bi..]),
load_u16x8_i32_slice(&filt[bi + 1..]),
load_u16x8_i32_slice(&filt[bi + 2..]),
);
store_i32x8_u16_max(d1, v, bitdepth_max);
}
let done16 = chunks16.len() * 16;
let (chunks, rem) = rem16.as_chunks_mut::<8>();
for (ci, d) in chunks.iter_mut().enumerate() {
let bi = base_const + done16 + ci * 8;
let v = dr_filter8_hbd_avx2(
f,
bitdepth_max,
load_u16x8_i32_slice(&filt[bi - 1..]),
load_u16x8_i32_slice(&filt[bi..]),
load_u16x8_i32_slice(&filt[bi + 1..]),
load_u16x8_i32_slice(&filt[bi + 2..]),
);
store_i32x8_u16_max(d, v, bitdepth_max);
}
let base_x = done16 + chunks.len() * 8;
for (xi, d) in rem.iter_mut().enumerate() {
let bi = base_const + base_x + xi;
let v = f.a as i32 * filt[bi - 1] as i32
+ f.b as i32 * filt[bi] as i32
+ f.c as i32 * filt[bi + 1] as i32
+ f.d as i32 * filt[bi + 2] as i32;
*d = (((v + 64) >> 7).clamp(0, bitdepth_max as i32)) as u16;
}
fill_tail.fill(fill);
}
#[inline]
#[target_feature(enable = "avx2")]
fn z_chroma_hbd8_avx2(a: &[u16], b: &[u16], weights: __m256i, rnd: __m256i) -> __m256i {
debug_assert!(a.len() >= 8);
debug_assert!(b.len() >= 8);
let a = unsafe { _mm_loadu_si128(a.as_ptr() as *const __m128i) };
let b = unsafe { _mm_loadu_si128(b.as_ptr() as *const __m128i) };
let lo = _mm_unpacklo_epi16(a, b);
let hi = _mm_unpackhi_epi16(a, b);
let pairs = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(lo), hi);
_mm256_srai_epi32::<5>(_mm256_add_epi32(_mm256_madd_epi16(pairs, weights), rnd))
}
#[inline]
#[target_feature(enable = "avx2")]
fn z3_chroma_hbd8_avx2(filt: &[u16], bi: i32, weights: __m256i, rnd: __m256i) -> __m256i {
let mut pairs = [0i16; 16];
for lane in 0..8 {
let idx = (bi - lane as i32) as usize;
pairs[lane * 2] = filt[idx] as i16;
pairs[lane * 2 + 1] = filt[idx - 1] as i16;
}
let samples = unsafe { _mm256_loadu_si256(pairs.as_ptr() as *const __m256i) };
_mm256_srai_epi32::<5>(_mm256_add_epi32(_mm256_madd_epi16(samples, weights), rnd))
}
#[inline]
#[target_feature(enable = "avx2")]
fn z1_chroma_row_hbd_avx2(
filt: &[u16],
top_off: usize,
base0: i32,
max_base_x: i32,
fill: u16,
shift: usize,
dst_row: &mut [u16],
w: usize,
bitdepth_max: u16,
) {
let n_filter = ((max_base_x - base0 + 1).max(0) as usize).min(w);
let weights = _mm256_set1_epi32((((shift as u32) << 16) | ((32 - shift) as u32)) as i32);
let rnd = _mm256_set1_epi32(16);
let base_const = (top_off as i32 + base0) as usize;
let (body, fill_tail) = dst_row[..w].split_at_mut(n_filter);
let (chunks16, rem16) = body.as_chunks_mut::<16>();
for (ci, d) in chunks16.iter_mut().enumerate() {
let bi = base_const + ci * 16;
let (d0, d1) = d.split_at_mut(8);
let v = z_chroma_hbd8_avx2(&filt[bi..], &filt[bi + 1..], weights, rnd);
store_i32x8_u16_max(d0, v, bitdepth_max);
let bi = bi + 8;
let v = z_chroma_hbd8_avx2(&filt[bi..], &filt[bi + 1..], weights, rnd);
store_i32x8_u16_max(d1, v, bitdepth_max);
}
let done16 = chunks16.len() * 16;
let (chunks, rem) = rem16.as_chunks_mut::<8>();
for (ci, d) in chunks.iter_mut().enumerate() {
let bi = base_const + done16 + ci * 8;
let v = z_chroma_hbd8_avx2(&filt[bi..], &filt[bi + 1..], weights, rnd);
store_i32x8_u16_max(d, v, bitdepth_max);
}
let base_x = done16 + chunks.len() * 8;
for (xi, d) in rem.iter_mut().enumerate() {
let bi = base_const + base_x + xi;
let v = (32 - shift as i32) * filt[bi] as i32 + shift as i32 * filt[bi + 1] as i32;
*d = (((v + 16) >> 5).clamp(0, bitdepth_max as i32)) as u16;
}
fill_tail.fill(fill);
}
#[inline]
#[target_feature(enable = "avx2")]
fn z1_row_hbd_avx2(
filt: &[u16],
top_off: usize,
max_base_x: i32,
is_luma: bool,
ypos: i32,
dst_row: &mut [u16],
w: usize,
bitdepth_max: u16,
) -> Option<u16> {
let base0 = ypos >> 6;
let fill = filt[top_off + max_base_x as usize];
if base0 > max_base_x {
return Some(fill);
}
let shift = ((ypos & 0x3F) >> 1) as usize;
let f = &crate::ipred::DR_INTERP_FILTER[shift];
if is_luma {
z1_luma_row_hbd_avx2(
filt,
top_off,
base0,
max_base_x,
fill,
f,
dst_row,
w,
bitdepth_max,
);
} else {
z1_chroma_row_hbd_avx2(
filt,
top_off,
base0,
max_base_x,
fill,
shift,
dst_row,
w,
bitdepth_max,
);
}
None
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "avx2")]
pub(crate) fn ipred_z1_hbd_avx2(
dst: &mut [u16],
stride: usize,
tl: &[u16],
o: usize,
w: usize,
h: usize,
angle: i32,
max_width: i32,
max_height: i32,
ibp_weights: &[[[u8; 16]; 16]; 7],
bitdepth_max: u16,
) {
use crate::levels::*;
let mrl_mul = angle & ANGLE_MULTI_MRL_FLAG != 0;
let is_luma = angle & ANGLE_IS_LUMA != 0;
let enable_ibp = angle & ANGLE_IBP_FLAG != 0;
let mrl_idx = ((angle & ANGLE_MRL_IDX_MASK) >> ANGLE_MRL_IDX_SHIFT) as usize;
let a = angle & 511;
if mrl_mul {
let e_stride = (w + h) * 2 + mrl_idx * 3 + 1;
let mut tmp = [0u16; 64 * 64];
let base_angle = a | ANGLE_IS_LUMA;
let first_angle = base_angle | ((mrl_idx as i32) << ANGLE_MRL_IDX_SHIFT);
ipred_z1_hbd_avx2(
tmp.as_mut_slice(),
64,
tl,
o,
w,
h,
first_angle,
max_width,
max_height,
ibp_weights,
bitdepth_max,
);
ipred_z1_hbd_avx2(
dst,
stride,
tl,
o + e_stride,
w,
h,
base_angle,
max_width,
max_height,
ibp_weights,
bitdepth_max,
);
avg_pred_hbd_avx2(dst, stride, tmp.as_slice(), w, h);
return;
}
if enable_ibp {
let angle_flags = angle & !(511 | ANGLE_IBP_FLAG);
let mode_idx = (10 - (a >> 3)).min(6) as usize;
let mut tmp = [0u16; 64 * 64];
ipred_z1_hbd_avx2(
dst,
stride,
tl,
o,
w,
h,
angle & !ANGLE_IBP_FLAG,
max_width,
max_height,
ibp_weights,
bitdepth_max,
);
ipred_z3_hbd_avx2(
tmp.as_mut_slice(),
64,
tl,
o,
w,
h,
(180 + a) | angle_flags,
max_width,
max_height,
ibp_weights,
bitdepth_max,
);
ibp_blend_hbd_avx2(
dst,
stride,
tmp.as_slice(),
w,
h,
false,
&ibp_weights[mode_idx],
bitdepth_max,
);
return;
}
let is_sm_t = angle & ANGLE_SMOOTH_TOP_EDGE_FLAG != 0;
let enable_intra_edge_filter = angle & ANGLE_USE_EDGE_FILTER_FLAG != 0;
let have_top = angle & ANGLE_HAS_TOP_FLAG != 0;
let dx = crate::tables::DR_INTRA_DERIVATIVE[a as usize] as i32;
let max_base_x = (w + h) as i32 - 1 + (mrl_idx as i32 * 2);
let mut filt = [0u16; 141];
let top_off = 2usize + mrl_idx;
let sz = 1 + mrl_idx + w + h + mrl_idx * 2;
let str = if enable_intra_edge_filter && have_top && mrl_idx == 0 {
crate::ipred::filter_strength((w + h) as i32, 90 - a, is_sm_t)
} else {
0
};
if str > 0 {
crate::ipred::filter_edge(
&mut filt[1..],
sz,
1,
sz as i32 + max_width - w as i32,
&tl[o..],
0,
sz as i32,
str as usize,
);
} else {
filt[1..1 + sz].copy_from_slice(&tl[o..o + sz]);
}
filt[0] = filt[1];
filt[sz + 1] = filt[sz];
filt[sz + 2] = filt[sz + 1];
let mut y = 0usize;
let mut ypos = dx * (1 + mrl_idx as i32);
while y + 1 < h {
let row0_off = y * stride;
if let Some(fill) = z1_row_hbd_avx2(
&filt,
top_off,
max_base_x,
is_luma,
ypos,
&mut dst[row0_off..row0_off + w],
w,
bitdepth_max,
) {
for row in dst[row0_off..].chunks_mut(stride).take(h - y) {
row[..w].fill(fill);
}
return;
}
ypos += dx;
let row1_off = row0_off + stride;
if let Some(fill) = z1_row_hbd_avx2(
&filt,
top_off,
max_base_x,
is_luma,
ypos,
&mut dst[row1_off..row1_off + w],
w,
bitdepth_max,
) {
for row in dst[row1_off..].chunks_mut(stride).take(h - y - 1) {
row[..w].fill(fill);
}
return;
}
ypos += dx;
y += 2;
}
if y < h {
let dst_row = &mut dst[y * stride..y * stride + w];
if let Some(fill) = z1_row_hbd_avx2(
&filt,
top_off,
max_base_x,
is_luma,
ypos,
dst_row,
w,
bitdepth_max,
) {
dst_row.fill(fill);
}
}
}
#[inline(always)]
fn setr_i32x8(a0: i32, a1: i32, a2: i32, a3: i32, a4: i32, a5: i32, a6: i32, a7: i32) -> __m256i {
unsafe { _mm256_setr_epi32(a0, a1, a2, a3, a4, a5, a6, a7) }
}
#[inline]
#[target_feature(enable = "avx2")]
fn z3_luma_col_hbd_avx2(
filt: &[u16],
left_off: usize,
base0: i32,
max_base_y: i32,
fill: u16,
f: &crate::ipred::DrFilter4Tap,
col: &mut [u16],
h: usize,
bitdepth_max: u16,
) {
let n_filter = ((max_base_y - base0 + 1).max(0) as usize).min(h);
let lob = left_off as i32 - base0;
let mut y = 0usize;
while y + 8 <= n_filter {
let bi = lob - y as i32;
let v = dr_filter8_hbd_avx2(
f,
bitdepth_max,
setr_i32x8(
filt[(bi + 1) as usize] as i32,
filt[bi as usize] as i32,
filt[(bi - 1) as usize] as i32,
filt[(bi - 2) as usize] as i32,
filt[(bi - 3) as usize] as i32,
filt[(bi - 4) as usize] as i32,
filt[(bi - 5) as usize] as i32,
filt[(bi - 6) as usize] as i32,
),
setr_i32x8(
filt[bi as usize] as i32,
filt[(bi - 1) as usize] as i32,
filt[(bi - 2) as usize] as i32,
filt[(bi - 3) as usize] as i32,
filt[(bi - 4) as usize] as i32,
filt[(bi - 5) as usize] as i32,
filt[(bi - 6) as usize] as i32,
filt[(bi - 7) as usize] as i32,
),
setr_i32x8(
filt[(bi - 1) as usize] as i32,
filt[(bi - 2) as usize] as i32,
filt[(bi - 3) as usize] as i32,
filt[(bi - 4) as usize] as i32,
filt[(bi - 5) as usize] as i32,
filt[(bi - 6) as usize] as i32,
filt[(bi - 7) as usize] as i32,
filt[(bi - 8) as usize] as i32,
),
setr_i32x8(
filt[(bi - 2) as usize] as i32,
filt[(bi - 3) as usize] as i32,
filt[(bi - 4) as usize] as i32,
filt[(bi - 5) as usize] as i32,
filt[(bi - 6) as usize] as i32,
filt[(bi - 7) as usize] as i32,
filt[(bi - 8) as usize] as i32,
filt[(bi - 9) as usize] as i32,
),
);
store_i32x8_u16_max(&mut col[y..], v, bitdepth_max);
y += 8;
}
while y < n_filter {
let bi = (lob - y as i32) as usize;
let v = f.a as i32 * filt[bi + 1] as i32
+ f.b as i32 * filt[bi] as i32
+ f.c as i32 * filt[bi - 1] as i32
+ f.d as i32 * filt[bi - 2] as i32;
col[y] = (((v + 64) >> 7).clamp(0, bitdepth_max as i32)) as u16;
y += 1;
}
col[n_filter..h].fill(fill);
}
#[inline]
#[target_feature(enable = "avx2")]
fn z3_chroma_col_hbd_avx2(
filt: &[u16],
left_off: usize,
base0: i32,
max_base_y: i32,
fill: u16,
shift: usize,
col: &mut [u16],
h: usize,
bitdepth_max: u16,
) {
let n_filter = ((max_base_y - base0 + 1).max(0) as usize).min(h);
let weights = _mm256_set1_epi32((((shift as u32) << 16) | ((32 - shift) as u32)) as i32);
let rnd = _mm256_set1_epi32(16);
let lob = left_off as i32 - base0;
let mut y = 0usize;
while y + 8 <= n_filter {
let bi = lob - y as i32;
let v = z3_chroma_hbd8_avx2(filt, bi, weights, rnd);
store_i32x8_u16_max(&mut col[y..], v, bitdepth_max);
y += 8;
}
while y < n_filter {
let bi = (lob - y as i32) as usize;
let v = (32 - shift as i32) * filt[bi] as i32 + shift as i32 * filt[bi - 1] as i32;
col[y] = (((v + 16) >> 5).clamp(0, bitdepth_max as i32)) as u16;
y += 1;
}
col[n_filter..h].fill(fill);
}
#[inline]
fn store_z3_cols4_u16(dst: &mut [u16], stride: usize, x: usize, h: usize, cols: &[[u16; 128]; 4]) {
for y in 0..h {
let row_off = y * stride + x;
let v = u64::from(cols[0][y])
| (u64::from(cols[1][y]) << 16)
| (u64::from(cols[2][y]) << 32)
| (u64::from(cols[3][y]) << 48);
unsafe {
core::ptr::write_unaligned(dst.as_mut_ptr().add(row_off).cast::<u64>(), v);
}
}
}
#[inline]
fn store_z3_cols2_u16(dst: &mut [u16], stride: usize, x: usize, h: usize, cols: &[[u16; 128]; 2]) {
for y in 0..h {
let row_off = y * stride + x;
dst[row_off] = cols[0][y];
dst[row_off + 1] = cols[1][y];
}
}
#[inline]
fn store_z3_col_u16(dst: &mut [u16], stride: usize, x: usize, h: usize, col: &[u16; 128]) {
for y in 0..h {
dst[y * stride + x] = col[y];
}
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "avx2")]
pub(crate) fn ipred_z3_hbd_avx2(
dst: &mut [u16],
stride: usize,
tl: &[u16],
o: usize,
w: usize,
h: usize,
angle: i32,
max_width: i32,
max_height: i32,
ibp_weights: &[[[u8; 16]; 16]; 7],
bitdepth_max: u16,
) {
use crate::levels::*;
let mrl_mul = angle & ANGLE_MULTI_MRL_FLAG != 0;
let is_luma = angle & ANGLE_IS_LUMA != 0;
let enable_ibp = angle & ANGLE_IBP_FLAG != 0;
let mrl_idx = ((angle & ANGLE_MRL_IDX_MASK) >> ANGLE_MRL_IDX_SHIFT) as usize;
let a = angle & 511;
if mrl_mul {
let e_stride = (w + h) * 2 + mrl_idx * 3 + 1;
let mut tmp = [0u16; 64 * 64];
let base_angle = a | ANGLE_IS_LUMA;
let first_angle = base_angle | ((mrl_idx as i32) << ANGLE_MRL_IDX_SHIFT);
ipred_z3_hbd_avx2(
tmp.as_mut_slice(),
64,
tl,
o,
w,
h,
first_angle,
max_width,
max_height,
ibp_weights,
bitdepth_max,
);
ipred_z3_hbd_avx2(
dst,
stride,
tl,
o + e_stride,
w,
h,
base_angle,
max_width,
max_height,
ibp_weights,
bitdepth_max,
);
avg_pred_hbd_avx2(dst, stride, tmp.as_slice(), w, h);
return;
}
if enable_ibp {
if h > 64 {
return crate::sse::ipred_z3_hbd_sse41(
dst,
stride,
tl,
o,
w,
h,
angle,
max_width,
max_height,
ibp_weights,
bitdepth_max,
);
}
let angle_flags = angle & !(511 | ANGLE_IBP_FLAG);
let mode_idx = ((a - 183) >> 3).min(6) as usize;
let mut tmp = [0u16; 64 * 64];
ipred_z3_hbd_avx2(
dst,
stride,
tl,
o,
w,
h,
angle & !ANGLE_IBP_FLAG,
max_width,
max_height,
ibp_weights,
bitdepth_max,
);
ipred_z1_hbd_avx2(
tmp.as_mut_slice(),
64,
tl,
o,
w,
h,
(a - 180) | angle_flags,
max_width,
max_height,
ibp_weights,
bitdepth_max,
);
ibp_blend_hbd_avx2(
dst,
stride,
tmp.as_slice(),
w,
h,
true,
&ibp_weights[mode_idx],
bitdepth_max,
);
return;
}
if h > 64 {
return crate::sse::ipred_z3_hbd_sse41(
dst,
stride,
tl,
o,
w,
h,
angle,
max_width,
max_height,
ibp_weights,
bitdepth_max,
);
}
let is_sm_l = angle & ANGLE_SMOOTH_LEFT_EDGE_FLAG != 0;
let enable_intra_edge_filter = angle & ANGLE_USE_EDGE_FILTER_FLAG != 0;
let have_left = angle & ANGLE_HAS_LEFT_FLAG != 0;
let dy = crate::tables::DR_INTRA_DERIVATIVE[(270 - a) as usize] as i32;
let max_base_y = (w + h) as i32 - 1 + (mrl_idx as i32 * 2);
let mut filt = [0u16; 141];
let left_off = 1 + w + h + mrl_idx * 2;
let sz = 1 + mrl_idx + w + h + mrl_idx * 2;
let str = if enable_intra_edge_filter && have_left && mrl_idx == 0 {
crate::ipred::filter_strength((w + h) as i32, a - 180, is_sm_l)
} else {
0
};
if str > 0 {
crate::ipred::filter_edge(
&mut filt[2..],
sz,
h as i32 - max_height,
sz as i32 - 1,
&tl[o + 1 - sz..],
0,
sz as i32,
str as usize,
);
} else {
filt[2..2 + sz].copy_from_slice(&tl[o + 1 - sz..o + 1]);
}
filt[0] = filt[2];
filt[1] = filt[2];
filt[sz + 2] = filt[sz + 1];
let mut cols4 = [[0u16; 128]; 4];
let mut cols2 = [[0u16; 128]; 2];
let mut col = [0u16; 128];
let mut ypos = dy * (1 + mrl_idx as i32);
let mut x = 0usize;
while x + 4 <= w {
for col in &mut cols4 {
let shift = ((ypos & 0x3F) >> 1) as usize;
let f = &crate::ipred::DR_INTERP_FILTER[shift];
let base0 = ypos >> 6;
let fill = filt[left_off - max_base_y as usize];
if is_luma {
z3_luma_col_hbd_avx2(
&filt,
left_off,
base0,
max_base_y,
fill,
f,
col,
h,
bitdepth_max,
);
} else {
z3_chroma_col_hbd_avx2(
&filt,
left_off,
base0,
max_base_y,
fill,
shift,
col,
h,
bitdepth_max,
);
}
ypos += dy;
}
store_z3_cols4_u16(dst, stride, x, h, &cols4);
x += 4;
}
if x + 2 <= w {
for col in &mut cols2 {
let shift = ((ypos & 0x3F) >> 1) as usize;
let f = &crate::ipred::DR_INTERP_FILTER[shift];
let base0 = ypos >> 6;
let fill = filt[left_off - max_base_y as usize];
if is_luma {
z3_luma_col_hbd_avx2(
&filt,
left_off,
base0,
max_base_y,
fill,
f,
col,
h,
bitdepth_max,
);
} else {
z3_chroma_col_hbd_avx2(
&filt,
left_off,
base0,
max_base_y,
fill,
shift,
col,
h,
bitdepth_max,
);
}
ypos += dy;
}
store_z3_cols2_u16(dst, stride, x, h, &cols2);
x += 2;
}
while x < w {
let shift = ((ypos & 0x3F) >> 1) as usize;
let f = &crate::ipred::DR_INTERP_FILTER[shift];
let base0 = ypos >> 6;
let fill = filt[left_off - max_base_y as usize];
if is_luma {
z3_luma_col_hbd_avx2(
&filt,
left_off,
base0,
max_base_y,
fill,
f,
&mut col,
h,
bitdepth_max,
);
} else {
z3_chroma_col_hbd_avx2(
&filt,
left_off,
base0,
max_base_y,
fill,
shift,
&mut col,
h,
bitdepth_max,
);
}
store_z3_col_u16(dst, stride, x, h, &col);
ypos += dy;
x += 1;
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn z2_top_span_hbd_avx2(
filt: &[u16],
top_off: usize,
mut xpos: i32,
f: &crate::ipred::DrFilter4Tap,
dst_row: &mut [u16],
x_start: usize,
w: usize,
bitdepth_max: u16,
) {
let mut x = x_start;
while x + 8 <= w {
let base_x = xpos >> 6;
let ti0 = top_off as i32 + base_x;
if ti0 + 1 < 0 || ti0 + 12 > filt.len() as i32 {
break;
}
let sa = (ti0 + 1) as usize;
let v = dr_filter8_hbd_avx2(
f,
bitdepth_max,
load_u16x8_i32_slice(&filt[sa..]),
load_u16x8_i32_slice(&filt[sa + 1..]),
load_u16x8_i32_slice(&filt[sa + 2..]),
load_u16x8_i32_slice(&filt[sa + 3..]),
);
store_i32x8_u16_max(&mut dst_row[x..], v, bitdepth_max);
x += 8;
xpos += 64 * 8;
}
while x < w {
let base_x = xpos >> 6;
let ti = top_off as i32 + base_x;
let v = f.a as i32 * filt[(ti + 1) as usize] as i32
+ f.b as i32 * filt[(ti + 2) as usize] as i32
+ f.c as i32 * filt[(ti + 3) as usize] as i32
+ f.d as i32 * filt[(ti + 4) as usize] as i32;
dst_row[x] = (((v + 64) >> 7).clamp(0, bitdepth_max as i32)) as u16;
x += 1;
xpos += 64;
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn z2_top_span_chroma_hbd_avx2(
filt: &[u16],
top_off: usize,
mut xpos: i32,
shift: usize,
dst_row: &mut [u16],
x_start: usize,
w: usize,
bitdepth_max: u16,
) {
let iw = _mm256_set1_epi32((32 - shift) as i32);
let sw = _mm256_set1_epi32(shift as i32);
let rnd = _mm256_set1_epi32(16);
let mut x = x_start;
while x + 8 <= w {
let base_x = xpos >> 6;
let ti0 = top_off as i32 + base_x;
if ti0 + 2 < 0 || ti0 + 11 > filt.len() as i32 {
break;
}
let sa = (ti0 + 2) as usize;
let a = load_u16x8_i32_slice(&filt[sa..]);
let b = load_u16x8_i32_slice(&filt[sa + 1..]);
let v = _mm256_srli_epi32::<5>(_mm256_add_epi32(
_mm256_add_epi32(_mm256_mullo_epi32(a, iw), _mm256_mullo_epi32(b, sw)),
rnd,
));
store_i32x8_u16_max(&mut dst_row[x..], v, bitdepth_max);
x += 8;
xpos += 64 * 8;
}
while x < w {
let base_x = xpos >> 6;
let ti = top_off as i32 + base_x;
let v = (32 - shift as i32) * filt[(ti + 2) as usize] as i32
+ shift as i32 * filt[(ti + 3) as usize] as i32;
dst_row[x] = (((v + 16) >> 5).clamp(0, bitdepth_max as i32)) as u16;
x += 1;
xpos += 64;
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn z2_left_span_luma_hbd_avx2(
filt: &[u16],
left_off: usize,
y: usize,
dy: i32,
mrl_idx: usize,
dst_row: &mut [u16],
mut x: usize,
w: usize,
mut xpos: i32,
bitdepth_max: u16,
) -> (usize, i32) {
let limit = -(64 * (1 + mrl_idx as i32));
let rnd = _mm256_set1_epi32(64);
while x + 8 <= w && xpos + 64 * 7 < limit {
let mut c01 = [0i16; 16];
let mut c23 = [0i16; 16];
let mut p01 = [0i16; 16];
let mut p23 = [0i16; 16];
for i in 0..8 {
let xpos_l = (x + i + 1) as i32;
let ypos_l = ((y as i32) << 6) - (xpos_l + mrl_idx as i32) * dy;
let base_y = ypos_l >> 6;
let shift = ((ypos_l & 0x3F) >> 1) as usize;
let bi = (left_off as i32 - base_y) as usize;
let f = &crate::ipred::DR_INTERP_FILTER[shift];
let j = i * 2;
c01[j] = f.a as i16;
c01[j + 1] = f.b as i16;
c23[j] = f.c as i16;
c23[j + 1] = f.d as i16;
p01[j] = filt[bi - 1] as i16;
p01[j + 1] = filt[bi - 2] as i16;
p23[j] = filt[bi - 3] as i16;
p23[j + 1] = filt[bi - 4] as i16;
}
let acc = _mm256_add_epi32(
_mm256_madd_epi16(unsafe { _mm256_loadu_si256(c01.as_ptr().cast()) }, unsafe {
_mm256_loadu_si256(p01.as_ptr().cast())
}),
_mm256_madd_epi16(unsafe { _mm256_loadu_si256(c23.as_ptr().cast()) }, unsafe {
_mm256_loadu_si256(p23.as_ptr().cast())
}),
);
let v = _mm256_srai_epi32::<7>(_mm256_add_epi32(acc, rnd));
store_i32x8_u16_max(&mut dst_row[x..], v, bitdepth_max);
x += 8;
xpos += 64 * 8;
}
while x < w && xpos < limit {
let xpos_l = (x + 1) as i32;
let ypos_l = ((y as i32) << 6) - (xpos_l + mrl_idx as i32) * dy;
let base_y = ypos_l >> 6;
let shift = ((ypos_l & 0x3F) >> 1) as usize;
let bi = (left_off as i32 - base_y) as usize;
let f = &crate::ipred::DR_INTERP_FILTER[shift];
let v = f.a as i32 * filt[bi - 1] as i32
+ f.b as i32 * filt[bi - 2] as i32
+ f.c as i32 * filt[bi - 3] as i32
+ f.d as i32 * filt[bi - 4] as i32;
dst_row[x] = (((v + 64) >> 7).clamp(0, bitdepth_max as i32)) as u16;
x += 1;
xpos += 64;
}
(x, xpos)
}
#[inline]
#[target_feature(enable = "avx2")]
fn z2_left_span_chroma_hbd_avx2(
filt: &[u16],
left_off: usize,
y: usize,
dy: i32,
mrl_idx: usize,
dst_row: &mut [u16],
mut x: usize,
w: usize,
mut xpos: i32,
bitdepth_max: u16,
) -> (usize, i32) {
let limit = -(64 * (1 + mrl_idx as i32));
let rnd = _mm256_set1_epi32(16);
while x + 8 <= w && xpos + 64 * 7 < limit {
let mut iw = [0i32; 8];
let mut sw = [0i32; 8];
let mut p0 = [0u16; 8];
let mut p1 = [0u16; 8];
for i in 0..8 {
let xpos_l = (x + i + 1) as i32;
let ypos_l = ((y as i32) << 6) - (xpos_l + mrl_idx as i32) * dy;
let base_y = ypos_l >> 6;
let shift = ((ypos_l & 0x3F) >> 1) as i32;
let bi = (left_off as i32 - base_y) as usize;
iw[i] = 32 - shift;
sw[i] = shift;
p0[i] = filt[bi - 2];
p1[i] = filt[bi - 3];
}
let v = _mm256_srli_epi32::<5>(_mm256_add_epi32(
_mm256_add_epi32(
_mm256_mullo_epi32(load_u16x8_i32_slice(&p0), unsafe {
_mm256_loadu_si256(iw.as_ptr().cast())
}),
_mm256_mullo_epi32(load_u16x8_i32_slice(&p1), unsafe {
_mm256_loadu_si256(sw.as_ptr().cast())
}),
),
rnd,
));
store_i32x8_u16_max(&mut dst_row[x..], v, bitdepth_max);
x += 8;
xpos += 64 * 8;
}
while x < w && xpos < limit {
let xpos_l = (x + 1) as i32;
let ypos_l = ((y as i32) << 6) - (xpos_l + mrl_idx as i32) * dy;
let base_y = ypos_l >> 6;
let shift = ((ypos_l & 0x3F) >> 1) as usize;
let bi = (left_off as i32 - base_y) as usize;
let v = (32 - shift as i32) * filt[bi - 2] as i32 + shift as i32 * filt[bi - 3] as i32;
dst_row[x] = (((v + 16) >> 5).clamp(0, bitdepth_max as i32)) as u16;
x += 1;
xpos += 64;
}
(x, xpos)
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "avx2")]
pub(crate) fn ipred_z2_hbd_avx2(
dst: &mut [u16],
stride: usize,
tl: &[u16],
o: usize,
w: usize,
h: usize,
angle: i32,
max_width: i32,
max_height: i32,
bitdepth_max: u16,
) {
use crate::levels::*;
let mrl_mul = angle & ANGLE_MULTI_MRL_FLAG != 0;
let is_luma = angle & ANGLE_IS_LUMA != 0;
let mrl_idx = ((angle & ANGLE_MRL_IDX_MASK) >> ANGLE_MRL_IDX_SHIFT) as usize;
let a = angle & 511;
if mrl_mul {
let e_stride = (w + h) * 2 + mrl_idx * 3 + 1;
let mut tmp = [0u16; 64 * 64];
let base_angle = a | ANGLE_IS_LUMA;
let first_angle = base_angle | ((mrl_idx as i32) << ANGLE_MRL_IDX_SHIFT);
ipred_z2_hbd_avx2(
tmp.as_mut_slice(),
64,
tl,
o,
w,
h,
first_angle,
max_width,
max_height,
bitdepth_max,
);
ipred_z2_hbd_avx2(
dst,
stride,
tl,
o + e_stride,
w,
h,
base_angle,
max_width,
max_height,
bitdepth_max,
);
avg_pred_hbd_avx2(dst, stride, tmp.as_slice(), w, h);
return;
}
let is_sm_l = angle & ANGLE_SMOOTH_LEFT_EDGE_FLAG != 0;
let is_sm_t = angle & ANGLE_SMOOTH_TOP_EDGE_FLAG != 0;
let enable_intra_edge_filter = angle & ANGLE_USE_EDGE_FILTER_FLAG != 0;
let have_top = angle & ANGLE_HAS_TOP_FLAG != 0;
let have_left = angle & ANGLE_HAS_LEFT_FLAG != 0;
let dy = crate::tables::DR_INTRA_DERIVATIVE[(a - 90) as usize] as i32;
let dx = crate::tables::DR_INTRA_DERIVATIVE[(180 - a) as usize] as i32;
let mut filt = [0u16; 72];
let top_off = mrl_idx;
let sz_t = 1 + w + mrl_idx;
let str_t = if enable_intra_edge_filter && have_top && mrl_idx == 0 {
crate::ipred::filter_strength((w + h) as i32, a - 90, is_sm_t)
} else {
0
};
if str_t > 0 {
crate::ipred::filter_edge(
&mut filt[1..],
sz_t,
1,
sz_t as i32 + max_width - w as i32,
&tl[o..],
0,
sz_t as i32,
str_t as usize,
);
} else {
filt[1..1 + sz_t].copy_from_slice(&tl[o..o + sz_t]);
}
filt[0] = filt[1];
filt[sz_t + 1] = filt[sz_t];
let mut filt2 = [0u16; 72];
let left_off: usize = h + 2;
let sz_l = 1 + h + mrl_idx;
let str_l = if enable_intra_edge_filter && have_left && mrl_idx == 0 {
crate::ipred::filter_strength((w + h) as i32, 180 - a, is_sm_l)
} else {
0
};
if str_l > 0 {
crate::ipred::filter_edge(
&mut filt2[1..],
sz_l,
h as i32 - max_height,
sz_l as i32 - 1,
&tl[o - (h + mrl_idx)..],
0,
sz_l as i32,
str_l as usize,
);
} else {
filt2[1..1 + sz_l].copy_from_slice(&tl[o - (h + mrl_idx)..o + 1]);
}
filt2[1 + sz_l] = filt2[sz_l];
filt2[0] = filt2[1];
for y in 0..h {
let ypos = (y + 1) as i32;
let mut xpos = -(ypos + mrl_idx as i32) * dx;
let mut x = 0usize;
let dst_row = &mut dst[y * stride..y * stride + w];
(x, xpos) = if is_luma {
z2_left_span_luma_hbd_avx2(
&filt2,
left_off,
y,
dy,
mrl_idx,
dst_row,
x,
w,
xpos,
bitdepth_max,
)
} else {
z2_left_span_chroma_hbd_avx2(
&filt2,
left_off,
y,
dy,
mrl_idx,
dst_row,
x,
w,
xpos,
bitdepth_max,
)
};
if x < w {
let shift = ((xpos & 0x3F) >> 1) as usize;
if is_luma {
let f = &crate::ipred::DR_INTERP_FILTER[shift];
z2_top_span_hbd_avx2(&filt, top_off, xpos, f, dst_row, x, w, bitdepth_max);
} else {
z2_top_span_chroma_hbd_avx2(
&filt,
top_off,
xpos,
shift,
dst_row,
x,
w,
bitdepth_max,
);
}
}
}
}
#[allow(clippy::too_many_arguments)]
#[target_feature(enable = "avx2")]
pub(crate) fn ipred_dip_hbd_avx2(
dst: &mut [u16],
stride: usize,
tl: &[u16],
o: usize,
width: usize,
height: usize,
mode: i32,
bitdepth_max: u16,
) {
let trans = (mode & 16) != 0;
let wd = width >> 2;
let hd = height >> 2;
let wl2 = ulog2(wd as u32);
let hl2 = ulog2(hd as u32);
let wrnd = width >> 3;
let hrnd = height >> 3;
let i_t: usize = if trans { 5 } else { 1 };
let i_l: usize = if trans { 1 } else { 5 };
let mut inp = [0i32; 11];
inp[0] = tl[o] as i32;
let mut in_sum = inp[0];
let mut ti = o + 1;
for i in 0..4 {
let mut sum = 0i32;
for _ in 0..wd {
sum += tl[ti] as i32;
ti += 1;
}
inp[i_t + i] = (sum + wrnd as i32) >> wl2;
in_sum += inp[i_t + i];
}
let mut li = o;
for i in 0..4 {
let mut sum = 0i32;
for _ in 0..hd {
li -= 1;
sum += tl[li] as i32;
}
inp[i_l + i] = (sum + hrnd as i32) >> hl2;
in_sum += inp[i_l + i];
}
let mut sum = 0i32;
for x in 0..wd {
sum += tl[o + x + width + 1] as i32;
}
let idx_tr = if trans { 10 } else { 9 };
inp[idx_tr] = (sum + wrnd as i32) >> wl2;
in_sum += inp[idx_tr];
sum = 0;
for y in 0..hd {
sum += tl[o - (y + height + 1)] as i32;
}
let idx_bl = if trans { 9 } else { 10 };
inp[idx_bl] = (sum + hrnd as i32) >> hl2;
in_sum += inp[idx_bl];
let m = (mode & 7) as usize;
let mut uwl2 = wl2 - 1;
let mut dwl2 = 0i32;
if uwl2 < 0 {
dwl2 = -uwl2;
uwl2 = 0;
}
let step_x = 1usize << uwl2;
let dw = 1usize << dwl2;
let mut uhl2 = hl2 - 1;
let mut dhl2 = 0i32;
if uhl2 < 0 {
dhl2 = -uhl2;
uhl2 = 0;
}
let step_y = 1usize << uhl2;
let dh = 1usize << dhl2;
let grid_h = 8usize >> dhl2;
let grid_w = 8usize >> dwl2;
let weights = &DIP_WEIGHTS[m];
let bitdepth_max_i32 = bitdepth_max as i32;
match step_x {
1 => dip_emit_hbd_avx2::<1>(
dst,
stride,
width,
trans,
dw,
dh,
step_y,
grid_h,
grid_w,
&inp,
weights,
in_sum,
bitdepth_max_i32,
),
2 => dip_emit_hbd_avx2::<2>(
dst,
stride,
width,
trans,
dw,
dh,
step_y,
grid_h,
grid_w,
&inp,
weights,
in_sum,
bitdepth_max_i32,
),
4 => dip_emit_hbd_avx2::<4>(
dst,
stride,
width,
trans,
dw,
dh,
step_y,
grid_h,
grid_w,
&inp,
weights,
in_sum,
bitdepth_max_i32,
),
8 => dip_emit_hbd_avx2::<8>(
dst,
stride,
width,
trans,
dw,
dh,
step_y,
grid_h,
grid_w,
&inp,
weights,
in_sum,
bitdepth_max_i32,
),
_ => unreachable!(),
}
if step_x > 1 {
let mut y = step_y - 1;
for _gy in 0..grid_h {
let mut p1 = tl[o - (y + 1)] as i32;
let mut x = 0usize;
let dst_row = &mut dst[y * stride..y * stride + width];
for _gx in 0..grid_w {
let p0 = p1;
p1 = dst_row[x + step_x - 1] as i32;
for z in 0..step_x - 1 {
let z1 = (z + 1) as i32;
dst_row[x + z] = ((p0 * (step_x as i32 - z1) + p1 * z1) >> uwl2) as u16;
}
x += step_x;
}
y += step_y;
}
}
dip_vertical_interp_hbd_avx2(dst, stride, tl, o, width, step_y, uhl2, grid_h);
}