use std::arch::x86_64::*;
use crate::itx_2d::ITX_TMP_PIXELS;
#[inline(always)]
fn with_avx2_itx_i16_scratch<R>(len: usize, f: impl FnOnce(&mut [i16]) -> R) -> R {
match len {
16 => {
let mut scratch = [0i16; 16];
f(&mut scratch)
}
32 => {
let mut scratch = [0i16; 32];
f(&mut scratch)
}
64 => {
let mut scratch = [0i16; 64];
f(&mut scratch)
}
128 => {
let mut scratch = [0i16; 128];
f(&mut scratch)
}
256 => {
let mut scratch = [0i16; 256];
f(&mut scratch)
}
512 => {
let mut scratch = [0i16; 512];
f(&mut scratch)
}
1024 => {
let mut scratch = [0i16; 1024];
f(&mut scratch)
}
_ => unreachable!("unsupported ITX i16 scratch len {}", len),
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct16_i32x4_impl(s: &[__m128i; 16]) -> [__m128i; 16] {
let z = _mm_setzero_si128();
let mut out = [z; 16];
let mut m = 0usize;
while m < 16 {
let mut acc = z;
let mut j = 0usize;
while j < 16 {
let k = _mm_set1_epi32(crate::itx_2d::DCT16_DENSE_KERNEL[j * 16 + m]);
acc = _mm_add_epi32(acc, _mm_mullo_epi32(s[j], k));
j += 1;
}
out[m] = acc;
m += 1;
}
out
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_adst16_i32x4_impl(s: &[__m128i; 16], flip: bool) -> [__m128i; 16] {
let rows = if flip {
&crate::itx_1d::FLIPADST16_KERNEL_ROWS
} else {
&crate::itx_1d::ADST16_KERNEL_ROWS
};
let z = _mm_setzero_si128();
let mut out = [z; 16];
let mut m = 0usize;
while m < 16 {
let row = &rows[m];
let mut acc = z;
let mut j = 0usize;
while j < 16 {
let k = _mm_set1_epi32(row[j] as i32);
acc = _mm_add_epi32(acc, _mm_mullo_epi32(s[j], k));
j += 1;
}
out[m] = acc;
m += 1;
}
out
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_tx16_i32x4_impl(s: &[__m128i; 16], kind: usize) -> [__m128i; 16] {
match kind {
crate::itx_2d::TX_KIND_DCT => avx2_dct16_i32x4_impl(s),
crate::itx_2d::TX_KIND_ADST => avx2_adst16_i32x4_impl(s, false),
crate::itx_2d::TX_KIND_FLIPADST => avx2_adst16_i32x4_impl(s, true),
_ => unreachable!(),
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn iadst_dequant_16x16_avx2_i32_impl(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
if is_rect2 {
iadst_dequant_16x16_avx2_i32_impl_const::<true>(
coeff,
tmp,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
} else {
iadst_dequant_16x16_avx2_i32_impl_const::<false>(
coeff,
tmp,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn iadst_dequant_16x16_avx2_i32_impl_const<const IS_RECT2: bool>(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
debug_assert!(coeff.len() >= 256);
let off = usize::from(crate::scan::LAST_EOB_PER_COL.offset[tx]);
let last_eob = &crate::scan::LAST_EOB_PER_COL.table[off..];
let mut ngrp = 0usize;
while ngrp < 4 {
ngrp += 1;
if eob <= last_eob[ngrp - 1] as i32 {
break;
}
}
let ncols = ngrp * 4;
let rnd = _mm_set1_epi32((1 << shift0) >> 1);
let sh = _mm_cvtsi32_si128(shift0);
let minv = _mm_set1_epi32(row_clip_min);
let maxv = _mm_set1_epi32(row_clip_max);
let rnd256 = _mm256_set1_epi32((1 << shift0) >> 1);
let minv256 = _mm256_set1_epi32(row_clip_min);
let maxv256 = _mm256_set1_epi32(row_clip_max);
let mut y = 0usize;
while y + 8 <= ncols {
let out = avx2_tx16_i32x8_from_coeff8_const::<IS_RECT2>(coeff, y, 16, first_kind);
avx2_store8x8_i32_clip(
tmp,
y * 32,
array_ref8_i32x8(&out, 0),
rnd256,
sh,
minv256,
maxv256,
);
avx2_store8x8_i32_clip(
tmp,
y * 32 + 8,
array_ref8_i32x8(&out, 8),
rnd256,
sh,
minv256,
maxv256,
);
y += 8;
}
if y + 4 <= ncols {
let mut s = [_mm_setzero_si128(); 16];
let mut j = 0usize;
while j < 16 {
let mut v = unsafe { _mm_loadu_si128(coeff.as_ptr().add(y + j * 16).cast()) };
if IS_RECT2 {
v = _mm_srai_epi32::<8>(_mm_add_epi32(
_mm_mullo_epi32(v, _mm_set1_epi32(181)),
_mm_set1_epi32(128),
));
}
s[j] = v;
j += 1;
}
let out = avx2_tx16_i32x4_impl(&s, first_kind);
let mut x = 0usize;
while x < 16 {
let g = [out[x], out[x + 1], out[x + 2], out[x + 3]];
avx2_store4x4_i32_clip(tmp, y * 32 + x, &g, rnd, sh, minv, maxv);
x += 4;
}
y += 4;
}
while y < 16 {
tmp[y * 32..y * 32 + 16].fill(0);
y += 1;
}
coeff[..256].fill(0);
let mut x = 0usize;
while x + 8 <= 16 {
let out = avx2_tx16_i32x8_from_tmp8(tmp, x, second_kind);
let mut m = 0usize;
while m < 16 {
unsafe {
_mm256_storeu_si256(tmp.as_mut_ptr().add(x + m * 32).cast(), out[m]);
}
m += 1;
}
x += 8;
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_store4x4_i32_clip(
dst: &mut [i32; ITX_TMP_PIXELS],
off: usize,
v: &[__m128i; 4],
rnd: __m128i,
sh: __m128i,
minv: __m128i,
maxv: __m128i,
) {
unsafe {
macro_rules! clip {
($x:expr) => {{
_mm_min_epi32(
_mm_max_epi32(_mm_sra_epi32(_mm_add_epi32($x, rnd), sh), minv),
maxv,
)
}};
}
let c0 = clip!(v[0]);
let c1 = clip!(v[1]);
let c2 = clip!(v[2]);
let c3 = clip!(v[3]);
let t0 = _mm_unpacklo_epi32(c0, c1);
let t1 = _mm_unpackhi_epi32(c0, c1);
let t2 = _mm_unpacklo_epi32(c2, c3);
let t3 = _mm_unpackhi_epi32(c2, c3);
let r0 = _mm_unpacklo_epi64(t0, t2);
let r1 = _mm_unpackhi_epi64(t0, t2);
let r2 = _mm_unpacklo_epi64(t1, t3);
let r3 = _mm_unpackhi_epi64(t1, t3);
_mm_storeu_si128(tmp_ptr(dst, off), r0);
_mm_storeu_si128(tmp_ptr(dst, off + 32), r1);
_mm_storeu_si128(tmp_ptr(dst, off + 64), r2);
_mm_storeu_si128(tmp_ptr(dst, off + 96), r3);
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_store8x8_i32_clip(
dst: &mut [i32; ITX_TMP_PIXELS],
off: usize,
v: &[__m256i; 8],
rnd: __m256i,
sh: __m128i,
minv: __m256i,
maxv: __m256i,
) {
unsafe {
debug_assert!(off + 7 * 32 + 8 <= dst.len());
macro_rules! clip {
($x:expr) => {{
_mm256_min_epi32(
_mm256_max_epi32(_mm256_sra_epi32(_mm256_add_epi32($x, rnd), sh), minv),
maxv,
)
}};
}
let r0 = clip!(v[0]);
let r1 = clip!(v[1]);
let r2 = clip!(v[2]);
let r3 = clip!(v[3]);
let r4 = clip!(v[4]);
let r5 = clip!(v[5]);
let r6 = clip!(v[6]);
let r7 = clip!(v[7]);
let t0 = _mm256_unpacklo_epi32(r0, r1);
let t1 = _mm256_unpackhi_epi32(r0, r1);
let t2 = _mm256_unpacklo_epi32(r2, r3);
let t3 = _mm256_unpackhi_epi32(r2, r3);
let t4 = _mm256_unpacklo_epi32(r4, r5);
let t5 = _mm256_unpackhi_epi32(r4, r5);
let t6 = _mm256_unpacklo_epi32(r6, r7);
let t7 = _mm256_unpackhi_epi32(r6, r7);
let u0 = _mm256_unpacklo_epi64(t0, t2);
let u1 = _mm256_unpackhi_epi64(t0, t2);
let u2 = _mm256_unpacklo_epi64(t1, t3);
let u3 = _mm256_unpackhi_epi64(t1, t3);
let u4 = _mm256_unpacklo_epi64(t4, t6);
let u5 = _mm256_unpackhi_epi64(t4, t6);
let u6 = _mm256_unpacklo_epi64(t5, t7);
let u7 = _mm256_unpackhi_epi64(t5, t7);
let ptr = dst.as_mut_ptr().add(off).cast();
_mm256_storeu_si256(ptr, _mm256_permute2x128_si256::<0x20>(u0, u4));
_mm256_storeu_si256(ptr.add(32 / 8), _mm256_permute2x128_si256::<0x20>(u1, u5));
_mm256_storeu_si256(
ptr.add((2 * 32) / 8),
_mm256_permute2x128_si256::<0x20>(u2, u6),
);
_mm256_storeu_si256(
ptr.add((3 * 32) / 8),
_mm256_permute2x128_si256::<0x20>(u3, u7),
);
_mm256_storeu_si256(
ptr.add((4 * 32) / 8),
_mm256_permute2x128_si256::<0x31>(u0, u4),
);
_mm256_storeu_si256(
ptr.add((5 * 32) / 8),
_mm256_permute2x128_si256::<0x31>(u1, u5),
);
_mm256_storeu_si256(
ptr.add((6 * 32) / 8),
_mm256_permute2x128_si256::<0x31>(u2, u6),
);
_mm256_storeu_si256(
ptr.add((7 * 32) / 8),
_mm256_permute2x128_si256::<0x31>(u3, u7),
);
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_store_i32x8_rows(tmp: &mut [i32; ITX_TMP_PIXELS], base: usize, out: &[__m256i; 8]) {
unsafe {
debug_assert!(base + 7 * 32 + 8 <= tmp.len());
let ptr = tmp.as_mut_ptr();
_mm256_storeu_si256(ptr.add(base).cast(), out[0]);
_mm256_storeu_si256(ptr.add(base + 32).cast(), out[1]);
_mm256_storeu_si256(ptr.add(base + 2 * 32).cast(), out[2]);
_mm256_storeu_si256(ptr.add(base + 3 * 32).cast(), out[3]);
_mm256_storeu_si256(ptr.add(base + 4 * 32).cast(), out[4]);
_mm256_storeu_si256(ptr.add(base + 5 * 32).cast(), out[5]);
_mm256_storeu_si256(ptr.add(base + 6 * 32).cast(), out[6]);
_mm256_storeu_si256(ptr.add(base + 7 * 32).cast(), out[7]);
}
}
#[inline(always)]
fn array_ref8_i32x8(src: &[__m256i], off: usize) -> &[__m256i; 8] {
debug_assert!(off + 8 <= src.len());
unsafe { &*(src.as_ptr().add(off) as *const [__m256i; 8]) }
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_store4x4_i16_clip<const STRIDE: usize>(
scratch: &mut [i16],
off: usize,
v0: __m128i,
v1: __m128i,
v2: __m128i,
v3: __m128i,
rnd: __m128i,
sh: __m128i,
minv: __m128i,
maxv: __m128i,
) {
unsafe {
debug_assert!(STRIDE == 4 || STRIDE == 8 || STRIDE == 16 || STRIDE == 32);
debug_assert!(off + 3 * STRIDE + 4 <= scratch.len());
macro_rules! clip {
($x:expr) => {{
_mm_min_epi32(
_mm_max_epi32(_mm_sra_epi32(_mm_add_epi32($x, rnd), sh), minv),
maxv,
)
}};
}
let c0 = clip!(v0);
let c1 = clip!(v1);
let c2 = clip!(v2);
let c3 = clip!(v3);
let t0 = _mm_unpacklo_epi32(c0, c1);
let t1 = _mm_unpackhi_epi32(c0, c1);
let t2 = _mm_unpacklo_epi32(c2, c3);
let t3 = _mm_unpackhi_epi32(c2, c3);
let r0 = _mm_unpacklo_epi64(t0, t2);
let r1 = _mm_unpackhi_epi64(t0, t2);
let r2 = _mm_unpacklo_epi64(t1, t3);
let r3 = _mm_unpackhi_epi64(t1, t3);
_mm_storel_epi64(
scratch.as_mut_ptr().add(off).cast(),
_mm_packs_epi32(r0, _mm_setzero_si128()),
);
_mm_storel_epi64(
scratch.as_mut_ptr().add(off + STRIDE).cast(),
_mm_packs_epi32(r1, _mm_setzero_si128()),
);
_mm_storel_epi64(
scratch.as_mut_ptr().add(off + 2 * STRIDE).cast(),
_mm_packs_epi32(r2, _mm_setzero_si128()),
);
_mm_storel_epi64(
scratch.as_mut_ptr().add(off + 3 * STRIDE).cast(),
_mm_packs_epi32(r3, _mm_setzero_si128()),
);
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_load4_i16_scratch(src: &[i16], off: usize) -> __m128i {
debug_assert!(off + 4 <= src.len());
unsafe { _mm_loadl_epi64(src.as_ptr().add(off).cast()) }
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_load8_i16_scratch(src: &[i16], off: usize) -> __m128i {
debug_assert!(off + 8 <= src.len());
unsafe { _mm_loadu_si128(src.as_ptr().add(off).cast()) }
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_load16_i16_scratch(src: &[i16], off: usize) -> __m256i {
debug_assert!(off + 16 <= src.len());
unsafe { _mm256_loadu_si256(src.as_ptr().add(off).cast()) }
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_pair8_i16(a: __m128i, b: __m128i) -> __m256i {
let lo = _mm_unpacklo_epi16(a, b);
let hi = _mm_unpackhi_epi16(a, b);
_mm256_inserti128_si256::<1>(_mm256_castsi128_si256(lo), hi)
}
#[inline]
fn tmp_ptr(dst: &mut [i32; ITX_TMP_PIXELS], off: usize) -> *mut __m128i {
unsafe { dst.as_mut_ptr().add(off).cast() }
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_load8_i32_coeff_const<const IS_RECT2: bool>(src: &[i32], off: usize) -> __m256i {
debug_assert!(off + 8 <= src.len());
let mut v = unsafe { _mm256_loadu_si256(src.as_ptr().add(off).cast()) };
if IS_RECT2 {
v = _mm256_srai_epi32::<8>(_mm256_add_epi32(
_mm256_mullo_epi32(v, _mm256_set1_epi32(181)),
_mm256_set1_epi32(128),
));
}
v
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct16_i32x8_impl(s: &[__m256i; 16]) -> [__m256i; 16] {
let z = _mm256_setzero_si256();
let mut out = [z; 16];
let mut m = 0usize;
while m < 16 {
let mut acc = z;
let mut j = 0usize;
while j < 16 {
let k = _mm256_set1_epi32(crate::itx_2d::DCT16_DENSE_KERNEL[j * 16 + m]);
acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(s[j], k));
j += 1;
}
out[m] = acc;
m += 1;
}
out
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_adst16_i32x8_impl(s: &[__m256i; 16], flip: bool) -> [__m256i; 16] {
let rows = if flip {
&crate::itx_1d::FLIPADST16_KERNEL_ROWS
} else {
&crate::itx_1d::ADST16_KERNEL_ROWS
};
let z = _mm256_setzero_si256();
let mut out = [z; 16];
let mut m = 0usize;
while m < 16 {
let row = &rows[m];
let mut acc = z;
let mut j = 0usize;
while j < 16 {
let k = _mm256_set1_epi32(row[j] as i32);
acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(s[j], k));
j += 1;
}
out[m] = acc;
m += 1;
}
out
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_tx16_i32x8_impl(s: &[__m256i; 16], kind: usize) -> [__m256i; 16] {
match kind {
crate::itx_2d::TX_KIND_DCT => avx2_dct16_i32x8_impl(s),
crate::itx_2d::TX_KIND_ADST => avx2_adst16_i32x8_impl(s, false),
crate::itx_2d::TX_KIND_FLIPADST => avx2_adst16_i32x8_impl(s, true),
_ => unreachable!(),
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_tx16_i32x8_from_coeff8_const<const IS_RECT2: bool>(
coeff: &[i32],
base: usize,
stride: usize,
kind: usize,
) -> [__m256i; 16] {
let z = _mm256_setzero_si256();
let mut s = [z; 16];
let mut j = 0usize;
while j < 16 {
s[j] = avx2_load8_i32_coeff_const::<IS_RECT2>(coeff, base + j * stride);
j += 1;
}
avx2_tx16_i32x8_impl(&s, kind)
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_tx16_i32x4_from_coeff4_const<const IS_RECT2: bool>(
coeff: &[i32],
base: usize,
stride: usize,
kind: usize,
) -> [__m128i; 16] {
let z = _mm_setzero_si128();
let mut s = [z; 16];
let rect_mul = _mm_set1_epi32(181);
let rect_rnd = _mm_set1_epi32(128);
let mut j = 0usize;
while j < 16 {
let mut v = unsafe { _mm_loadu_si128(coeff.as_ptr().add(base + j * stride).cast()) };
if IS_RECT2 {
v = _mm_srai_epi32::<8>(_mm_add_epi32(_mm_mullo_epi32(v, rect_mul), rect_rnd));
}
s[j] = v;
j += 1;
}
avx2_tx16_i32x4_impl(&s, kind)
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_tx16_i32x8_from_tmp8(
tmp: &[i32; ITX_TMP_PIXELS],
base: usize,
kind: usize,
) -> [__m256i; 16] {
let z = _mm256_setzero_si256();
let mut s = [z; 16];
let mut j = 0usize;
while j < 16 {
unsafe {
s[j] = _mm256_loadu_si256(tmp.as_ptr().add(base + j * 32).cast());
}
j += 1;
}
avx2_tx16_i32x8_impl(&s, kind)
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct32_i32x8_from_coeff8_const<const IS_RECT2: bool>(
coeff: &[i32],
base: usize,
m: usize,
) -> [__m256i; 8] {
let z = _mm256_setzero_si256();
let mut out = [z; 8];
let mut j = 0usize;
while j < 32 {
let v = avx2_load8_i32_coeff_const::<IS_RECT2>(coeff, base + j * 32);
let mut lane = 0usize;
while lane < 8 {
let k = _mm256_set1_epi32(crate::itx_2d::DCT32_DENSE_KERNEL[j * 32 + m + lane]);
out[lane] = _mm256_add_epi32(out[lane], _mm256_mullo_epi32(v, k));
lane += 1;
}
j += 1;
}
out
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct32_i32x8_from_tmp8(tmp: &[i32; ITX_TMP_PIXELS], base: usize, m: usize) -> [__m256i; 8] {
let z = _mm256_setzero_si256();
let mut out = [z; 8];
let mut j = 0usize;
while j < 32 {
let v = unsafe { _mm256_loadu_si256(tmp.as_ptr().add(base + j * 32).cast()) };
let mut lane = 0usize;
while lane < 8 {
let k = _mm256_set1_epi32(crate::itx_2d::DCT32_DENSE_KERNEL[j * 32 + m + lane]);
out[lane] = _mm256_add_epi32(out[lane], _mm256_mullo_epi32(v, k));
lane += 1;
}
j += 1;
}
out
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_tx8_i32x8_from_coeff8_const<const IS_RECT2: bool>(
coeff: &[i32],
base: usize,
kind: usize,
) -> [__m256i; 8] {
let z = _mm256_setzero_si256();
let mut out = [z; 8];
let mut j = 0usize;
while j < 8 {
let v = avx2_load8_i32_coeff_const::<IS_RECT2>(coeff, base + j * 8);
let mut lane = 0usize;
while lane < 8 {
let k = _mm256_set1_epi32(tx8_coeff(kind, lane, j));
out[lane] = _mm256_add_epi32(out[lane], _mm256_mullo_epi32(v, k));
lane += 1;
}
j += 1;
}
out
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_tx8_i32x8_from_tmp8(tmp: &[i32; ITX_TMP_PIXELS], base: usize, kind: usize) -> [__m256i; 8] {
let z = _mm256_setzero_si256();
let mut out = [z; 8];
let mut j = 0usize;
while j < 8 {
let v = unsafe { _mm256_loadu_si256(tmp.as_ptr().add(base + j * 32).cast()) };
let mut lane = 0usize;
while lane < 8 {
let k = _mm256_set1_epi32(tx8_coeff(kind, lane, j));
out[lane] = _mm256_add_epi32(out[lane], _mm256_mullo_epi32(v, k));
lane += 1;
}
j += 1;
}
out
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_tx_dense_i32x8_from_coeff8_const<const IS_RECT2: bool, const W: usize, const H: usize>(
coeff: &[i32],
base: usize,
kind: usize,
m: usize,
) -> [__m256i; 8] {
let z = _mm256_setzero_si256();
let mut out = [z; 8];
let mut j = 0usize;
while j < W {
let v = avx2_load8_i32_coeff_const::<IS_RECT2>(coeff, base + j * H);
let mut lane = 0usize;
while lane < 8 {
let k = _mm256_set1_epi32(avx2_tx_dense_coeff(kind, W, m + lane, j));
out[lane] = _mm256_add_epi32(out[lane], _mm256_mullo_epi32(v, k));
lane += 1;
}
j += 1;
}
out
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_tx_dense_i32x8_from_tmp8<const H: usize>(
vin: &[__m256i; H],
kind: usize,
m: usize,
) -> [__m256i; 8] {
let z = _mm256_setzero_si256();
let mut out = [z; 8];
let mut j = 0usize;
while j < H {
let v = vin[j];
let mut lane = 0usize;
while lane < 8 {
let k = _mm256_set1_epi32(avx2_tx_dense_coeff(kind, H, m + lane, j));
out[lane] = _mm256_add_epi32(out[lane], _mm256_mullo_epi32(v, k));
lane += 1;
}
j += 1;
}
out
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct32_i32x4_from_coeff4_const<const IS_RECT2: bool>(
coeff: &[i32],
base: usize,
m: usize,
) -> [__m128i; 4] {
unsafe {
let z = _mm_setzero_si128();
let mut a0 = z;
let mut a1 = z;
let mut a2 = z;
let mut a3 = z;
let rect_mul = _mm_set1_epi32(181);
let rect_rnd = _mm_set1_epi32(128);
let mut j = 0usize;
while j < 32 {
let mut v = _mm_loadu_si128(coeff.as_ptr().add(base + j * 32).cast());
if IS_RECT2 {
v = _mm_srai_epi32::<8>(_mm_add_epi32(_mm_mullo_epi32(v, rect_mul), rect_rnd));
}
a0 = _mm_add_epi32(
a0,
_mm_mullo_epi32(
v,
_mm_set1_epi32(crate::itx_2d::DCT32_DENSE_KERNEL[j * 32 + m]),
),
);
a1 = _mm_add_epi32(
a1,
_mm_mullo_epi32(
v,
_mm_set1_epi32(crate::itx_2d::DCT32_DENSE_KERNEL[j * 32 + m + 1]),
),
);
a2 = _mm_add_epi32(
a2,
_mm_mullo_epi32(
v,
_mm_set1_epi32(crate::itx_2d::DCT32_DENSE_KERNEL[j * 32 + m + 2]),
),
);
a3 = _mm_add_epi32(
a3,
_mm_mullo_epi32(
v,
_mm_set1_epi32(crate::itx_2d::DCT32_DENSE_KERNEL[j * 32 + m + 3]),
),
);
j += 1;
}
[a0, a1, a2, a3]
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_tx8_i32x4_from_coeff4_const<const IS_RECT2: bool>(
coeff: &[i32],
base: usize,
kind: usize,
m: usize,
) -> [__m128i; 4] {
unsafe {
let z = _mm_setzero_si128();
let mut a0 = z;
let mut a1 = z;
let mut a2 = z;
let mut a3 = z;
let rect_mul = _mm_set1_epi32(181);
let rect_rnd = _mm_set1_epi32(128);
let mut j = 0usize;
while j < 8 {
let mut v = _mm_loadu_si128(coeff.as_ptr().add(base + j * 8).cast());
if IS_RECT2 {
v = _mm_srai_epi32::<8>(_mm_add_epi32(_mm_mullo_epi32(v, rect_mul), rect_rnd));
}
a0 = _mm_add_epi32(
a0,
_mm_mullo_epi32(v, _mm_set1_epi32(tx8_coeff(kind, m, j))),
);
a1 = _mm_add_epi32(
a1,
_mm_mullo_epi32(v, _mm_set1_epi32(tx8_coeff(kind, m + 1, j))),
);
a2 = _mm_add_epi32(
a2,
_mm_mullo_epi32(v, _mm_set1_epi32(tx8_coeff(kind, m + 2, j))),
);
a3 = _mm_add_epi32(
a3,
_mm_mullo_epi32(v, _mm_set1_epi32(tx8_coeff(kind, m + 3, j))),
);
j += 1;
}
[a0, a1, a2, a3]
}
}
#[inline]
fn tx8_coeff(kind: usize, out: usize, input: usize) -> i32 {
match kind {
crate::itx_2d::TX_KIND_DCT => crate::itx_2d::DCT8_KW[out * 8 + input] as i32,
crate::itx_2d::TX_KIND_ADST => crate::itx_2d::ADST8_KW[out * 8 + input] as i32,
crate::itx_2d::TX_KIND_FLIPADST => crate::itx_2d::ADST8_KW[(7 - out) * 8 + input] as i32,
_ => unreachable!(),
}
}
#[inline]
fn avx2_identity_coeff(n: usize, out: usize, input: usize) -> i32 {
if out != input {
0
} else {
match n {
4 => 128,
8 => 181,
16 => 256,
32 => 362,
_ => unreachable!(),
}
}
}
#[inline]
fn avx2_identity_scale(n: usize) -> i32 {
match n {
4 => 128,
8 => 181,
16 => 256,
32 => 362,
_ => unreachable!(),
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_identity_i16x4_coeff_to_i32<const IS_RECT2: bool>(
coeff: &[i16],
off: usize,
scale: __m128i,
) -> __m128i {
let v = avx2_load4_i16_coeff_packed_const::<IS_RECT2>(coeff, off);
_mm_mullo_epi32(_mm_cvtepi16_epi32(v), scale)
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_identity_i16x4_scratch_to_i32(scratch: &[i16], off: usize, scale: __m128i) -> __m128i {
let v = avx2_load4_i16_scratch(scratch, off);
_mm_mullo_epi32(_mm_cvtepi16_epi32(v), scale)
}
#[inline]
fn avx2_tx_dense_coeff(kind: usize, n: usize, out: usize, input: usize) -> i32 {
match (kind, n) {
(crate::itx_2d::TX_KIND_DCT, 4) => crate::itx_2d::DCT4_KW[out * 8 + input] as i32,
(crate::itx_2d::TX_KIND_DCT, 8) => crate::itx_2d::DCT8_KW[out * 8 + input] as i32,
(crate::itx_2d::TX_KIND_DCT, 16) => crate::itx_2d::DCT16_DENSE_KERNEL[input * 16 + out],
(crate::itx_2d::TX_KIND_DCT, 32) => crate::itx_2d::DCT32_DENSE_KERNEL[input * 32 + out],
(crate::itx_2d::TX_KIND_IDENTITY, 4)
| (crate::itx_2d::TX_KIND_IDENTITY, 8)
| (crate::itx_2d::TX_KIND_IDENTITY, 16)
| (crate::itx_2d::TX_KIND_IDENTITY, 32) => avx2_identity_coeff(n, out, input),
(crate::itx_2d::TX_KIND_ADST, 4) => crate::itx_1d::ADST4_KERNEL_ROWS[out][input] as i32,
(crate::itx_2d::TX_KIND_ADST, 8) => crate::itx_1d::ADST8_KERNEL_ROWS[out][input] as i32,
(crate::itx_2d::TX_KIND_ADST, 16) => crate::itx_1d::ADST16_KERNEL_ROWS[out][input] as i32,
(crate::itx_2d::TX_KIND_FLIPADST, 4) => {
crate::itx_1d::FLIPADST4_KERNEL_ROWS[out][input] as i32
}
(crate::itx_2d::TX_KIND_FLIPADST, 8) => {
crate::itx_1d::ADST8_KERNEL_ROWS[7 - out][input] as i32
}
(crate::itx_2d::TX_KIND_FLIPADST, 16) => {
crate::itx_1d::FLIPADST16_KERNEL_ROWS[out][input] as i32
}
_ => unreachable!(),
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_tx_dense_coeff_pair(kind: usize, n: usize, out: usize, input: usize) -> __m128i {
debug_assert_eq!(input & 1, 0);
let (table, idx): (&[i32], usize) = match (kind, n) {
(crate::itx_2d::TX_KIND_DCT, 4) => (&crate::itx_2d::DCT4_KP_X4, out * 4 + (input >> 1)),
(crate::itx_2d::TX_KIND_DCT, 8) => (&crate::itx_2d::DCT8_KP_X4, out * 4 + (input >> 1)),
(crate::itx_2d::TX_KIND_DCT, 16) => {
(&crate::itx_2d::DCT16_DENSE_PAIR_X4, out * 8 + (input >> 1))
}
(crate::itx_2d::TX_KIND_DCT, 32) => {
(&crate::itx_2d::DCT32_DENSE_PAIR_X4, out * 16 + (input >> 1))
}
(crate::itx_2d::TX_KIND_IDENTITY, 4)
| (crate::itx_2d::TX_KIND_IDENTITY, 8)
| (crate::itx_2d::TX_KIND_IDENTITY, 16)
| (crate::itx_2d::TX_KIND_IDENTITY, 32) => {
let k0 = avx2_identity_coeff(n, out, input) as i16;
let k1 = avx2_identity_coeff(n, out, input + 1) as i16;
return avx2_coeff_pair_from_scalars_i16(k0, k1);
}
(crate::itx_2d::TX_KIND_ADST, 4) => (&crate::itx_2d::ADST4_KP_X4, out * 4 + (input >> 1)),
(crate::itx_2d::TX_KIND_ADST, 8) => (&crate::itx_2d::ADST8_KP_X4, out * 4 + (input >> 1)),
(crate::itx_2d::TX_KIND_ADST, 16) => (&crate::itx_2d::ADST16_KP_X4, out * 8 + (input >> 1)),
(crate::itx_2d::TX_KIND_FLIPADST, 4) => {
(&crate::itx_2d::FLIPADST4_KP_X4, out * 4 + (input >> 1))
}
(crate::itx_2d::TX_KIND_FLIPADST, 8) => {
(&crate::itx_2d::ADST8_KP_X4, (7 - out) * 4 + (input >> 1))
}
(crate::itx_2d::TX_KIND_FLIPADST, 16) => {
(&crate::itx_2d::FLIPADST16_KP_X4, out * 8 + (input >> 1))
}
_ => unreachable!(),
};
avx2_coeff_pair_i16(table, idx)
}
#[inline(always)]
fn avx2_tx16_dense_pair_table<const KIND: usize>() -> &'static [i32] {
match KIND {
crate::itx_2d::TX_KIND_DCT => &crate::itx_2d::DCT16_DENSE_PAIR_X4,
crate::itx_2d::TX_KIND_ADST => &crate::itx_2d::ADST16_KP_X4,
crate::itx_2d::TX_KIND_FLIPADST => &crate::itx_2d::FLIPADST16_KP_X4,
_ => unreachable!(),
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_tx16_dense_pair_i16x8<const KIND: usize>(out: usize, input: usize) -> __m256i {
debug_assert_eq!(input & 1, 0);
avx2_coeff_pair_i16x8(avx2_tx16_dense_pair_table::<KIND>(), out * 8 + (input >> 1))
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_tx16_dense_i16x8_store_quads_from_coeff8_stride_const<
const IS_RECT2: bool,
const COEFF_STRIDE: usize,
const KIND: usize,
>(
coeff: &[i16],
scratch: &mut [i16],
y: usize,
rnd: __m128i,
sh: __m128i,
minv: __m128i,
maxv: __m128i,
) {
debug_assert!(KIND == crate::itx_2d::TX_KIND_ADST || KIND == crate::itx_2d::TX_KIND_FLIPADST);
let z = _mm256_setzero_si256();
let mut out = 0usize;
while out < 16 {
let mut a0 = z;
let mut a1 = z;
let mut a2 = z;
let mut a3 = z;
let mut j = 0usize;
while j < 16 {
let x0 = avx2_load8_i16_coeff_packed_const::<IS_RECT2>(coeff, y + j * COEFF_STRIDE);
let x1 =
avx2_load8_i16_coeff_packed_const::<IS_RECT2>(coeff, y + (j + 1) * COEFF_STRIDE);
let x01 = avx2_pair8_i16_from_rows(x0, x1);
a0 = _mm256_add_epi32(
a0,
_mm256_madd_epi16(x01, avx2_tx16_dense_pair_i16x8::<KIND>(out, j)),
);
a1 = _mm256_add_epi32(
a1,
_mm256_madd_epi16(x01, avx2_tx16_dense_pair_i16x8::<KIND>(out + 1, j)),
);
a2 = _mm256_add_epi32(
a2,
_mm256_madd_epi16(x01, avx2_tx16_dense_pair_i16x8::<KIND>(out + 2, j)),
);
a3 = _mm256_add_epi32(
a3,
_mm256_madd_epi16(x01, avx2_tx16_dense_pair_i16x8::<KIND>(out + 3, j)),
);
j += 2;
}
avx2_store4x8_i16_clip256::<16>(scratch, y * 16 + out, a0, a1, a2, a3, rnd, sh, minv, maxv);
out += 4;
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_tx16_dense_i16x16_store_quads_from_coeff16_stride_const<
const IS_RECT2: bool,
const COEFF_STRIDE: usize,
const KIND: usize,
>(
coeff: &[i16],
scratch: &mut [i16],
y: usize,
rnd: __m128i,
sh: __m128i,
minv: __m128i,
maxv: __m128i,
) {
debug_assert!(KIND == crate::itx_2d::TX_KIND_ADST || KIND == crate::itx_2d::TX_KIND_FLIPADST);
let mut out = 0usize;
while out < 16 {
let mut a0 = avx2_i32x16_zero();
let mut a1 = avx2_i32x16_zero();
let mut a2 = avx2_i32x16_zero();
let mut a3 = avx2_i32x16_zero();
let mut j = 0usize;
while j < 16 {
let x0 = avx2_load16_i16_coeff_packed_const::<IS_RECT2>(coeff, y + j * COEFF_STRIDE);
let x1 =
avx2_load16_i16_coeff_packed_const::<IS_RECT2>(coeff, y + (j + 1) * COEFF_STRIDE);
let k0 = avx2_tx16_dense_pair_i16x8::<KIND>(out, j);
let k1 = avx2_tx16_dense_pair_i16x8::<KIND>(out + 1, j);
let k2 = avx2_tx16_dense_pair_i16x8::<KIND>(out + 2, j);
let k3 = avx2_tx16_dense_pair_i16x8::<KIND>(out + 3, j);
a0 = avx2_i32x16_add(a0, avx2_madd_i16x16_pair(x0, x1, k0));
a1 = avx2_i32x16_add(a1, avx2_madd_i16x16_pair(x0, x1, k1));
a2 = avx2_i32x16_add(a2, avx2_madd_i16x16_pair(x0, x1, k2));
a3 = avx2_i32x16_add(a3, avx2_madd_i16x16_pair(x0, x1, k3));
j += 2;
}
avx2_store4x16_i16_clip256::<16>(
scratch,
y * 16 + out,
a0,
a1,
a2,
a3,
rnd,
sh,
minv,
maxv,
);
out += 4;
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_tx16_dense_i16_coeff_rows_to_scratch<
const IS_RECT2: bool,
const COEFF_STRIDE: usize,
const KIND: usize,
>(
coeff: &[i16],
scratch: &mut [i16],
mut y: usize,
nrows: usize,
rnd: __m128i,
sh: __m128i,
minv: __m128i,
maxv: __m128i,
) -> usize {
debug_assert!(KIND == crate::itx_2d::TX_KIND_ADST || KIND == crate::itx_2d::TX_KIND_FLIPADST);
while y + 16 <= nrows {
avx2_tx16_dense_i16x16_store_quads_from_coeff16_stride_const::<IS_RECT2, COEFF_STRIDE, KIND>(
coeff, scratch, y, rnd, sh, minv, maxv,
);
y += 16;
}
while y + 8 <= nrows {
avx2_tx16_dense_i16x8_store_quads_from_coeff8_stride_const::<IS_RECT2, COEFF_STRIDE, KIND>(
coeff, scratch, y, rnd, sh, minv, maxv,
);
y += 8;
}
y
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_tx16_dense_i16x8_scratch_store_quads<const STRIDE: usize, const KIND: usize>(
scratch: &[i16],
base: usize,
tmp: &mut [i32; ITX_TMP_PIXELS],
) {
debug_assert!(KIND == crate::itx_2d::TX_KIND_ADST || KIND == crate::itx_2d::TX_KIND_FLIPADST);
let z = _mm256_setzero_si256();
let mut out = 0usize;
while out < 16 {
let mut a0 = z;
let mut a1 = z;
let mut a2 = z;
let mut a3 = z;
let mut j = 0usize;
while j < 16 {
let x0 = avx2_load8_i16_scratch(scratch, base + j * STRIDE);
let x1 = avx2_load8_i16_scratch(scratch, base + (j + 1) * STRIDE);
let x01 = avx2_pair8_i16_from_rows(x0, x1);
a0 = _mm256_add_epi32(
a0,
_mm256_madd_epi16(x01, avx2_tx16_dense_pair_i16x8::<KIND>(out, j)),
);
a1 = _mm256_add_epi32(
a1,
_mm256_madd_epi16(x01, avx2_tx16_dense_pair_i16x8::<KIND>(out + 1, j)),
);
a2 = _mm256_add_epi32(
a2,
_mm256_madd_epi16(x01, avx2_tx16_dense_pair_i16x8::<KIND>(out + 2, j)),
);
a3 = _mm256_add_epi32(
a3,
_mm256_madd_epi16(x01, avx2_tx16_dense_pair_i16x8::<KIND>(out + 3, j)),
);
j += 2;
}
unsafe {
_mm256_storeu_si256(tmp.as_mut_ptr().add(base + out * 32).cast(), a0);
_mm256_storeu_si256(tmp.as_mut_ptr().add(base + (out + 1) * 32).cast(), a1);
_mm256_storeu_si256(tmp.as_mut_ptr().add(base + (out + 2) * 32).cast(), a2);
_mm256_storeu_si256(tmp.as_mut_ptr().add(base + (out + 3) * 32).cast(), a3);
}
out += 4;
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_tx16_dense_i16x8_scratch_add_u8_quads<const STRIDE: usize, const KIND: usize>(
scratch: &[i16],
base: usize,
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
rnd: __m256i,
sh: __m128i,
) {
debug_assert!(KIND == crate::itx_2d::TX_KIND_ADST || KIND == crate::itx_2d::TX_KIND_FLIPADST);
let z = _mm256_setzero_si256();
let mut out = 0usize;
while out < 16 {
let mut a0 = z;
let mut a1 = z;
let mut a2 = z;
let mut a3 = z;
let mut j = 0usize;
while j < 16 {
let x0 = avx2_load8_i16_scratch(scratch, base + j * STRIDE);
let x1 = avx2_load8_i16_scratch(scratch, base + (j + 1) * STRIDE);
let x01 = avx2_pair8_i16_from_rows(x0, x1);
a0 = _mm256_add_epi32(
a0,
_mm256_madd_epi16(x01, avx2_tx16_dense_pair_i16x8::<KIND>(out, j)),
);
a1 = _mm256_add_epi32(
a1,
_mm256_madd_epi16(x01, avx2_tx16_dense_pair_i16x8::<KIND>(out + 1, j)),
);
a2 = _mm256_add_epi32(
a2,
_mm256_madd_epi16(x01, avx2_tx16_dense_pair_i16x8::<KIND>(out + 2, j)),
);
a3 = _mm256_add_epi32(
a3,
_mm256_madd_epi16(x01, avx2_tx16_dense_pair_i16x8::<KIND>(out + 3, j)),
);
j += 2;
}
avx2_writeback8_i32_u8::<STRIDE, 16>(
dst, dst_off, dst_stride, out_w, out_h, base, out, a0, rnd, sh,
);
avx2_writeback8_i32_u8::<STRIDE, 16>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
out + 1,
a1,
rnd,
sh,
);
avx2_writeback8_i32_u8::<STRIDE, 16>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
out + 2,
a2,
rnd,
sh,
);
avx2_writeback8_i32_u8::<STRIDE, 16>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
out + 3,
a3,
rnd,
sh,
);
out += 4;
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_tx16_dense_i16x16_scratch_store_quads<const STRIDE: usize, const KIND: usize>(
scratch: &[i16],
base: usize,
tmp: &mut [i32; ITX_TMP_PIXELS],
) {
debug_assert!(KIND == crate::itx_2d::TX_KIND_ADST || KIND == crate::itx_2d::TX_KIND_FLIPADST);
let mut out = 0usize;
while out < 16 {
let mut a0 = avx2_i32x16_zero();
let mut a1 = avx2_i32x16_zero();
let mut a2 = avx2_i32x16_zero();
let mut a3 = avx2_i32x16_zero();
let mut j = 0usize;
while j < 16 {
let x0 = avx2_load16_i16_scratch(scratch, base + j * STRIDE);
let x1 = avx2_load16_i16_scratch(scratch, base + (j + 1) * STRIDE);
a0 = avx2_i32x16_add(
a0,
avx2_madd_i16x16_pair(x0, x1, avx2_tx16_dense_pair_i16x8::<KIND>(out, j)),
);
a1 = avx2_i32x16_add(
a1,
avx2_madd_i16x16_pair(x0, x1, avx2_tx16_dense_pair_i16x8::<KIND>(out + 1, j)),
);
a2 = avx2_i32x16_add(
a2,
avx2_madd_i16x16_pair(x0, x1, avx2_tx16_dense_pair_i16x8::<KIND>(out + 2, j)),
);
a3 = avx2_i32x16_add(
a3,
avx2_madd_i16x16_pair(x0, x1, avx2_tx16_dense_pair_i16x8::<KIND>(out + 3, j)),
);
j += 2;
}
avx2_store_i32x16_row(tmp, base + out * 32, a0);
avx2_store_i32x16_row(tmp, base + (out + 1) * 32, a1);
avx2_store_i32x16_row(tmp, base + (out + 2) * 32, a2);
avx2_store_i32x16_row(tmp, base + (out + 3) * 32, a3);
out += 4;
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_tx16_dense_i16x16_scratch_add_u8_quads<const STRIDE: usize, const KIND: usize>(
scratch: &[i16],
base: usize,
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
rnd: __m256i,
sh: __m128i,
) {
debug_assert!(KIND == crate::itx_2d::TX_KIND_ADST || KIND == crate::itx_2d::TX_KIND_FLIPADST);
let mut out = 0usize;
while out < 16 {
let mut a0 = avx2_i32x16_zero();
let mut a1 = avx2_i32x16_zero();
let mut a2 = avx2_i32x16_zero();
let mut a3 = avx2_i32x16_zero();
let mut j = 0usize;
while j < 16 {
let x0 = avx2_load16_i16_scratch(scratch, base + j * STRIDE);
let x1 = avx2_load16_i16_scratch(scratch, base + (j + 1) * STRIDE);
a0 = avx2_i32x16_add(
a0,
avx2_madd_i16x16_pair(x0, x1, avx2_tx16_dense_pair_i16x8::<KIND>(out, j)),
);
a1 = avx2_i32x16_add(
a1,
avx2_madd_i16x16_pair(x0, x1, avx2_tx16_dense_pair_i16x8::<KIND>(out + 1, j)),
);
a2 = avx2_i32x16_add(
a2,
avx2_madd_i16x16_pair(x0, x1, avx2_tx16_dense_pair_i16x8::<KIND>(out + 2, j)),
);
a3 = avx2_i32x16_add(
a3,
avx2_madd_i16x16_pair(x0, x1, avx2_tx16_dense_pair_i16x8::<KIND>(out + 3, j)),
);
j += 2;
}
avx2_writeback16_i32_u8::<STRIDE, 16>(
dst, dst_off, dst_stride, out_w, out_h, base, out, a0, rnd, sh,
);
avx2_writeback16_i32_u8::<STRIDE, 16>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
out + 1,
a1,
rnd,
sh,
);
avx2_writeback16_i32_u8::<STRIDE, 16>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
out + 2,
a2,
rnd,
sh,
);
avx2_writeback16_i32_u8::<STRIDE, 16>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
out + 3,
a3,
rnd,
sh,
);
out += 4;
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_store4x8_i16_clip256<const STRIDE: usize>(
scratch: &mut [i16],
off: usize,
v0: __m256i,
v1: __m256i,
v2: __m256i,
v3: __m256i,
rnd: __m128i,
sh: __m128i,
minv: __m128i,
maxv: __m128i,
) {
unsafe {
debug_assert!(STRIDE == 16 || STRIDE == 32);
debug_assert!(off + 7 * STRIDE + 4 <= scratch.len());
let rnd256 = _mm256_broadcastsi128_si256(rnd);
let min256 = _mm256_broadcastsi128_si256(minv);
let max256 = _mm256_broadcastsi128_si256(maxv);
macro_rules! clip_pack8 {
($x:expr) => {{
let c = _mm256_min_epi32(
_mm256_max_epi32(_mm256_sra_epi32(_mm256_add_epi32($x, rnd256), sh), min256),
max256,
);
_mm_packs_epi32(_mm256_castsi256_si128(c), _mm256_extracti128_si256::<1>(c))
}};
}
let r0 = clip_pack8!(v0);
let r1 = clip_pack8!(v1);
let r2 = clip_pack8!(v2);
let r3 = clip_pack8!(v3);
macro_rules! transpose4x8_store {
($base:expr, $a0:expr, $a1:expr, $a2:expr, $a3:expr) => {{
let t0 = _mm_unpacklo_epi16($a0, $a1);
let t1 = _mm_unpacklo_epi16($a2, $a3);
let t2 = _mm_unpackhi_epi16($a0, $a1);
let t3 = _mm_unpackhi_epi16($a2, $a3);
let r01 = _mm_unpacklo_epi32(t0, t1);
let r23 = _mm_unpackhi_epi32(t0, t1);
let r45 = _mm_unpacklo_epi32(t2, t3);
let r67 = _mm_unpackhi_epi32(t2, t3);
let ptr = scratch.as_mut_ptr();
_mm_storel_epi64(ptr.add(off + ($base) * STRIDE).cast(), r01);
_mm_storel_epi64(
ptr.add(off + (($base) + 1) * STRIDE).cast(),
_mm_srli_si128::<8>(r01),
);
_mm_storel_epi64(ptr.add(off + (($base) + 2) * STRIDE).cast(), r23);
_mm_storel_epi64(
ptr.add(off + (($base) + 3) * STRIDE).cast(),
_mm_srli_si128::<8>(r23),
);
_mm_storel_epi64(ptr.add(off + (($base) + 4) * STRIDE).cast(), r45);
_mm_storel_epi64(
ptr.add(off + (($base) + 5) * STRIDE).cast(),
_mm_srli_si128::<8>(r45),
);
_mm_storel_epi64(ptr.add(off + (($base) + 6) * STRIDE).cast(), r67);
_mm_storel_epi64(
ptr.add(off + (($base) + 7) * STRIDE).cast(),
_mm_srli_si128::<8>(r67),
);
}};
}
transpose4x8_store!(0usize, r0, r1, r2, r3);
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_store4x16_i16_clip256<const STRIDE: usize>(
scratch: &mut [i16],
off: usize,
v0: Avx2I32x16,
v1: Avx2I32x16,
v2: Avx2I32x16,
v3: Avx2I32x16,
rnd: __m128i,
sh: __m128i,
minv: __m128i,
maxv: __m128i,
) {
unsafe {
debug_assert!(STRIDE == 16 || STRIDE == 32);
debug_assert!(off + 15 * STRIDE + 4 <= scratch.len());
let rnd256 = _mm256_broadcastsi128_si256(rnd);
let min256 = _mm256_broadcastsi128_si256(minv);
let max256 = _mm256_broadcastsi128_si256(maxv);
macro_rules! clip_pack16 {
($x:expr) => {{
let x = $x;
let l = _mm256_min_epi32(
_mm256_max_epi32(_mm256_sra_epi32(_mm256_add_epi32(x.lo, rnd256), sh), min256),
max256,
);
let h = _mm256_min_epi32(
_mm256_max_epi32(_mm256_sra_epi32(_mm256_add_epi32(x.hi, rnd256), sh), min256),
max256,
);
let lo = _mm256_permute2x128_si256::<0x20>(l, h);
let hi = _mm256_permute2x128_si256::<0x31>(l, h);
_mm256_permute4x64_epi64::<0xd8>(_mm256_packs_epi32(lo, hi))
}};
}
let r0 = clip_pack16!(v0);
let r1 = clip_pack16!(v1);
let r2 = clip_pack16!(v2);
let r3 = clip_pack16!(v3);
let r0_lo = _mm256_castsi256_si128(r0);
let r1_lo = _mm256_castsi256_si128(r1);
let r2_lo = _mm256_castsi256_si128(r2);
let r3_lo = _mm256_castsi256_si128(r3);
let r0_hi = _mm256_extracti128_si256::<1>(r0);
let r1_hi = _mm256_extracti128_si256::<1>(r1);
let r2_hi = _mm256_extracti128_si256::<1>(r2);
let r3_hi = _mm256_extracti128_si256::<1>(r3);
macro_rules! transpose4x8_store {
($base:expr, $a0:expr, $a1:expr, $a2:expr, $a3:expr) => {{
let t0 = _mm_unpacklo_epi16($a0, $a1);
let t1 = _mm_unpacklo_epi16($a2, $a3);
let t2 = _mm_unpackhi_epi16($a0, $a1);
let t3 = _mm_unpackhi_epi16($a2, $a3);
let r01 = _mm_unpacklo_epi32(t0, t1);
let r23 = _mm_unpackhi_epi32(t0, t1);
let r45 = _mm_unpacklo_epi32(t2, t3);
let r67 = _mm_unpackhi_epi32(t2, t3);
let ptr = scratch.as_mut_ptr();
_mm_storel_epi64(ptr.add(off + ($base) * STRIDE).cast(), r01);
_mm_storel_epi64(
ptr.add(off + (($base) + 1) * STRIDE).cast(),
_mm_srli_si128::<8>(r01),
);
_mm_storel_epi64(ptr.add(off + (($base) + 2) * STRIDE).cast(), r23);
_mm_storel_epi64(
ptr.add(off + (($base) + 3) * STRIDE).cast(),
_mm_srli_si128::<8>(r23),
);
_mm_storel_epi64(ptr.add(off + (($base) + 4) * STRIDE).cast(), r45);
_mm_storel_epi64(
ptr.add(off + (($base) + 5) * STRIDE).cast(),
_mm_srli_si128::<8>(r45),
);
_mm_storel_epi64(ptr.add(off + (($base) + 6) * STRIDE).cast(), r67);
_mm_storel_epi64(
ptr.add(off + (($base) + 7) * STRIDE).cast(),
_mm_srli_si128::<8>(r67),
);
}};
}
transpose4x8_store!(0usize, r0_lo, r1_lo, r2_lo, r3_lo);
transpose4x8_store!(8usize, r0_hi, r1_hi, r2_hi, r3_hi);
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_load4_i16_coeff_packed_const<const IS_RECT2: bool>(src: &[i16], off: usize) -> __m128i {
debug_assert!(off + 4 <= src.len());
let mut v = unsafe { _mm_loadl_epi64(src.as_ptr().add(off).cast()) };
if IS_RECT2 {
v = _mm_mulhrs_epi16(v, _mm_set1_epi16(0x5a80));
}
v
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_load8_i16_coeff_packed_const<const IS_RECT2: bool>(src: &[i16], off: usize) -> __m128i {
debug_assert!(off + 8 <= src.len());
let mut v = unsafe { _mm_loadu_si128(src.as_ptr().add(off).cast()) };
if IS_RECT2 {
v = _mm_mulhrs_epi16(v, _mm_set1_epi16(0x5a80));
}
v
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_load16_i16_coeff_packed_const<const IS_RECT2: bool>(src: &[i16], off: usize) -> __m256i {
debug_assert!(off + 16 <= src.len());
let lo = unsafe { _mm_loadu_si128(src.as_ptr().add(off).cast()) };
let hi = unsafe { _mm_loadu_si128(src.as_ptr().add(off + 8).cast()) };
let mut v = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(lo), hi);
if IS_RECT2 {
v = _mm256_mulhrs_epi16(v, _mm256_set1_epi16(0x5a80));
}
v
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_pair8_i16_from_rows(a: __m128i, b: __m128i) -> __m256i {
let lo = _mm_unpacklo_epi16(a, b);
let hi = _mm_unpackhi_epi16(a, b);
_mm256_inserti128_si256::<1>(_mm256_castsi128_si256(lo), hi)
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_coeff_pair_from_scalars_i16(k0: i16, k1: i16) -> __m128i {
_mm_set1_epi32(((k1 as u16 as i32) << 16) | (k0 as u16 as i32))
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_coeff_pair_i16(table: &[i32], idx: usize) -> __m128i {
debug_assert!(idx * 4 + 4 <= table.len());
unsafe { _mm_loadu_si128(table.as_ptr().add(idx * 4).cast()) }
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_coeff_pair_i16x8(table: &[i32], idx: usize) -> __m256i {
_mm256_broadcastsi128_si256(avx2_coeff_pair_i16(table, idx))
}
#[derive(Clone, Copy)]
struct Avx2I32x16 {
lo: __m256i,
hi: __m256i,
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_i32x16_zero() -> Avx2I32x16 {
let z = _mm256_setzero_si256();
Avx2I32x16 { lo: z, hi: z }
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_i32x16_add(a: Avx2I32x16, b: Avx2I32x16) -> Avx2I32x16 {
Avx2I32x16 {
lo: _mm256_add_epi32(a.lo, b.lo),
hi: _mm256_add_epi32(a.hi, b.hi),
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_i32x16_sub(a: Avx2I32x16, b: Avx2I32x16) -> Avx2I32x16 {
Avx2I32x16 {
lo: _mm256_sub_epi32(a.lo, b.lo),
hi: _mm256_sub_epi32(a.hi, b.hi),
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_madd_i16x16_pair(a: __m256i, b: __m256i, k: __m256i) -> Avx2I32x16 {
Avx2I32x16 {
lo: _mm256_madd_epi16(_mm256_unpacklo_epi16(a, b), k),
hi: _mm256_madd_epi16(_mm256_unpackhi_epi16(a, b), k),
}
}
macro_rules! avx2_dct16_i16x4_all_body {
() => {{
let z = _mm_setzero_si128();
let mut b = [z; 8];
let mut m = 0usize;
while m < 8 {
let base = m * 8;
let mut acc = z;
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(1), load!(3)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KBP_X4, base >> 1),
),
);
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(5), load!(7)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KBP_X4, (base >> 1) + 1),
),
);
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(9), load!(11)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KBP_X4, (base >> 1) + 2),
),
);
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(13), load!(15)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KBP_X4, (base >> 1) + 3),
),
);
b[m] = acc;
m += 1;
}
let mut d = [z; 4];
m = 0;
while m < 4 {
let base = m * 8;
let mut acc = z;
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(2), load!(6)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KDP_X4, base >> 1),
),
);
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(10), load!(14)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KDP_X4, (base >> 1) + 1),
),
);
d[m] = acc;
m += 1;
}
let f0 = _mm_add_epi32(
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(4), load!(12)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KFP_X4, 0),
),
z,
);
let f1 = _mm_add_epi32(
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(4), load!(12)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KFP_X4, 1),
),
z,
);
let g0 = _mm_madd_epi16(
_mm_unpacklo_epi16(load!(0), load!(8)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KGP_X4, 0),
);
let g1 = _mm_madd_epi16(
_mm_unpacklo_epi16(load!(0), load!(8)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KGP_X4, 1),
);
let cc = [
_mm_add_epi32(g0, f0),
_mm_add_epi32(g1, f1),
_mm_sub_epi32(g1, f1),
_mm_sub_epi32(g0, f0),
];
let mut a = [z; 8];
let mut i = 0usize;
while i < 4 {
a[i] = _mm_add_epi32(cc[i], d[i]);
i += 1;
}
while i < 8 {
a[i] = _mm_sub_epi32(cc[7 - i], d[7 - i]);
i += 1;
}
let mut out = [z; 16];
let mut k = 0usize;
while k < 8 {
out[k] = _mm_add_epi32(a[k], b[k]);
out[k + 8] = _mm_sub_epi32(a[7 - k], b[7 - k]);
k += 1;
}
out
}};
}
macro_rules! avx2_dct32_i16x4_all_body {
() => {{
let z = _mm_setzero_si128();
let mut b = [z; 16];
let mut m = 0usize;
while m < 16 {
let base = m * 16;
let mut acc = z;
let mut p = 0usize;
while p < 16 {
let cb = base + p;
let i0 = 2 * p + 1;
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(i0), load!(i0 + 2)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KBP_X4, cb >> 1),
),
);
p += 2;
}
b[m] = acc;
m += 1;
}
let mut d = [z; 8];
m = 0;
while m < 8 {
let base = m * 8;
let mut acc = z;
let mut p = 0usize;
while p < 8 {
let i0 = 4 * p + 2;
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(i0), load!(i0 + 4)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KDP_X4, (base + p) >> 1),
),
);
p += 2;
}
d[m] = acc;
m += 1;
}
let mut f = [z; 4];
m = 0;
while m < 4 {
let base = m * 8;
let mut acc = z;
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(4), load!(12)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KFP_X4, base >> 1),
),
);
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(20), load!(28)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KFP_X4, (base >> 1) + 1),
),
);
f[m] = acc;
m += 1;
}
let h0 = _mm_madd_epi16(
_mm_unpacklo_epi16(load!(8), load!(24)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KHP_X4, 0),
);
let h1 = _mm_madd_epi16(
_mm_unpacklo_epi16(load!(8), load!(24)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KHP_X4, 1),
);
let g0 = _mm_madd_epi16(
_mm_unpacklo_epi16(load!(0), load!(16)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KGP_X4, 0),
);
let g1 = _mm_madd_epi16(
_mm_unpacklo_epi16(load!(0), load!(16)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KGP_X4, 1),
);
let e = [
_mm_add_epi32(g0, h0),
_mm_add_epi32(g1, h1),
_mm_sub_epi32(g1, h1),
_mm_sub_epi32(g0, h0),
];
let mut cc = [z; 8];
let mut i = 0usize;
while i < 4 {
cc[i] = _mm_add_epi32(e[i], f[i]);
i += 1;
}
while i < 8 {
cc[i] = _mm_sub_epi32(e[7 - i], f[7 - i]);
i += 1;
}
let mut a = [z; 16];
i = 0;
while i < 8 {
a[i] = _mm_add_epi32(cc[i], d[i]);
i += 1;
}
while i < 16 {
a[i] = _mm_sub_epi32(cc[15 - i], d[15 - i]);
i += 1;
}
let mut out = [z; 32];
let mut k = 0usize;
while k < 16 {
out[k] = _mm_add_epi32(a[k], b[k]);
out[k + 16] = _mm_sub_epi32(a[15 - k], b[15 - k]);
k += 1;
}
out
}};
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_residual_add_u8x8(dst: *mut u8, v: __m256i, rnd: __m256i, sh: __m128i) {
unsafe {
let r = _mm256_sra_epi32(_mm256_add_epi32(v, rnd), sh);
let r16 = _mm_packs_epi32(_mm256_castsi256_si128(r), _mm256_extracti128_si256::<1>(r));
let p8 = _mm_loadl_epi64(dst.cast());
let p16 = _mm_cvtepu8_epi16(p8);
let sum = _mm_adds_epi16(p16, r16);
let out = _mm_packus_epi16(sum, _mm_setzero_si128());
_mm_storel_epi64(dst.cast(), out);
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_residual_add_u8x8_expand_x2(dst: *mut u8, v: __m256i, rnd: __m256i, sh: __m128i) {
unsafe {
let r = _mm256_sra_epi32(_mm256_add_epi32(v, rnd), sh);
let r16 = _mm_packs_epi32(_mm256_castsi256_si128(r), _mm256_extracti128_si256::<1>(r));
let rlo = _mm_unpacklo_epi16(r16, r16);
let rhi = _mm_unpackhi_epi16(r16, r16);
let p8 = _mm_loadu_si128(dst.cast());
let plo = _mm_cvtepu8_epi16(p8);
let phi = _mm_cvtepu8_epi16(_mm_srli_si128::<8>(p8));
let slo = _mm_adds_epi16(plo, rlo);
let shi = _mm_adds_epi16(phi, rhi);
let out = _mm_packus_epi16(slo, shi);
_mm_storeu_si128(dst.cast(), out);
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_residual_add_u8x4(dst: &mut [u8], off: usize, v: __m128i, rnd: __m128i, sh: __m128i) {
unsafe {
debug_assert!(off + 4 <= dst.len());
let r = _mm_sra_epi32(_mm_add_epi32(v, rnd), sh);
let r16 = _mm_packs_epi32(r, _mm_setzero_si128());
let p8 = _mm_castps_si128(_mm_load_ss(dst.as_ptr().add(off).cast()));
let p16 = _mm_unpacklo_epi8(p8, _mm_setzero_si128());
let sum = _mm_adds_epi16(p16, r16);
let out = _mm_packus_epi16(sum, _mm_setzero_si128());
_mm_store_ss(dst.as_mut_ptr().add(off).cast(), _mm_castsi128_ps(out));
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_residual_add_u8x4_expand_x2(
dst: &mut [u8],
off: usize,
v: __m128i,
rnd: __m128i,
sh: __m128i,
) {
unsafe {
debug_assert!(off + 8 <= dst.len());
let r = _mm_sra_epi32(_mm_add_epi32(v, rnd), sh);
let r16 = _mm_packs_epi32(r, _mm_setzero_si128());
let r16x2 = _mm_unpacklo_epi16(r16, r16);
let p = dst.as_mut_ptr().add(off);
let d8 = _mm_loadl_epi64(p.cast());
let d16 = _mm_unpacklo_epi8(d8, _mm_setzero_si128());
let sum = _mm_adds_epi16(d16, r16x2);
let out = _mm_packus_epi16(sum, _mm_setzero_si128());
_mm_storel_epi64(p.cast(), out);
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_writeback8_i32_u8<const W: usize, const H: usize>(
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
x: usize,
y: usize,
v: __m256i,
rnd: __m256i,
sh: __m128i,
) {
debug_assert!(x + 8 <= W);
debug_assert!(y < H);
unsafe {
if out_w > W {
let ox = x * 2;
let oy = if out_h > H { y * 2 } else { y };
let off0 = dst_off + oy * dst_stride + ox;
avx2_residual_add_u8x8_expand_x2(dst.as_mut_ptr().add(off0), v, rnd, sh);
if out_h > H {
avx2_residual_add_u8x8_expand_x2(
dst.as_mut_ptr().add(off0 + dst_stride),
v,
rnd,
sh,
);
}
} else {
let ox = x;
let oy = if out_h > H { y * 2 } else { y };
let off0 = dst_off + oy * dst_stride + ox;
avx2_residual_add_u8x8(dst.as_mut_ptr().add(off0), v, rnd, sh);
if out_h > H {
avx2_residual_add_u8x8(dst.as_mut_ptr().add(off0 + dst_stride), v, rnd, sh);
}
}
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_i32x16_contiguous(v: Avx2I32x16) -> (__m256i, __m256i) {
(
_mm256_permute2x128_si256::<0x20>(v.lo, v.hi),
_mm256_permute2x128_si256::<0x31>(v.lo, v.hi),
)
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_store_i32x16_row(tmp: &mut [i32; ITX_TMP_PIXELS], off: usize, v: Avx2I32x16) {
unsafe {
debug_assert!(off + 16 <= tmp.len());
let (lo, hi) = avx2_i32x16_contiguous(v);
let ptr = tmp.as_mut_ptr().add(off).cast();
_mm256_storeu_si256(ptr, lo);
_mm256_storeu_si256(ptr.add(1), hi);
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_residual_add_u8x16(dst: *mut u8, v: Avx2I32x16, rnd: __m256i, sh: __m128i) {
unsafe {
let (lo, hi) = avx2_i32x16_contiguous(v);
let rlo = _mm256_sra_epi32(_mm256_add_epi32(lo, rnd), sh);
let rhi = _mm256_sra_epi32(_mm256_add_epi32(hi, rnd), sh);
let r16 = _mm256_permute4x64_epi64::<0xd8>(_mm256_packs_epi32(rlo, rhi));
let p8 = _mm_loadu_si128(dst.cast());
let p16 = _mm256_cvtepu8_epi16(p8);
let sum = _mm256_adds_epi16(p16, r16);
let out = _mm_packus_epi16(
_mm256_castsi256_si128(sum),
_mm256_extracti128_si256::<1>(sum),
);
_mm_storeu_si128(dst.cast(), out);
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_residual_add_u8x16_expand_x2(dst: *mut u8, v: Avx2I32x16, rnd: __m256i, sh: __m128i) {
let (lo, hi) = avx2_i32x16_contiguous(v);
unsafe {
avx2_residual_add_u8x8_expand_x2(dst, lo, rnd, sh);
avx2_residual_add_u8x8_expand_x2(dst.add(16), hi, rnd, sh);
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_writeback16_i32_u8<const W: usize, const H: usize>(
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
x: usize,
y: usize,
v: Avx2I32x16,
rnd: __m256i,
sh: __m128i,
) {
debug_assert!(x + 16 <= W);
debug_assert!(y < H);
unsafe {
if out_w > W {
let ox = x * 2;
let oy = if out_h > H { y * 2 } else { y };
let off0 = dst_off + oy * dst_stride + ox;
avx2_residual_add_u8x16_expand_x2(dst.as_mut_ptr().add(off0), v, rnd, sh);
if out_h > H {
avx2_residual_add_u8x16_expand_x2(
dst.as_mut_ptr().add(off0 + dst_stride),
v,
rnd,
sh,
);
}
} else {
let ox = x;
let oy = if out_h > H { y * 2 } else { y };
let off0 = dst_off + oy * dst_stride + ox;
avx2_residual_add_u8x16(dst.as_mut_ptr().add(off0), v, rnd, sh);
if out_h > H {
avx2_residual_add_u8x16(dst.as_mut_ptr().add(off0 + dst_stride), v, rnd, sh);
}
}
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_writeback4_i32_u8<const W: usize, const H: usize>(
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
x: usize,
y: usize,
v: __m128i,
rnd: __m128i,
sh: __m128i,
) {
debug_assert!(x + 4 <= W);
debug_assert!(y < H);
if out_w > W {
let ox = x * 2;
let oy = if out_h > H { y * 2 } else { y };
let off0 = dst_off + oy * dst_stride + ox;
avx2_residual_add_u8x4_expand_x2(dst, off0, v, rnd, sh);
if out_h > H {
let off1 = off0 + dst_stride;
avx2_residual_add_u8x4_expand_x2(dst, off1, v, rnd, sh);
}
} else {
let ox = x;
let oy = if out_h > H { y * 2 } else { y };
let off0 = dst_off + oy * dst_stride + ox;
avx2_residual_add_u8x4(dst, off0, v, rnd, sh);
if out_h > H {
let off1 = off0 + dst_stride;
avx2_residual_add_u8x4(dst, off1, v, rnd, sh);
}
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct16_i16x16_scratch16_stride_active_store<const STRIDE: usize, const ACTIVE: usize>(
scratch: &[i16],
base: usize,
tmp: &mut [i32; ITX_TMP_PIXELS],
) {
debug_assert!(ACTIVE == 4 || ACTIVE == 8 || ACTIVE == 16);
debug_assert!(base + 16 <= STRIDE);
debug_assert!(base + (ACTIVE - 1) * STRIDE + 16 <= scratch.len());
let zero_lane = _mm256_setzero_si256();
macro_rules! load {
($idx:expr) => {
if ($idx) < ACTIVE {
avx2_load16_i16_scratch(scratch, base + ($idx) * STRIDE)
} else {
zero_lane
}
};
}
macro_rules! madd_pair {
($a:expr, $b:expr, $tbl:expr, $idx:expr) => {
avx2_madd_i16x16_pair(load!($a), load!($b), avx2_coeff_pair_i16x8($tbl, $idx))
};
}
macro_rules! b_at {
($m:expr) => {{
let kbase = ($m) * 8;
let mut acc = avx2_i32x16_zero();
acc = avx2_i32x16_add(
acc,
madd_pair!(1, 3, &crate::itx_2d::DCT16_KBP_X4, kbase >> 1),
);
acc = avx2_i32x16_add(
acc,
madd_pair!(5, 7, &crate::itx_2d::DCT16_KBP_X4, (kbase >> 1) + 1),
);
acc = avx2_i32x16_add(
acc,
madd_pair!(9, 11, &crate::itx_2d::DCT16_KBP_X4, (kbase >> 1) + 2),
);
avx2_i32x16_add(
acc,
madd_pair!(13, 15, &crate::itx_2d::DCT16_KBP_X4, (kbase >> 1) + 3),
)
}};
}
macro_rules! d_at {
($m:expr) => {{
let kbase = ($m) * 8;
avx2_i32x16_add(
madd_pair!(2, 6, &crate::itx_2d::DCT16_KDP_X4, kbase >> 1),
madd_pair!(10, 14, &crate::itx_2d::DCT16_KDP_X4, (kbase >> 1) + 1),
)
}};
}
let f0 = madd_pair!(4, 12, &crate::itx_2d::DCT16_KFP_X4, 0);
let f1 = madd_pair!(4, 12, &crate::itx_2d::DCT16_KFP_X4, 1);
let g0 = madd_pair!(0, 8, &crate::itx_2d::DCT16_KGP_X4, 0);
let g1 = madd_pair!(0, 8, &crate::itx_2d::DCT16_KGP_X4, 1);
macro_rules! cc_at {
($k:expr) => {{
match $k {
0 => avx2_i32x16_add(g0, f0),
1 => avx2_i32x16_add(g1, f1),
2 => avx2_i32x16_sub(g1, f1),
_ => avx2_i32x16_sub(g0, f0),
}
}};
}
macro_rules! a_at {
($k:expr) => {{
let k = $k;
if k < 4 {
avx2_i32x16_add(cc_at!(k), d_at!(k))
} else {
avx2_i32x16_sub(cc_at!(7 - k), d_at!(7 - k))
}
}};
}
macro_rules! store_group {
($k:expr) => {{
let k = $k;
let a0 = a_at!(k);
let b0 = b_at!(k);
avx2_store_i32x16_row(tmp, base + k * 32, avx2_i32x16_add(a0, b0));
let a1 = a_at!(k + 1);
let b1 = b_at!(k + 1);
avx2_store_i32x16_row(tmp, base + (k + 1) * 32, avx2_i32x16_add(a1, b1));
let a2 = a_at!(k + 2);
let b2 = b_at!(k + 2);
avx2_store_i32x16_row(tmp, base + (k + 2) * 32, avx2_i32x16_add(a2, b2));
let a3 = a_at!(k + 3);
let b3 = b_at!(k + 3);
avx2_store_i32x16_row(tmp, base + (k + 3) * 32, avx2_i32x16_add(a3, b3));
avx2_store_i32x16_row(tmp, base + (8 + 7 - (k + 3)) * 32, avx2_i32x16_sub(a3, b3));
avx2_store_i32x16_row(tmp, base + (8 + 7 - (k + 2)) * 32, avx2_i32x16_sub(a2, b2));
avx2_store_i32x16_row(tmp, base + (8 + 7 - (k + 1)) * 32, avx2_i32x16_sub(a1, b1));
avx2_store_i32x16_row(tmp, base + (8 + 7 - k) * 32, avx2_i32x16_sub(a0, b0));
}};
}
store_group!(0usize);
store_group!(4usize);
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct32_i16x16_scratch16_stride_active_store<const STRIDE: usize, const ACTIVE: usize>(
scratch: &[i16],
base: usize,
tmp: &mut [i32; ITX_TMP_PIXELS],
) {
debug_assert!(ACTIVE == 4 || ACTIVE == 8 || ACTIVE == 16 || ACTIVE == 32);
debug_assert!(base + 16 <= STRIDE);
debug_assert!(base + (ACTIVE - 1) * STRIDE + 16 <= scratch.len());
let zero_lane = _mm256_setzero_si256();
macro_rules! load {
($idx:expr) => {
if ($idx) < ACTIVE {
avx2_load16_i16_scratch(scratch, base + ($idx) * STRIDE)
} else {
zero_lane
}
};
}
macro_rules! madd_pair {
($a:expr, $b:expr, $tbl:expr, $idx:expr) => {
avx2_madd_i16x16_pair(load!($a), load!($b), avx2_coeff_pair_i16x8($tbl, $idx))
};
}
macro_rules! b_at {
($m:expr) => {{
let m = $m;
let kbase = m * 16;
let mut acc = avx2_i32x16_zero();
let mut pair = 0usize;
while pair < 8 {
let i0 = 4 * pair + 1;
acc = avx2_i32x16_add(
acc,
madd_pair!(
i0,
i0 + 2,
&crate::itx_2d::DCT32_KBP_X4,
(kbase >> 1) + pair
),
);
pair += 1;
}
acc
}};
}
macro_rules! d_at {
($m:expr) => {{
let m = $m;
let kbase = m * 8;
let mut acc = avx2_i32x16_zero();
let mut pair = 0usize;
while pair < 4 {
let i0 = 8 * pair + 2;
acc = avx2_i32x16_add(
acc,
madd_pair!(
i0,
i0 + 4,
&crate::itx_2d::DCT32_KDP_X4,
(kbase >> 1) + pair
),
);
pair += 1;
}
acc
}};
}
macro_rules! f_at {
($m:expr) => {{
let kbase = ($m) * 8;
avx2_i32x16_add(
madd_pair!(4, 12, &crate::itx_2d::DCT32_KFP_X4, kbase >> 1),
madd_pair!(20, 28, &crate::itx_2d::DCT32_KFP_X4, (kbase >> 1) + 1),
)
}};
}
let h0 = madd_pair!(8, 24, &crate::itx_2d::DCT32_KHP_X4, 0);
let h1 = madd_pair!(8, 24, &crate::itx_2d::DCT32_KHP_X4, 1);
let g0 = madd_pair!(0, 16, &crate::itx_2d::DCT32_KGP_X4, 0);
let g1 = madd_pair!(0, 16, &crate::itx_2d::DCT32_KGP_X4, 1);
macro_rules! e_at {
($k:expr) => {{
match $k {
0 => avx2_i32x16_add(g0, h0),
1 => avx2_i32x16_add(g1, h1),
2 => avx2_i32x16_sub(g1, h1),
_ => avx2_i32x16_sub(g0, h0),
}
}};
}
macro_rules! cc_at {
($k:expr) => {{
let k = $k;
if k < 4 {
avx2_i32x16_add(e_at!(k), f_at!(k))
} else {
avx2_i32x16_sub(e_at!(7 - k), f_at!(7 - k))
}
}};
}
macro_rules! a_at {
($k:expr) => {{
let k = $k;
if k < 8 {
avx2_i32x16_add(cc_at!(k), d_at!(k))
} else {
avx2_i32x16_sub(cc_at!(15 - k), d_at!(15 - k))
}
}};
}
macro_rules! store_group {
($k:expr) => {{
let k = $k;
let a0 = a_at!(k);
let b0 = b_at!(k);
let l0 = avx2_i32x16_add(a0, b0);
let h0 = avx2_i32x16_sub(a0, b0);
let a1 = a_at!(k + 1);
let b1 = b_at!(k + 1);
let l1 = avx2_i32x16_add(a1, b1);
let h1 = avx2_i32x16_sub(a1, b1);
let a2 = a_at!(k + 2);
let b2 = b_at!(k + 2);
let l2 = avx2_i32x16_add(a2, b2);
let h2 = avx2_i32x16_sub(a2, b2);
let a3 = a_at!(k + 3);
let b3 = b_at!(k + 3);
let l3 = avx2_i32x16_add(a3, b3);
let h3 = avx2_i32x16_sub(a3, b3);
avx2_store_i32x16_row(tmp, base + k * 32, l0);
avx2_store_i32x16_row(tmp, base + (k + 1) * 32, l1);
avx2_store_i32x16_row(tmp, base + (k + 2) * 32, l2);
avx2_store_i32x16_row(tmp, base + (k + 3) * 32, l3);
avx2_store_i32x16_row(tmp, base + (16 + 15 - (k + 3)) * 32, h3);
avx2_store_i32x16_row(tmp, base + (16 + 15 - (k + 2)) * 32, h2);
avx2_store_i32x16_row(tmp, base + (16 + 15 - (k + 1)) * 32, h1);
avx2_store_i32x16_row(tmp, base + (16 + 15 - k) * 32, h0);
}};
}
store_group!(0usize);
store_group!(4usize);
store_group!(8usize);
store_group!(12usize);
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct16_i16x16_scratch16_stride_active_add_u8<const STRIDE: usize, const ACTIVE: usize>(
scratch: &[i16],
base: usize,
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
rnd1: __m256i,
sh1: __m128i,
) {
debug_assert!(ACTIVE == 4 || ACTIVE == 8 || ACTIVE == 16);
debug_assert!(base + 16 <= STRIDE);
debug_assert!(base + (ACTIVE - 1) * STRIDE + 16 <= scratch.len());
let zero_lane = _mm256_setzero_si256();
macro_rules! load {
($idx:expr) => {
if ($idx) < ACTIVE {
avx2_load16_i16_scratch(scratch, base + ($idx) * STRIDE)
} else {
zero_lane
}
};
}
macro_rules! madd_pair {
($a:expr, $b:expr, $tbl:expr, $idx:expr) => {
avx2_madd_i16x16_pair(load!($a), load!($b), avx2_coeff_pair_i16x8($tbl, $idx))
};
}
macro_rules! b_at {
($m:expr) => {{
let kbase = ($m) * 8;
let mut acc = avx2_i32x16_zero();
acc = avx2_i32x16_add(
acc,
madd_pair!(1, 3, &crate::itx_2d::DCT16_KBP_X4, kbase >> 1),
);
acc = avx2_i32x16_add(
acc,
madd_pair!(5, 7, &crate::itx_2d::DCT16_KBP_X4, (kbase >> 1) + 1),
);
acc = avx2_i32x16_add(
acc,
madd_pair!(9, 11, &crate::itx_2d::DCT16_KBP_X4, (kbase >> 1) + 2),
);
avx2_i32x16_add(
acc,
madd_pair!(13, 15, &crate::itx_2d::DCT16_KBP_X4, (kbase >> 1) + 3),
)
}};
}
macro_rules! d_at {
($m:expr) => {{
let kbase = ($m) * 8;
avx2_i32x16_add(
madd_pair!(2, 6, &crate::itx_2d::DCT16_KDP_X4, kbase >> 1),
madd_pair!(10, 14, &crate::itx_2d::DCT16_KDP_X4, (kbase >> 1) + 1),
)
}};
}
let f0 = madd_pair!(4, 12, &crate::itx_2d::DCT16_KFP_X4, 0);
let f1 = madd_pair!(4, 12, &crate::itx_2d::DCT16_KFP_X4, 1);
let g0 = madd_pair!(0, 8, &crate::itx_2d::DCT16_KGP_X4, 0);
let g1 = madd_pair!(0, 8, &crate::itx_2d::DCT16_KGP_X4, 1);
macro_rules! cc_at {
($k:expr) => {{
match $k {
0 => avx2_i32x16_add(g0, f0),
1 => avx2_i32x16_add(g1, f1),
2 => avx2_i32x16_sub(g1, f1),
_ => avx2_i32x16_sub(g0, f0),
}
}};
}
macro_rules! a_at {
($k:expr) => {{
let k = $k;
if k < 4 {
avx2_i32x16_add(cc_at!(k), d_at!(k))
} else {
avx2_i32x16_sub(cc_at!(7 - k), d_at!(7 - k))
}
}};
}
macro_rules! write_group {
($k:expr) => {{
let k = $k;
let a0 = a_at!(k);
let b0 = b_at!(k);
avx2_writeback16_i32_u8::<STRIDE, 16>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
k,
avx2_i32x16_add(a0, b0),
rnd1,
sh1,
);
let a1 = a_at!(k + 1);
let b1 = b_at!(k + 1);
avx2_writeback16_i32_u8::<STRIDE, 16>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
k + 1,
avx2_i32x16_add(a1, b1),
rnd1,
sh1,
);
let a2 = a_at!(k + 2);
let b2 = b_at!(k + 2);
avx2_writeback16_i32_u8::<STRIDE, 16>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
k + 2,
avx2_i32x16_add(a2, b2),
rnd1,
sh1,
);
let a3 = a_at!(k + 3);
let b3 = b_at!(k + 3);
avx2_writeback16_i32_u8::<STRIDE, 16>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
k + 3,
avx2_i32x16_add(a3, b3),
rnd1,
sh1,
);
avx2_writeback16_i32_u8::<STRIDE, 16>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
8 + 7 - (k + 3),
avx2_i32x16_sub(a3, b3),
rnd1,
sh1,
);
avx2_writeback16_i32_u8::<STRIDE, 16>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
8 + 7 - (k + 2),
avx2_i32x16_sub(a2, b2),
rnd1,
sh1,
);
avx2_writeback16_i32_u8::<STRIDE, 16>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
8 + 7 - (k + 1),
avx2_i32x16_sub(a1, b1),
rnd1,
sh1,
);
avx2_writeback16_i32_u8::<STRIDE, 16>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
8 + 7 - k,
avx2_i32x16_sub(a0, b0),
rnd1,
sh1,
);
}};
}
write_group!(0usize);
write_group!(4usize);
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct32_i16x16_scratch16_stride_active_add_u8<const STRIDE: usize, const ACTIVE: usize>(
scratch: &[i16],
base: usize,
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
rnd1: __m256i,
sh1: __m128i,
) {
debug_assert!(ACTIVE == 4 || ACTIVE == 8 || ACTIVE == 16 || ACTIVE == 32);
debug_assert!(base + 16 <= STRIDE);
debug_assert!(base + (ACTIVE - 1) * STRIDE + 16 <= scratch.len());
let zero_lane = _mm256_setzero_si256();
macro_rules! load {
($idx:expr) => {
if ($idx) < ACTIVE {
avx2_load16_i16_scratch(scratch, base + ($idx) * STRIDE)
} else {
zero_lane
}
};
}
macro_rules! madd_pair {
($a:expr, $b:expr, $tbl:expr, $idx:expr) => {
avx2_madd_i16x16_pair(load!($a), load!($b), avx2_coeff_pair_i16x8($tbl, $idx))
};
}
macro_rules! b_at {
($m:expr) => {{
let m = $m;
let kbase = m * 16;
let mut acc = avx2_i32x16_zero();
let mut pair = 0usize;
while pair < 8 {
let i0 = 4 * pair + 1;
acc = avx2_i32x16_add(
acc,
madd_pair!(
i0,
i0 + 2,
&crate::itx_2d::DCT32_KBP_X4,
(kbase >> 1) + pair
),
);
pair += 1;
}
acc
}};
}
macro_rules! d_at {
($m:expr) => {{
let m = $m;
let kbase = m * 8;
let mut acc = avx2_i32x16_zero();
let mut pair = 0usize;
while pair < 4 {
let i0 = 8 * pair + 2;
acc = avx2_i32x16_add(
acc,
madd_pair!(
i0,
i0 + 4,
&crate::itx_2d::DCT32_KDP_X4,
(kbase >> 1) + pair
),
);
pair += 1;
}
acc
}};
}
macro_rules! f_at {
($m:expr) => {{
let kbase = ($m) * 8;
avx2_i32x16_add(
madd_pair!(4, 12, &crate::itx_2d::DCT32_KFP_X4, kbase >> 1),
madd_pair!(20, 28, &crate::itx_2d::DCT32_KFP_X4, (kbase >> 1) + 1),
)
}};
}
let h0 = madd_pair!(8, 24, &crate::itx_2d::DCT32_KHP_X4, 0);
let h1 = madd_pair!(8, 24, &crate::itx_2d::DCT32_KHP_X4, 1);
let g0 = madd_pair!(0, 16, &crate::itx_2d::DCT32_KGP_X4, 0);
let g1 = madd_pair!(0, 16, &crate::itx_2d::DCT32_KGP_X4, 1);
macro_rules! e_at {
($k:expr) => {{
match $k {
0 => avx2_i32x16_add(g0, h0),
1 => avx2_i32x16_add(g1, h1),
2 => avx2_i32x16_sub(g1, h1),
_ => avx2_i32x16_sub(g0, h0),
}
}};
}
macro_rules! cc_at {
($k:expr) => {{
let k = $k;
if k < 4 {
avx2_i32x16_add(e_at!(k), f_at!(k))
} else {
avx2_i32x16_sub(e_at!(7 - k), f_at!(7 - k))
}
}};
}
macro_rules! a_at {
($k:expr) => {{
let k = $k;
if k < 8 {
avx2_i32x16_add(cc_at!(k), d_at!(k))
} else {
avx2_i32x16_sub(cc_at!(15 - k), d_at!(15 - k))
}
}};
}
macro_rules! write_group {
($k:expr) => {{
let k = $k;
let a0 = a_at!(k);
let b0 = b_at!(k);
let l0 = avx2_i32x16_add(a0, b0);
let h0 = avx2_i32x16_sub(a0, b0);
let a1 = a_at!(k + 1);
let b1 = b_at!(k + 1);
let l1 = avx2_i32x16_add(a1, b1);
let h1 = avx2_i32x16_sub(a1, b1);
let a2 = a_at!(k + 2);
let b2 = b_at!(k + 2);
let l2 = avx2_i32x16_add(a2, b2);
let h2 = avx2_i32x16_sub(a2, b2);
let a3 = a_at!(k + 3);
let b3 = b_at!(k + 3);
let l3 = avx2_i32x16_add(a3, b3);
let h3 = avx2_i32x16_sub(a3, b3);
avx2_writeback16_i32_u8::<STRIDE, 32>(
dst, dst_off, dst_stride, out_w, out_h, base, k, l0, rnd1, sh1,
);
avx2_writeback16_i32_u8::<STRIDE, 32>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
k + 1,
l1,
rnd1,
sh1,
);
avx2_writeback16_i32_u8::<STRIDE, 32>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
k + 2,
l2,
rnd1,
sh1,
);
avx2_writeback16_i32_u8::<STRIDE, 32>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
k + 3,
l3,
rnd1,
sh1,
);
avx2_writeback16_i32_u8::<STRIDE, 32>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
16 + 15 - (k + 3),
h3,
rnd1,
sh1,
);
avx2_writeback16_i32_u8::<STRIDE, 32>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
16 + 15 - (k + 2),
h2,
rnd1,
sh1,
);
avx2_writeback16_i32_u8::<STRIDE, 32>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
16 + 15 - (k + 1),
h1,
rnd1,
sh1,
);
avx2_writeback16_i32_u8::<STRIDE, 32>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
base,
16 + 15 - k,
h0,
rnd1,
sh1,
);
}};
}
write_group!(0usize);
write_group!(4usize);
write_group!(8usize);
write_group!(12usize);
}
#[target_feature(enable = "avx2")]
fn avx2_dct16_i16x8_scratch8_stride_active_store<const STRIDE: usize, const ACTIVE: usize>(
scratch: &[i16],
base: usize,
tmp: &mut [i32; ITX_TMP_PIXELS],
) {
unsafe {
debug_assert!(ACTIVE == 4 || ACTIVE == 8 || ACTIVE == 16);
debug_assert!(base + 8 <= STRIDE);
debug_assert!(base + (ACTIVE - 1) * STRIDE + 8 <= scratch.len());
let z128 = _mm_setzero_si128();
let z = _mm256_setzero_si256();
macro_rules! load {
($idx:expr) => {
if ($idx) < ACTIVE {
avx2_load8_i16_scratch(scratch, base + ($idx) * STRIDE)
} else {
z128
}
};
}
macro_rules! madd_pair {
($i0:expr, $i1:expr, $tbl:expr, $idx:expr) => {
_mm256_madd_epi16(
avx2_pair8_i16(load!($i0), load!($i1)),
avx2_coeff_pair_i16x8($tbl, $idx),
)
};
}
let mut b = [z; 8];
let mut m = 0usize;
while m < 8 {
let kbase = m * 8;
let mut acc = z;
if ACTIVE > 1 {
acc = _mm256_add_epi32(
acc,
madd_pair!(1, 3, &crate::itx_2d::DCT16_KBP_X4, kbase >> 1),
);
}
if ACTIVE > 5 {
acc = _mm256_add_epi32(
acc,
madd_pair!(5, 7, &crate::itx_2d::DCT16_KBP_X4, (kbase >> 1) + 1),
);
}
if ACTIVE > 9 {
acc = _mm256_add_epi32(
acc,
madd_pair!(9, 11, &crate::itx_2d::DCT16_KBP_X4, (kbase >> 1) + 2),
);
}
if ACTIVE > 13 {
acc = _mm256_add_epi32(
acc,
madd_pair!(13, 15, &crate::itx_2d::DCT16_KBP_X4, (kbase >> 1) + 3),
);
}
b[m] = acc;
m += 1;
}
let mut d = [z; 4];
m = 0;
while m < 4 {
let kbase = m * 8;
let mut acc = z;
if ACTIVE > 2 {
acc = _mm256_add_epi32(
acc,
madd_pair!(2, 6, &crate::itx_2d::DCT16_KDP_X4, kbase >> 1),
);
}
if ACTIVE > 10 {
acc = _mm256_add_epi32(
acc,
madd_pair!(10, 14, &crate::itx_2d::DCT16_KDP_X4, (kbase >> 1) + 1),
);
}
d[m] = acc;
m += 1;
}
let f0 = if ACTIVE > 4 {
madd_pair!(4, 12, &crate::itx_2d::DCT16_KFP_X4, 0)
} else {
z
};
let f1 = if ACTIVE > 4 {
madd_pair!(4, 12, &crate::itx_2d::DCT16_KFP_X4, 1)
} else {
z
};
let g0 = madd_pair!(0, 8, &crate::itx_2d::DCT16_KGP_X4, 0);
let g1 = madd_pair!(0, 8, &crate::itx_2d::DCT16_KGP_X4, 1);
let cc0 = _mm256_add_epi32(g0, f0);
let cc1 = _mm256_add_epi32(g1, f1);
let cc2 = _mm256_sub_epi32(g1, f1);
let cc3 = _mm256_sub_epi32(g0, f0);
let a0 = _mm256_add_epi32(cc0, d[0]);
let a1 = _mm256_add_epi32(cc1, d[1]);
let a2 = _mm256_add_epi32(cc2, d[2]);
let a3 = _mm256_add_epi32(cc3, d[3]);
let a4 = _mm256_sub_epi32(cc3, d[3]);
let a5 = _mm256_sub_epi32(cc2, d[2]);
let a6 = _mm256_sub_epi32(cc1, d[1]);
let a7 = _mm256_sub_epi32(cc0, d[0]);
let a = [a0, a1, a2, a3, a4, a5, a6, a7];
let mut k = 0usize;
while k < 8 {
_mm256_storeu_si256(
tmp.as_mut_ptr().add(base + k * 32).cast(),
_mm256_add_epi32(a[k], b[k]),
);
_mm256_storeu_si256(
tmp.as_mut_ptr().add(base + (k + 8) * 32).cast(),
_mm256_sub_epi32(a[7 - k], b[7 - k]),
);
k += 1;
}
}
}
#[target_feature(enable = "avx2")]
fn avx2_dct16_i16x8_scratch8_stride_active_add_u8<const STRIDE: usize, const ACTIVE: usize>(
scratch: &[i16],
base: usize,
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
rnd1: __m256i,
sh1: __m128i,
) {
debug_assert!(ACTIVE == 4 || ACTIVE == 8 || ACTIVE == 16);
debug_assert!(base + 8 <= STRIDE);
debug_assert!(base + (ACTIVE - 1) * STRIDE + 8 <= scratch.len());
let z128 = _mm_setzero_si128();
let z = _mm256_setzero_si256();
macro_rules! load {
($idx:expr) => {
if ($idx) < ACTIVE {
avx2_load8_i16_scratch(scratch, base + ($idx) * STRIDE)
} else {
z128
}
};
}
macro_rules! madd_pair {
($i0:expr, $i1:expr, $tbl:expr, $idx:expr) => {
_mm256_madd_epi16(
avx2_pair8_i16(load!($i0), load!($i1)),
avx2_coeff_pair_i16x8($tbl, $idx),
)
};
}
let mut b = [z; 8];
let mut m = 0usize;
while m < 8 {
let kbase = m * 8;
let mut acc = z;
if ACTIVE > 1 {
acc = _mm256_add_epi32(
acc,
madd_pair!(1, 3, &crate::itx_2d::DCT16_KBP_X4, kbase >> 1),
);
}
if ACTIVE > 5 {
acc = _mm256_add_epi32(
acc,
madd_pair!(5, 7, &crate::itx_2d::DCT16_KBP_X4, (kbase >> 1) + 1),
);
}
if ACTIVE > 9 {
acc = _mm256_add_epi32(
acc,
madd_pair!(9, 11, &crate::itx_2d::DCT16_KBP_X4, (kbase >> 1) + 2),
);
}
if ACTIVE > 13 {
acc = _mm256_add_epi32(
acc,
madd_pair!(13, 15, &crate::itx_2d::DCT16_KBP_X4, (kbase >> 1) + 3),
);
}
b[m] = acc;
m += 1;
}
let mut d = [z; 4];
m = 0;
while m < 4 {
let kbase = m * 8;
let mut acc = z;
if ACTIVE > 2 {
acc = _mm256_add_epi32(
acc,
madd_pair!(2, 6, &crate::itx_2d::DCT16_KDP_X4, kbase >> 1),
);
}
if ACTIVE > 10 {
acc = _mm256_add_epi32(
acc,
madd_pair!(10, 14, &crate::itx_2d::DCT16_KDP_X4, (kbase >> 1) + 1),
);
}
d[m] = acc;
m += 1;
}
let f0 = if ACTIVE > 4 {
madd_pair!(4, 12, &crate::itx_2d::DCT16_KFP_X4, 0)
} else {
z
};
let f1 = if ACTIVE > 4 {
madd_pair!(4, 12, &crate::itx_2d::DCT16_KFP_X4, 1)
} else {
z
};
let g0 = madd_pair!(0, 8, &crate::itx_2d::DCT16_KGP_X4, 0);
let g1 = madd_pair!(0, 8, &crate::itx_2d::DCT16_KGP_X4, 1);
let cc0 = _mm256_add_epi32(g0, f0);
let cc1 = _mm256_add_epi32(g1, f1);
let cc2 = _mm256_sub_epi32(g1, f1);
let cc3 = _mm256_sub_epi32(g0, f0);
let a0 = _mm256_add_epi32(cc0, d[0]);
let a1 = _mm256_add_epi32(cc1, d[1]);
let a2 = _mm256_add_epi32(cc2, d[2]);
let a3 = _mm256_add_epi32(cc3, d[3]);
let a4 = _mm256_sub_epi32(cc3, d[3]);
let a5 = _mm256_sub_epi32(cc2, d[2]);
let a6 = _mm256_sub_epi32(cc1, d[1]);
let a7 = _mm256_sub_epi32(cc0, d[0]);
macro_rules! write_row {
($row:expr, $v:expr) => {
avx2_writeback8_i32_u8::<STRIDE, 16>(
dst, dst_off, dst_stride, out_w, out_h, base, $row, $v, rnd1, sh1,
);
};
}
write_row!(0, _mm256_add_epi32(a0, b[0]));
write_row!(1, _mm256_add_epi32(a1, b[1]));
write_row!(2, _mm256_add_epi32(a2, b[2]));
write_row!(3, _mm256_add_epi32(a3, b[3]));
write_row!(4, _mm256_add_epi32(a4, b[4]));
write_row!(5, _mm256_add_epi32(a5, b[5]));
write_row!(6, _mm256_add_epi32(a6, b[6]));
write_row!(7, _mm256_add_epi32(a7, b[7]));
write_row!(8, _mm256_sub_epi32(a7, b[7]));
write_row!(9, _mm256_sub_epi32(a6, b[6]));
write_row!(10, _mm256_sub_epi32(a5, b[5]));
write_row!(11, _mm256_sub_epi32(a4, b[4]));
write_row!(12, _mm256_sub_epi32(a3, b[3]));
write_row!(13, _mm256_sub_epi32(a2, b[2]));
write_row!(14, _mm256_sub_epi32(a1, b[1]));
write_row!(15, _mm256_sub_epi32(a0, b[0]));
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct32_i16x8_scratch8_stride_active_store<const STRIDE: usize, const ACTIVE: usize>(
scratch: &[i16],
base: usize,
tmp: &mut [i32; ITX_TMP_PIXELS],
) {
unsafe {
debug_assert!(ACTIVE == 4 || ACTIVE == 8 || ACTIVE == 16 || ACTIVE == 32);
debug_assert!(base + 8 <= STRIDE);
debug_assert!(base + (ACTIVE - 1) * STRIDE + 8 <= scratch.len());
let z128 = _mm_setzero_si128();
let z = _mm256_setzero_si256();
macro_rules! load {
($idx:expr) => {
if ($idx) < ACTIVE {
avx2_load8_i16_scratch(scratch, base + ($idx) * STRIDE)
} else {
z128
}
};
}
macro_rules! madd_pair {
($i0:expr, $i1:expr, $tbl:expr, $idx:expr) => {
_mm256_madd_epi16(
avx2_pair8_i16(load!($i0), load!($i1)),
avx2_coeff_pair_i16x8($tbl, $idx),
)
};
}
let mut b = [z; 16];
let mut m = 0usize;
while m < 16 {
let kbase = m * 16;
let mut acc = z;
let mut pair = 0usize;
while pair < 8 {
let i0 = 4 * pair + 1;
if ACTIVE > i0 {
acc = _mm256_add_epi32(
acc,
madd_pair!(
i0,
i0 + 2,
&crate::itx_2d::DCT32_KBP_X4,
(kbase >> 1) + pair
),
);
}
pair += 1;
}
b[m] = acc;
m += 1;
}
let mut d = [z; 8];
m = 0;
while m < 8 {
let kbase = m * 8;
let mut acc = z;
let mut pair = 0usize;
while pair < 4 {
let i0 = 8 * pair + 2;
if ACTIVE > i0 {
acc = _mm256_add_epi32(
acc,
madd_pair!(
i0,
i0 + 4,
&crate::itx_2d::DCT32_KDP_X4,
(kbase >> 1) + pair
),
);
}
pair += 1;
}
d[m] = acc;
m += 1;
}
let mut f = [z; 4];
m = 0;
while m < 4 {
let kbase = m * 8;
let mut acc = z;
if ACTIVE > 4 {
acc = _mm256_add_epi32(
acc,
madd_pair!(4, 12, &crate::itx_2d::DCT32_KFP_X4, kbase >> 1),
);
}
if ACTIVE > 20 {
acc = _mm256_add_epi32(
acc,
madd_pair!(20, 28, &crate::itx_2d::DCT32_KFP_X4, (kbase >> 1) + 1),
);
}
f[m] = acc;
m += 1;
}
let h0 = if ACTIVE > 8 {
madd_pair!(8, 24, &crate::itx_2d::DCT32_KHP_X4, 0)
} else {
z
};
let h1 = if ACTIVE > 8 {
madd_pair!(8, 24, &crate::itx_2d::DCT32_KHP_X4, 1)
} else {
z
};
let g0 = madd_pair!(0, 16, &crate::itx_2d::DCT32_KGP_X4, 0);
let g1 = madd_pair!(0, 16, &crate::itx_2d::DCT32_KGP_X4, 1);
let e0 = _mm256_add_epi32(g0, h0);
let e1 = _mm256_add_epi32(g1, h1);
let e2 = _mm256_sub_epi32(g1, h1);
let e3 = _mm256_sub_epi32(g0, h0);
let cc0 = _mm256_add_epi32(e0, f[0]);
let cc1 = _mm256_add_epi32(e1, f[1]);
let cc2 = _mm256_add_epi32(e2, f[2]);
let cc3 = _mm256_add_epi32(e3, f[3]);
let cc4 = _mm256_sub_epi32(e3, f[3]);
let cc5 = _mm256_sub_epi32(e2, f[2]);
let cc6 = _mm256_sub_epi32(e1, f[1]);
let cc7 = _mm256_sub_epi32(e0, f[0]);
let cc = [cc0, cc1, cc2, cc3, cc4, cc5, cc6, cc7];
let a0 = _mm256_add_epi32(cc[0], d[0]);
let a1 = _mm256_add_epi32(cc[1], d[1]);
let a2 = _mm256_add_epi32(cc[2], d[2]);
let a3 = _mm256_add_epi32(cc[3], d[3]);
let a4 = _mm256_add_epi32(cc[4], d[4]);
let a5 = _mm256_add_epi32(cc[5], d[5]);
let a6 = _mm256_add_epi32(cc[6], d[6]);
let a7 = _mm256_add_epi32(cc[7], d[7]);
let a8 = _mm256_sub_epi32(cc[7], d[7]);
let a9 = _mm256_sub_epi32(cc[6], d[6]);
let a10 = _mm256_sub_epi32(cc[5], d[5]);
let a11 = _mm256_sub_epi32(cc[4], d[4]);
let a12 = _mm256_sub_epi32(cc[3], d[3]);
let a13 = _mm256_sub_epi32(cc[2], d[2]);
let a14 = _mm256_sub_epi32(cc[1], d[1]);
let a15 = _mm256_sub_epi32(cc[0], d[0]);
let a = [
a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15,
];
let mut k = 0usize;
while k < 16 {
_mm256_storeu_si256(
tmp.as_mut_ptr().add(base + k * 32).cast(),
_mm256_add_epi32(a[k], b[k]),
);
_mm256_storeu_si256(
tmp.as_mut_ptr().add(base + (k + 16) * 32).cast(),
_mm256_sub_epi32(a[15 - k], b[15 - k]),
);
k += 1;
}
}
}
#[target_feature(enable = "avx2")]
fn avx2_dct32_i16x8_scratch8_stride_active_add_u8<const STRIDE: usize, const ACTIVE: usize>(
scratch: &[i16],
base: usize,
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
rnd1: __m256i,
sh1: __m128i,
) {
debug_assert!(ACTIVE == 4 || ACTIVE == 8 || ACTIVE == 16 || ACTIVE == 32);
debug_assert!(base + 8 <= STRIDE);
debug_assert!(base + (ACTIVE - 1) * STRIDE + 8 <= scratch.len());
let z128 = _mm_setzero_si128();
let z = _mm256_setzero_si256();
macro_rules! load {
($idx:expr) => {
if ($idx) < ACTIVE {
avx2_load8_i16_scratch(scratch, base + ($idx) * STRIDE)
} else {
z128
}
};
}
macro_rules! madd_pair {
($i0:expr, $i1:expr, $tbl:expr, $idx:expr) => {
_mm256_madd_epi16(
avx2_pair8_i16(load!($i0), load!($i1)),
avx2_coeff_pair_i16x8($tbl, $idx),
)
};
}
let mut b = [z; 16];
let mut m = 0usize;
while m < 16 {
let kbase = m * 16;
let mut acc = z;
let mut pair = 0usize;
while pair < 8 {
let i0 = 4 * pair + 1;
if ACTIVE > i0 {
acc = _mm256_add_epi32(
acc,
madd_pair!(
i0,
i0 + 2,
&crate::itx_2d::DCT32_KBP_X4,
(kbase >> 1) + pair
),
);
}
pair += 1;
}
b[m] = acc;
m += 1;
}
let mut d = [z; 8];
m = 0;
while m < 8 {
let kbase = m * 8;
let mut acc = z;
let mut pair = 0usize;
while pair < 4 {
let i0 = 8 * pair + 2;
if ACTIVE > i0 {
acc = _mm256_add_epi32(
acc,
madd_pair!(
i0,
i0 + 4,
&crate::itx_2d::DCT32_KDP_X4,
(kbase >> 1) + pair
),
);
}
pair += 1;
}
d[m] = acc;
m += 1;
}
let mut f = [z; 4];
m = 0;
while m < 4 {
let kbase = m * 8;
let mut acc = z;
if ACTIVE > 4 {
acc = _mm256_add_epi32(
acc,
madd_pair!(4, 12, &crate::itx_2d::DCT32_KFP_X4, kbase >> 1),
);
}
if ACTIVE > 20 {
acc = _mm256_add_epi32(
acc,
madd_pair!(20, 28, &crate::itx_2d::DCT32_KFP_X4, (kbase >> 1) + 1),
);
}
f[m] = acc;
m += 1;
}
let h0 = if ACTIVE > 8 {
madd_pair!(8, 24, &crate::itx_2d::DCT32_KHP_X4, 0)
} else {
z
};
let h1 = if ACTIVE > 8 {
madd_pair!(8, 24, &crate::itx_2d::DCT32_KHP_X4, 1)
} else {
z
};
let g0 = madd_pair!(0, 16, &crate::itx_2d::DCT32_KGP_X4, 0);
let g1 = madd_pair!(0, 16, &crate::itx_2d::DCT32_KGP_X4, 1);
let e0 = _mm256_add_epi32(g0, h0);
let e1 = _mm256_add_epi32(g1, h1);
let e2 = _mm256_sub_epi32(g1, h1);
let e3 = _mm256_sub_epi32(g0, h0);
let cc0 = _mm256_add_epi32(e0, f[0]);
let cc1 = _mm256_add_epi32(e1, f[1]);
let cc2 = _mm256_add_epi32(e2, f[2]);
let cc3 = _mm256_add_epi32(e3, f[3]);
let cc4 = _mm256_sub_epi32(e3, f[3]);
let cc5 = _mm256_sub_epi32(e2, f[2]);
let cc6 = _mm256_sub_epi32(e1, f[1]);
let cc7 = _mm256_sub_epi32(e0, f[0]);
let a0 = _mm256_add_epi32(cc0, d[0]);
let a1 = _mm256_add_epi32(cc1, d[1]);
let a2 = _mm256_add_epi32(cc2, d[2]);
let a3 = _mm256_add_epi32(cc3, d[3]);
let a4 = _mm256_add_epi32(cc4, d[4]);
let a5 = _mm256_add_epi32(cc5, d[5]);
let a6 = _mm256_add_epi32(cc6, d[6]);
let a7 = _mm256_add_epi32(cc7, d[7]);
let a8 = _mm256_sub_epi32(cc7, d[7]);
let a9 = _mm256_sub_epi32(cc6, d[6]);
let a10 = _mm256_sub_epi32(cc5, d[5]);
let a11 = _mm256_sub_epi32(cc4, d[4]);
let a12 = _mm256_sub_epi32(cc3, d[3]);
let a13 = _mm256_sub_epi32(cc2, d[2]);
let a14 = _mm256_sub_epi32(cc1, d[1]);
let a15 = _mm256_sub_epi32(cc0, d[0]);
macro_rules! write_row {
($row:expr, $v:expr) => {
avx2_writeback8_i32_u8::<STRIDE, 32>(
dst, dst_off, dst_stride, out_w, out_h, base, $row, $v, rnd1, sh1,
);
};
}
write_row!(0, _mm256_add_epi32(a0, b[0]));
write_row!(1, _mm256_add_epi32(a1, b[1]));
write_row!(2, _mm256_add_epi32(a2, b[2]));
write_row!(3, _mm256_add_epi32(a3, b[3]));
write_row!(4, _mm256_add_epi32(a4, b[4]));
write_row!(5, _mm256_add_epi32(a5, b[5]));
write_row!(6, _mm256_add_epi32(a6, b[6]));
write_row!(7, _mm256_add_epi32(a7, b[7]));
write_row!(8, _mm256_add_epi32(a8, b[8]));
write_row!(9, _mm256_add_epi32(a9, b[9]));
write_row!(10, _mm256_add_epi32(a10, b[10]));
write_row!(11, _mm256_add_epi32(a11, b[11]));
write_row!(12, _mm256_add_epi32(a12, b[12]));
write_row!(13, _mm256_add_epi32(a13, b[13]));
write_row!(14, _mm256_add_epi32(a14, b[14]));
write_row!(15, _mm256_add_epi32(a15, b[15]));
write_row!(16, _mm256_sub_epi32(a15, b[15]));
write_row!(17, _mm256_sub_epi32(a14, b[14]));
write_row!(18, _mm256_sub_epi32(a13, b[13]));
write_row!(19, _mm256_sub_epi32(a12, b[12]));
write_row!(20, _mm256_sub_epi32(a11, b[11]));
write_row!(21, _mm256_sub_epi32(a10, b[10]));
write_row!(22, _mm256_sub_epi32(a9, b[9]));
write_row!(23, _mm256_sub_epi32(a8, b[8]));
write_row!(24, _mm256_sub_epi32(a7, b[7]));
write_row!(25, _mm256_sub_epi32(a6, b[6]));
write_row!(26, _mm256_sub_epi32(a5, b[5]));
write_row!(27, _mm256_sub_epi32(a4, b[4]));
write_row!(28, _mm256_sub_epi32(a3, b[3]));
write_row!(29, _mm256_sub_epi32(a2, b[2]));
write_row!(30, _mm256_sub_epi32(a1, b[1]));
write_row!(31, _mm256_sub_epi32(a0, b[0]));
}
#[target_feature(enable = "avx2")]
fn avx2_dct16_i16x16_scratch16_stride_eob_store<const STRIDE: usize>(
scratch: &[i16],
base: usize,
active: usize,
tmp: &mut [i32; ITX_TMP_PIXELS],
) {
if active <= 4 {
avx2_dct16_i16x16_scratch16_stride_active_store::<STRIDE, 4>(scratch, base, tmp)
} else if active <= 8 {
avx2_dct16_i16x16_scratch16_stride_active_store::<STRIDE, 8>(scratch, base, tmp)
} else {
avx2_dct16_i16x16_scratch16_stride_active_store::<STRIDE, 16>(scratch, base, tmp)
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct32_i16x16_scratch16_stride_eob_store<const STRIDE: usize>(
scratch: &[i16],
base: usize,
active: usize,
tmp: &mut [i32; ITX_TMP_PIXELS],
) {
if active <= 4 {
avx2_dct32_i16x16_scratch16_stride_active_store::<STRIDE, 4>(scratch, base, tmp)
} else if active <= 8 {
avx2_dct32_i16x16_scratch16_stride_active_store::<STRIDE, 8>(scratch, base, tmp)
} else if active <= 16 {
avx2_dct32_i16x16_scratch16_stride_active_store::<STRIDE, 16>(scratch, base, tmp)
} else {
avx2_dct32_i16x16_scratch16_stride_active_store::<STRIDE, 32>(scratch, base, tmp)
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct16_i16x16_scratch16_stride_eob_add_u8<const STRIDE: usize>(
scratch: &[i16],
base: usize,
active: usize,
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
rnd1: __m256i,
sh1: __m128i,
) {
if active <= 4 {
avx2_dct16_i16x16_scratch16_stride_active_add_u8::<STRIDE, 4>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
} else if active <= 8 {
avx2_dct16_i16x16_scratch16_stride_active_add_u8::<STRIDE, 8>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
} else {
avx2_dct16_i16x16_scratch16_stride_active_add_u8::<STRIDE, 16>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct32_i16x16_scratch16_stride_eob_add_u8<const STRIDE: usize>(
scratch: &[i16],
base: usize,
active: usize,
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
rnd1: __m256i,
sh1: __m128i,
) {
if active <= 4 {
avx2_dct32_i16x16_scratch16_stride_active_add_u8::<STRIDE, 4>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
} else if active <= 8 {
avx2_dct32_i16x16_scratch16_stride_active_add_u8::<STRIDE, 8>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
} else if active <= 16 {
avx2_dct32_i16x16_scratch16_stride_active_add_u8::<STRIDE, 16>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
} else {
avx2_dct32_i16x16_scratch16_stride_active_add_u8::<STRIDE, 32>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct16_i16x8_scratch8_stride_eob_store<const STRIDE: usize>(
scratch: &[i16],
base: usize,
active: usize,
tmp: &mut [i32; ITX_TMP_PIXELS],
) {
if active <= 4 {
avx2_dct16_i16x8_scratch8_stride_active_store::<STRIDE, 4>(scratch, base, tmp)
} else if active <= 8 {
avx2_dct16_i16x8_scratch8_stride_active_store::<STRIDE, 8>(scratch, base, tmp)
} else {
avx2_dct16_i16x8_scratch8_stride_active_store::<STRIDE, 16>(scratch, base, tmp)
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct32_i16x8_scratch8_stride_eob_store<const STRIDE: usize>(
scratch: &[i16],
base: usize,
active: usize,
tmp: &mut [i32; ITX_TMP_PIXELS],
) {
if active <= 4 {
avx2_dct32_i16x8_scratch8_stride_active_store::<STRIDE, 4>(scratch, base, tmp)
} else if active <= 8 {
avx2_dct32_i16x8_scratch8_stride_active_store::<STRIDE, 8>(scratch, base, tmp)
} else if active <= 16 {
avx2_dct32_i16x8_scratch8_stride_active_store::<STRIDE, 16>(scratch, base, tmp)
} else {
avx2_dct32_i16x8_scratch8_stride_active_store::<STRIDE, 32>(scratch, base, tmp)
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct16_i16x8_scratch8_stride_eob_add_u8<const STRIDE: usize>(
scratch: &[i16],
base: usize,
active: usize,
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
rnd1: __m256i,
sh1: __m128i,
) {
if active <= 4 {
avx2_dct16_i16x8_scratch8_stride_active_add_u8::<STRIDE, 4>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
} else if active <= 8 {
avx2_dct16_i16x8_scratch8_stride_active_add_u8::<STRIDE, 8>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
} else {
avx2_dct16_i16x8_scratch8_stride_active_add_u8::<STRIDE, 16>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
}
}
#[target_feature(enable = "avx2")]
fn avx2_dct32_i16x8_scratch8_stride_eob_add_u8<const STRIDE: usize>(
scratch: &[i16],
base: usize,
active: usize,
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
rnd1: __m256i,
sh1: __m128i,
) {
if active <= 4 {
avx2_dct32_i16x8_scratch8_stride_active_add_u8::<STRIDE, 4>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
} else if active <= 8 {
avx2_dct32_i16x8_scratch8_stride_active_add_u8::<STRIDE, 8>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
} else if active <= 16 {
avx2_dct32_i16x8_scratch8_stride_active_add_u8::<STRIDE, 16>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
} else {
avx2_dct32_i16x8_scratch8_stride_active_add_u8::<STRIDE, 32>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct16_i16x4_scratch4_stride_active_store<const STRIDE: usize, const ACTIVE: usize>(
scratch: &[i16],
base: usize,
tmp: &mut [i32; ITX_TMP_PIXELS],
) {
unsafe {
debug_assert!(ACTIVE == 4 || ACTIVE == 8 || ACTIVE == 16);
debug_assert!(base + (ACTIVE - 1) * STRIDE + 4 <= scratch.len());
let z = _mm_setzero_si128();
macro_rules! load {
($idx:expr) => {
if ($idx) < ACTIVE {
avx2_load4_i16_scratch(scratch, base + ($idx) * STRIDE)
} else {
z
}
};
}
let mut b = [z; 8];
let mut m = 0usize;
while m < 8 {
let kbase = m * 8;
let mut acc = z;
if ACTIVE > 1 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(1), load!(3)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KBP_X4, kbase >> 1),
),
);
}
if ACTIVE > 5 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(5), load!(7)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KBP_X4, (kbase >> 1) + 1),
),
);
}
if ACTIVE > 9 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(9), load!(11)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KBP_X4, (kbase >> 1) + 2),
),
);
}
if ACTIVE > 13 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(13), load!(15)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KBP_X4, (kbase >> 1) + 3),
),
);
}
b[m] = acc;
m += 1;
}
let mut d = [z; 4];
m = 0;
while m < 4 {
let kbase = m * 8;
let mut acc = z;
if ACTIVE > 2 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(2), load!(6)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KDP_X4, kbase >> 1),
),
);
}
if ACTIVE > 10 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(10), load!(14)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KDP_X4, (kbase >> 1) + 1),
),
);
}
d[m] = acc;
m += 1;
}
let x412 = _mm_unpacklo_epi16(load!(4), load!(12));
let f0 = if ACTIVE > 4 {
_mm_madd_epi16(x412, avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KFP_X4, 0))
} else {
z
};
let f1 = if ACTIVE > 4 {
_mm_madd_epi16(x412, avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KFP_X4, 1))
} else {
z
};
let x08 = _mm_unpacklo_epi16(load!(0), load!(8));
let g0 = _mm_madd_epi16(x08, avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KGP_X4, 0));
let g1 = _mm_madd_epi16(x08, avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KGP_X4, 1));
let cc = [
_mm_add_epi32(g0, f0),
_mm_add_epi32(g1, f1),
_mm_sub_epi32(g1, f1),
_mm_sub_epi32(g0, f0),
];
let mut a = [z; 8];
let mut i = 0usize;
while i < 4 {
a[i] = _mm_add_epi32(cc[i], d[i]);
i += 1;
}
while i < 8 {
a[i] = _mm_sub_epi32(cc[7 - i], d[7 - i]);
i += 1;
}
let mut k = 0usize;
while k < 8 {
_mm_storeu_si128(
tmp.as_mut_ptr().add(base + k * 32).cast(),
_mm_add_epi32(a[k], b[k]),
);
_mm_storeu_si128(
tmp.as_mut_ptr().add(base + (k + 8) * 32).cast(),
_mm_sub_epi32(a[7 - k], b[7 - k]),
);
k += 1;
}
}
}
#[target_feature(enable = "avx2")]
fn avx2_dct32_i16x4_scratch4_stride_active_store<const STRIDE: usize, const ACTIVE: usize>(
scratch: &[i16],
base: usize,
tmp: &mut [i32; ITX_TMP_PIXELS],
) {
unsafe {
debug_assert!(ACTIVE == 4 || ACTIVE == 8 || ACTIVE == 16 || ACTIVE == 32);
debug_assert!(base + (ACTIVE - 1) * STRIDE + 4 <= scratch.len());
let z = _mm_setzero_si128();
macro_rules! load {
($idx:expr) => {
if ($idx) < ACTIVE {
avx2_load4_i16_scratch(scratch, base + ($idx) * STRIDE)
} else {
z
}
};
}
let mut b = [z; 16];
let mut m = 0usize;
while m < 16 {
let kbase = m * 16;
let mut acc = z;
let mut pair = 0usize;
while pair < 8 {
let i0 = 4 * pair + 1;
if ACTIVE > i0 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(i0), load!(i0 + 2)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KBP_X4, (kbase >> 1) + pair),
),
);
}
pair += 1;
}
b[m] = acc;
m += 1;
}
let mut d = [z; 8];
m = 0;
while m < 8 {
let kbase = m * 8;
let mut acc = z;
let mut pair = 0usize;
while pair < 4 {
let i0 = 8 * pair + 2;
if ACTIVE > i0 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(i0), load!(i0 + 4)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KDP_X4, (kbase >> 1) + pair),
),
);
}
pair += 1;
}
d[m] = acc;
m += 1;
}
let mut f = [z; 4];
m = 0;
while m < 4 {
let kbase = m * 8;
let mut acc = z;
if ACTIVE > 4 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(4), load!(12)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KFP_X4, kbase >> 1),
),
);
}
if ACTIVE > 20 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(20), load!(28)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KFP_X4, (kbase >> 1) + 1),
),
);
}
f[m] = acc;
m += 1;
}
let x824 = _mm_unpacklo_epi16(load!(8), load!(24));
let h0 = if ACTIVE > 8 {
_mm_madd_epi16(x824, avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KHP_X4, 0))
} else {
z
};
let h1 = if ACTIVE > 8 {
_mm_madd_epi16(x824, avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KHP_X4, 1))
} else {
z
};
let x016 = _mm_unpacklo_epi16(load!(0), load!(16));
let g0 = _mm_madd_epi16(x016, avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KGP_X4, 0));
let g1 = _mm_madd_epi16(x016, avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KGP_X4, 1));
let e = [
_mm_add_epi32(g0, h0),
_mm_add_epi32(g1, h1),
_mm_sub_epi32(g1, h1),
_mm_sub_epi32(g0, h0),
];
let mut cc = [z; 8];
let mut i = 0usize;
while i < 4 {
cc[i] = _mm_add_epi32(e[i], f[i]);
i += 1;
}
while i < 8 {
cc[i] = _mm_sub_epi32(e[7 - i], f[7 - i]);
i += 1;
}
let mut a = [z; 16];
i = 0;
while i < 8 {
a[i] = _mm_add_epi32(cc[i], d[i]);
i += 1;
}
while i < 16 {
a[i] = _mm_sub_epi32(cc[15 - i], d[15 - i]);
i += 1;
}
let mut k = 0usize;
while k < 16 {
_mm_storeu_si128(
tmp.as_mut_ptr().add(base + k * 32).cast(),
_mm_add_epi32(a[k], b[k]),
);
_mm_storeu_si128(
tmp.as_mut_ptr().add(base + (k + 16) * 32).cast(),
_mm_sub_epi32(a[15 - k], b[15 - k]),
);
k += 1;
}
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct16_i16x4_scratch4_stride_eob_store<const STRIDE: usize>(
scratch: &[i16],
base: usize,
active: usize,
tmp: &mut [i32; ITX_TMP_PIXELS],
) {
if active <= 4 {
avx2_dct16_i16x4_scratch4_stride_active_store::<STRIDE, 4>(scratch, base, tmp)
} else if active <= 8 {
avx2_dct16_i16x4_scratch4_stride_active_store::<STRIDE, 8>(scratch, base, tmp)
} else {
avx2_dct16_i16x4_scratch4_stride_active_store::<STRIDE, 16>(scratch, base, tmp)
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct32_i16x4_scratch4_stride_eob_store<const STRIDE: usize>(
scratch: &[i16],
base: usize,
active: usize,
tmp: &mut [i32; ITX_TMP_PIXELS],
) {
if active <= 4 {
avx2_dct32_i16x4_scratch4_stride_active_store::<STRIDE, 4>(scratch, base, tmp)
} else if active <= 8 {
avx2_dct32_i16x4_scratch4_stride_active_store::<STRIDE, 8>(scratch, base, tmp)
} else if active <= 16 {
avx2_dct32_i16x4_scratch4_stride_active_store::<STRIDE, 16>(scratch, base, tmp)
} else {
avx2_dct32_i16x4_scratch4_stride_active_store::<STRIDE, 32>(scratch, base, tmp)
}
}
#[target_feature(enable = "avx2")]
fn avx2_dct16_i16x4_scratch4_stride_active_add_u8<const STRIDE: usize, const ACTIVE: usize>(
scratch: &[i16],
base: usize,
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
rnd1: __m128i,
sh1: __m128i,
) {
debug_assert!(ACTIVE == 4 || ACTIVE == 8 || ACTIVE == 16);
debug_assert!(base + (ACTIVE - 1) * STRIDE + 4 <= scratch.len());
let z = _mm_setzero_si128();
macro_rules! load {
($idx:expr) => {
if ($idx) < ACTIVE {
avx2_load4_i16_scratch(scratch, base + ($idx) * STRIDE)
} else {
z
}
};
}
let mut b = [z; 8];
let mut m = 0usize;
while m < 8 {
let kbase = m * 8;
let mut acc = z;
if ACTIVE > 1 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(1), load!(3)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KBP_X4, kbase >> 1),
),
);
}
if ACTIVE > 5 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(5), load!(7)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KBP_X4, (kbase >> 1) + 1),
),
);
}
if ACTIVE > 9 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(9), load!(11)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KBP_X4, (kbase >> 1) + 2),
),
);
}
if ACTIVE > 13 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(13), load!(15)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KBP_X4, (kbase >> 1) + 3),
),
);
}
b[m] = acc;
m += 1;
}
let mut d = [z; 4];
m = 0;
while m < 4 {
let kbase = m * 8;
let mut acc = z;
if ACTIVE > 2 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(2), load!(6)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KDP_X4, kbase >> 1),
),
);
}
if ACTIVE > 10 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(10), load!(14)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KDP_X4, (kbase >> 1) + 1),
),
);
}
d[m] = acc;
m += 1;
}
let x412 = _mm_unpacklo_epi16(load!(4), load!(12));
let f0 = if ACTIVE > 4 {
_mm_madd_epi16(x412, avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KFP_X4, 0))
} else {
z
};
let f1 = if ACTIVE > 4 {
_mm_madd_epi16(x412, avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KFP_X4, 1))
} else {
z
};
let x08 = _mm_unpacklo_epi16(load!(0), load!(8));
let g0 = _mm_madd_epi16(x08, avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KGP_X4, 0));
let g1 = _mm_madd_epi16(x08, avx2_coeff_pair_i16(&crate::itx_2d::DCT16_KGP_X4, 1));
let cc0 = _mm_add_epi32(g0, f0);
let cc1 = _mm_add_epi32(g1, f1);
let cc2 = _mm_sub_epi32(g1, f1);
let cc3 = _mm_sub_epi32(g0, f0);
let a0 = _mm_add_epi32(cc0, d[0]);
let a1 = _mm_add_epi32(cc1, d[1]);
let a2 = _mm_add_epi32(cc2, d[2]);
let a3 = _mm_add_epi32(cc3, d[3]);
let a4 = _mm_sub_epi32(cc3, d[3]);
let a5 = _mm_sub_epi32(cc2, d[2]);
let a6 = _mm_sub_epi32(cc1, d[1]);
let a7 = _mm_sub_epi32(cc0, d[0]);
macro_rules! write_row {
($row:expr, $v:expr) => {
avx2_writeback4_i32_u8::<STRIDE, 16>(
dst, dst_off, dst_stride, out_w, out_h, base, $row, $v, rnd1, sh1,
);
};
}
write_row!(0, _mm_add_epi32(a0, b[0]));
write_row!(1, _mm_add_epi32(a1, b[1]));
write_row!(2, _mm_add_epi32(a2, b[2]));
write_row!(3, _mm_add_epi32(a3, b[3]));
write_row!(4, _mm_add_epi32(a4, b[4]));
write_row!(5, _mm_add_epi32(a5, b[5]));
write_row!(6, _mm_add_epi32(a6, b[6]));
write_row!(7, _mm_add_epi32(a7, b[7]));
write_row!(8, _mm_sub_epi32(a7, b[7]));
write_row!(9, _mm_sub_epi32(a6, b[6]));
write_row!(10, _mm_sub_epi32(a5, b[5]));
write_row!(11, _mm_sub_epi32(a4, b[4]));
write_row!(12, _mm_sub_epi32(a3, b[3]));
write_row!(13, _mm_sub_epi32(a2, b[2]));
write_row!(14, _mm_sub_epi32(a1, b[1]));
write_row!(15, _mm_sub_epi32(a0, b[0]));
}
#[target_feature(enable = "avx2")]
fn avx2_dct32_i16x4_scratch4_stride_active_add_u8<const STRIDE: usize, const ACTIVE: usize>(
scratch: &[i16],
base: usize,
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
rnd1: __m128i,
sh1: __m128i,
) {
debug_assert!(ACTIVE == 4 || ACTIVE == 8 || ACTIVE == 16 || ACTIVE == 32);
debug_assert!(base + (ACTIVE - 1) * STRIDE + 4 <= scratch.len());
let z = _mm_setzero_si128();
macro_rules! load {
($idx:expr) => {
if ($idx) < ACTIVE {
avx2_load4_i16_scratch(scratch, base + ($idx) * STRIDE)
} else {
z
}
};
}
let mut b = [z; 16];
let mut m = 0usize;
while m < 16 {
let kbase = m * 16;
let mut acc = z;
let mut pair = 0usize;
while pair < 8 {
let i0 = 4 * pair + 1;
if ACTIVE > i0 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(i0), load!(i0 + 2)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KBP_X4, (kbase >> 1) + pair),
),
);
}
pair += 1;
}
b[m] = acc;
m += 1;
}
let mut d = [z; 8];
m = 0;
while m < 8 {
let kbase = m * 8;
let mut acc = z;
let mut pair = 0usize;
while pair < 4 {
let i0 = 8 * pair + 2;
if ACTIVE > i0 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(i0), load!(i0 + 4)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KDP_X4, (kbase >> 1) + pair),
),
);
}
pair += 1;
}
d[m] = acc;
m += 1;
}
let mut f = [z; 4];
m = 0;
while m < 4 {
let kbase = m * 8;
let mut acc = z;
if ACTIVE > 4 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(4), load!(12)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KFP_X4, kbase >> 1),
),
);
}
if ACTIVE > 20 {
acc = _mm_add_epi32(
acc,
_mm_madd_epi16(
_mm_unpacklo_epi16(load!(20), load!(28)),
avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KFP_X4, (kbase >> 1) + 1),
),
);
}
f[m] = acc;
m += 1;
}
let x824 = _mm_unpacklo_epi16(load!(8), load!(24));
let h0 = if ACTIVE > 8 {
_mm_madd_epi16(x824, avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KHP_X4, 0))
} else {
z
};
let h1 = if ACTIVE > 8 {
_mm_madd_epi16(x824, avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KHP_X4, 1))
} else {
z
};
let x016 = _mm_unpacklo_epi16(load!(0), load!(16));
let g0 = _mm_madd_epi16(x016, avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KGP_X4, 0));
let g1 = _mm_madd_epi16(x016, avx2_coeff_pair_i16(&crate::itx_2d::DCT32_KGP_X4, 1));
let e0 = _mm_add_epi32(g0, h0);
let e1 = _mm_add_epi32(g1, h1);
let e2 = _mm_sub_epi32(g1, h1);
let e3 = _mm_sub_epi32(g0, h0);
let cc0 = _mm_add_epi32(e0, f[0]);
let cc1 = _mm_add_epi32(e1, f[1]);
let cc2 = _mm_add_epi32(e2, f[2]);
let cc3 = _mm_add_epi32(e3, f[3]);
let cc4 = _mm_sub_epi32(e3, f[3]);
let cc5 = _mm_sub_epi32(e2, f[2]);
let cc6 = _mm_sub_epi32(e1, f[1]);
let cc7 = _mm_sub_epi32(e0, f[0]);
let a0 = _mm_add_epi32(cc0, d[0]);
let a1 = _mm_add_epi32(cc1, d[1]);
let a2 = _mm_add_epi32(cc2, d[2]);
let a3 = _mm_add_epi32(cc3, d[3]);
let a4 = _mm_add_epi32(cc4, d[4]);
let a5 = _mm_add_epi32(cc5, d[5]);
let a6 = _mm_add_epi32(cc6, d[6]);
let a7 = _mm_add_epi32(cc7, d[7]);
let a8 = _mm_sub_epi32(cc7, d[7]);
let a9 = _mm_sub_epi32(cc6, d[6]);
let a10 = _mm_sub_epi32(cc5, d[5]);
let a11 = _mm_sub_epi32(cc4, d[4]);
let a12 = _mm_sub_epi32(cc3, d[3]);
let a13 = _mm_sub_epi32(cc2, d[2]);
let a14 = _mm_sub_epi32(cc1, d[1]);
let a15 = _mm_sub_epi32(cc0, d[0]);
macro_rules! write_row {
($row:expr, $v:expr) => {
avx2_writeback4_i32_u8::<STRIDE, 32>(
dst, dst_off, dst_stride, out_w, out_h, base, $row, $v, rnd1, sh1,
);
};
}
write_row!(0, _mm_add_epi32(a0, b[0]));
write_row!(1, _mm_add_epi32(a1, b[1]));
write_row!(2, _mm_add_epi32(a2, b[2]));
write_row!(3, _mm_add_epi32(a3, b[3]));
write_row!(4, _mm_add_epi32(a4, b[4]));
write_row!(5, _mm_add_epi32(a5, b[5]));
write_row!(6, _mm_add_epi32(a6, b[6]));
write_row!(7, _mm_add_epi32(a7, b[7]));
write_row!(8, _mm_add_epi32(a8, b[8]));
write_row!(9, _mm_add_epi32(a9, b[9]));
write_row!(10, _mm_add_epi32(a10, b[10]));
write_row!(11, _mm_add_epi32(a11, b[11]));
write_row!(12, _mm_add_epi32(a12, b[12]));
write_row!(13, _mm_add_epi32(a13, b[13]));
write_row!(14, _mm_add_epi32(a14, b[14]));
write_row!(15, _mm_add_epi32(a15, b[15]));
write_row!(16, _mm_sub_epi32(a15, b[15]));
write_row!(17, _mm_sub_epi32(a14, b[14]));
write_row!(18, _mm_sub_epi32(a13, b[13]));
write_row!(19, _mm_sub_epi32(a12, b[12]));
write_row!(20, _mm_sub_epi32(a11, b[11]));
write_row!(21, _mm_sub_epi32(a10, b[10]));
write_row!(22, _mm_sub_epi32(a9, b[9]));
write_row!(23, _mm_sub_epi32(a8, b[8]));
write_row!(24, _mm_sub_epi32(a7, b[7]));
write_row!(25, _mm_sub_epi32(a6, b[6]));
write_row!(26, _mm_sub_epi32(a5, b[5]));
write_row!(27, _mm_sub_epi32(a4, b[4]));
write_row!(28, _mm_sub_epi32(a3, b[3]));
write_row!(29, _mm_sub_epi32(a2, b[2]));
write_row!(30, _mm_sub_epi32(a1, b[1]));
write_row!(31, _mm_sub_epi32(a0, b[0]));
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct16_i16x4_scratch4_stride_eob_add_u8<const STRIDE: usize>(
scratch: &[i16],
base: usize,
active: usize,
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
rnd1: __m128i,
sh1: __m128i,
) {
if active <= 4 {
avx2_dct16_i16x4_scratch4_stride_active_add_u8::<STRIDE, 4>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
} else if active <= 8 {
avx2_dct16_i16x4_scratch4_stride_active_add_u8::<STRIDE, 8>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
} else {
avx2_dct16_i16x4_scratch4_stride_active_add_u8::<STRIDE, 16>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn avx2_dct32_i16x4_scratch4_stride_eob_add_u8<const STRIDE: usize>(
scratch: &[i16],
base: usize,
active: usize,
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
rnd1: __m128i,
sh1: __m128i,
) {
if active <= 4 {
avx2_dct32_i16x4_scratch4_stride_active_add_u8::<STRIDE, 4>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
} else if active <= 8 {
avx2_dct32_i16x4_scratch4_stride_active_add_u8::<STRIDE, 8>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
} else if active <= 16 {
avx2_dct32_i16x4_scratch4_stride_active_add_u8::<STRIDE, 16>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
} else {
avx2_dct32_i16x4_scratch4_stride_active_add_u8::<STRIDE, 32>(
scratch, base, dst, dst_off, dst_stride, out_w, out_h, rnd1, sh1,
)
}
}
#[target_feature(enable = "avx2")]
fn avx2_dct16_i16x4_all_from_coeff4_stride_const<const IS_RECT2: bool, const STRIDE: usize>(
coeff: &[i16],
base: usize,
) -> [__m128i; 16] {
debug_assert!(base + 15 * STRIDE + 4 <= coeff.len());
macro_rules! load {
($idx:expr) => {
avx2_load4_i16_coeff_packed_const::<IS_RECT2>(coeff, base + ($idx) * STRIDE)
};
}
avx2_dct16_i16x4_all_body!()
}
#[target_feature(enable = "avx2")]
fn avx2_dct32_i16x4_all_from_coeff4_stride_const<const IS_RECT2: bool, const STRIDE: usize>(
coeff: &[i16],
base: usize,
) -> [__m128i; 32] {
debug_assert!(base + 31 * STRIDE + 4 <= coeff.len());
macro_rules! load {
($idx:expr) => {
avx2_load4_i16_coeff_packed_const::<IS_RECT2>(coeff, base + ($idx) * STRIDE)
};
}
avx2_dct32_i16x4_all_body!()
}
#[target_feature(enable = "avx2")]
fn avx2_dct32_i16x8_store_quads_from_coeff8_stride_const<
const IS_RECT2: bool,
const COEFF_STRIDE: usize,
const STRIDE: usize,
>(
coeff: &[i16],
coeff_base: usize,
scratch: &mut [i16],
off: usize,
rnd: __m128i,
sh: __m128i,
minv: __m128i,
maxv: __m128i,
) {
debug_assert!(coeff_base + 31 * COEFF_STRIDE + 8 <= coeff.len());
macro_rules! load {
($idx:expr) => {
avx2_load8_i16_coeff_packed_const::<IS_RECT2>(coeff, coeff_base + ($idx) * COEFF_STRIDE)
};
}
macro_rules! pair {
($a:expr, $b:expr) => {
avx2_pair8_i16_from_rows(load!($a), load!($b))
};
}
macro_rules! madd_pair {
($a:expr, $b:expr, $tbl:expr, $idx:expr) => {
_mm256_madd_epi16(pair!($a, $b), avx2_coeff_pair_i16x8($tbl, $idx))
};
}
macro_rules! b_at {
($m:expr) => {{
let m = $m;
let base = m * 16;
let mut acc = _mm256_setzero_si256();
let mut p = 0usize;
while p < 16 {
let cb = base + p;
let i0 = 2 * p + 1;
acc = _mm256_add_epi32(
acc,
madd_pair!(i0, i0 + 2, &crate::itx_2d::DCT32_KBP_X4, cb >> 1),
);
p += 2;
}
acc
}};
}
macro_rules! d_at {
($m:expr) => {{
let m = $m;
let base = m * 8;
let mut acc = _mm256_setzero_si256();
let mut p = 0usize;
while p < 8 {
let i0 = 4 * p + 2;
acc = _mm256_add_epi32(
acc,
madd_pair!(i0, i0 + 4, &crate::itx_2d::DCT32_KDP_X4, (base + p) >> 1),
);
p += 2;
}
acc
}};
}
macro_rules! f_at {
($m:expr) => {{
let base = ($m) * 8;
_mm256_add_epi32(
madd_pair!(4, 12, &crate::itx_2d::DCT32_KFP_X4, base >> 1),
madd_pair!(20, 28, &crate::itx_2d::DCT32_KFP_X4, (base >> 1) + 1),
)
}};
}
let hh0 = madd_pair!(8, 24, &crate::itx_2d::DCT32_KHP_X4, 0);
let hh1 = madd_pair!(8, 24, &crate::itx_2d::DCT32_KHP_X4, 1);
let gg0 = madd_pair!(0, 16, &crate::itx_2d::DCT32_KGP_X4, 0);
let gg1 = madd_pair!(0, 16, &crate::itx_2d::DCT32_KGP_X4, 1);
macro_rules! e_at {
($k:expr) => {{
match $k {
0 => _mm256_add_epi32(gg0, hh0),
1 => _mm256_add_epi32(gg1, hh1),
2 => _mm256_sub_epi32(gg1, hh1),
_ => _mm256_sub_epi32(gg0, hh0),
}
}};
}
macro_rules! cc_at {
($k:expr) => {{
let k = $k;
if k < 4 {
_mm256_add_epi32(e_at!(k), f_at!(k))
} else {
_mm256_sub_epi32(e_at!(7 - k), f_at!(7 - k))
}
}};
}
macro_rules! a_at {
($k:expr) => {{
let k = $k;
if k < 8 {
_mm256_add_epi32(cc_at!(k), d_at!(k))
} else {
_mm256_sub_epi32(cc_at!(15 - k), d_at!(15 - k))
}
}};
}
macro_rules! store_group {
($k:expr) => {{
let k = $k;
let a0 = a_at!(k);
let b0 = b_at!(k);
let l0 = _mm256_add_epi32(a0, b0);
let h0 = _mm256_sub_epi32(a0, b0);
let a1 = a_at!(k + 1);
let b1 = b_at!(k + 1);
let l1 = _mm256_add_epi32(a1, b1);
let h1 = _mm256_sub_epi32(a1, b1);
let a2 = a_at!(k + 2);
let b2 = b_at!(k + 2);
let l2 = _mm256_add_epi32(a2, b2);
let h2 = _mm256_sub_epi32(a2, b2);
let a3 = a_at!(k + 3);
let b3 = b_at!(k + 3);
let l3 = _mm256_add_epi32(a3, b3);
let h3 = _mm256_sub_epi32(a3, b3);
avx2_store4x8_i16_clip256::<STRIDE>(
scratch,
off + k,
l0,
l1,
l2,
l3,
rnd,
sh,
minv,
maxv,
);
avx2_store4x8_i16_clip256::<STRIDE>(
scratch,
off + 16 + 15 - (k + 3),
h3,
h2,
h1,
h0,
rnd,
sh,
minv,
maxv,
);
}};
}
store_group!(0usize);
store_group!(4usize);
store_group!(8usize);
store_group!(12usize);
}
#[target_feature(enable = "avx2")]
fn avx2_dct32_i16x16_store_quads_from_coeff16_stride_const<
const IS_RECT2: bool,
const COEFF_STRIDE: usize,
const STRIDE: usize,
>(
coeff: &[i16],
coeff_base: usize,
scratch: &mut [i16],
off: usize,
rnd: __m128i,
sh: __m128i,
minv: __m128i,
maxv: __m128i,
) {
debug_assert!(coeff_base + 31 * COEFF_STRIDE + 16 <= coeff.len());
macro_rules! load {
($idx:expr) => {
avx2_load16_i16_coeff_packed_const::<IS_RECT2>(
coeff,
coeff_base + ($idx) * COEFF_STRIDE,
)
};
}
macro_rules! madd_pair {
($a:expr, $b:expr, $tbl:expr, $idx:expr) => {
avx2_madd_i16x16_pair(load!($a), load!($b), avx2_coeff_pair_i16x8($tbl, $idx))
};
}
macro_rules! b_at {
($m:expr) => {{
let m = $m;
let base = m * 16;
let mut acc = avx2_i32x16_zero();
let mut p = 0usize;
while p < 16 {
let cb = base + p;
let i0 = 2 * p + 1;
acc = avx2_i32x16_add(
acc,
madd_pair!(i0, i0 + 2, &crate::itx_2d::DCT32_KBP_X4, cb >> 1),
);
p += 2;
}
acc
}};
}
macro_rules! d_at {
($m:expr) => {{
let m = $m;
let base = m * 8;
let mut acc = avx2_i32x16_zero();
let mut p = 0usize;
while p < 8 {
let i0 = 4 * p + 2;
acc = avx2_i32x16_add(
acc,
madd_pair!(i0, i0 + 4, &crate::itx_2d::DCT32_KDP_X4, (base + p) >> 1),
);
p += 2;
}
acc
}};
}
macro_rules! f_at {
($m:expr) => {{
let base = ($m) * 8;
avx2_i32x16_add(
madd_pair!(4, 12, &crate::itx_2d::DCT32_KFP_X4, base >> 1),
madd_pair!(20, 28, &crate::itx_2d::DCT32_KFP_X4, (base >> 1) + 1),
)
}};
}
let hh0 = madd_pair!(8, 24, &crate::itx_2d::DCT32_KHP_X4, 0);
let hh1 = madd_pair!(8, 24, &crate::itx_2d::DCT32_KHP_X4, 1);
let gg0 = madd_pair!(0, 16, &crate::itx_2d::DCT32_KGP_X4, 0);
let gg1 = madd_pair!(0, 16, &crate::itx_2d::DCT32_KGP_X4, 1);
macro_rules! e_at {
($k:expr) => {{
match $k {
0 => avx2_i32x16_add(gg0, hh0),
1 => avx2_i32x16_add(gg1, hh1),
2 => avx2_i32x16_sub(gg1, hh1),
_ => avx2_i32x16_sub(gg0, hh0),
}
}};
}
macro_rules! cc_at {
($k:expr) => {{
let k = $k;
if k < 4 {
avx2_i32x16_add(e_at!(k), f_at!(k))
} else {
avx2_i32x16_sub(e_at!(7 - k), f_at!(7 - k))
}
}};
}
macro_rules! a_at {
($k:expr) => {{
let k = $k;
if k < 8 {
avx2_i32x16_add(cc_at!(k), d_at!(k))
} else {
avx2_i32x16_sub(cc_at!(15 - k), d_at!(15 - k))
}
}};
}
macro_rules! store_group {
($k:expr) => {{
let k = $k;
let a0 = a_at!(k);
let b0 = b_at!(k);
let l0 = avx2_i32x16_add(a0, b0);
let h0 = avx2_i32x16_sub(a0, b0);
let a1 = a_at!(k + 1);
let b1 = b_at!(k + 1);
let l1 = avx2_i32x16_add(a1, b1);
let h1 = avx2_i32x16_sub(a1, b1);
let a2 = a_at!(k + 2);
let b2 = b_at!(k + 2);
let l2 = avx2_i32x16_add(a2, b2);
let h2 = avx2_i32x16_sub(a2, b2);
let a3 = a_at!(k + 3);
let b3 = b_at!(k + 3);
let l3 = avx2_i32x16_add(a3, b3);
let h3 = avx2_i32x16_sub(a3, b3);
avx2_store4x16_i16_clip256::<STRIDE>(
scratch,
off + k,
l0,
l1,
l2,
l3,
rnd,
sh,
minv,
maxv,
);
avx2_store4x16_i16_clip256::<STRIDE>(
scratch,
off + 16 + 15 - (k + 3),
h3,
h2,
h1,
h0,
rnd,
sh,
minv,
maxv,
);
}};
}
store_group!(0usize);
store_group!(4usize);
store_group!(8usize);
store_group!(12usize);
}
#[target_feature(enable = "avx2")]
fn avx2_dct16_i16x8_store_quads_from_coeff8_stride_const<
const IS_RECT2: bool,
const COEFF_STRIDE: usize,
const STRIDE: usize,
>(
coeff: &[i16],
coeff_base: usize,
scratch: &mut [i16],
off: usize,
rnd: __m128i,
sh: __m128i,
minv: __m128i,
maxv: __m128i,
) {
debug_assert!(coeff_base + 15 * COEFF_STRIDE + 8 <= coeff.len());
macro_rules! load {
($idx:expr) => {
avx2_load8_i16_coeff_packed_const::<IS_RECT2>(coeff, coeff_base + ($idx) * COEFF_STRIDE)
};
}
macro_rules! pair {
($a:expr, $b:expr) => {
avx2_pair8_i16_from_rows(load!($a), load!($b))
};
}
macro_rules! madd_pair {
($a:expr, $b:expr, $tbl:expr, $idx:expr) => {
_mm256_madd_epi16(pair!($a, $b), avx2_coeff_pair_i16x8($tbl, $idx))
};
}
macro_rules! b_at {
($m:expr) => {{
let base = ($m) * 8;
let mut acc = _mm256_setzero_si256();
acc = _mm256_add_epi32(
acc,
madd_pair!(1, 3, &crate::itx_2d::DCT16_KBP_X4, base >> 1),
);
acc = _mm256_add_epi32(
acc,
madd_pair!(5, 7, &crate::itx_2d::DCT16_KBP_X4, (base >> 1) + 1),
);
acc = _mm256_add_epi32(
acc,
madd_pair!(9, 11, &crate::itx_2d::DCT16_KBP_X4, (base >> 1) + 2),
);
_mm256_add_epi32(
acc,
madd_pair!(13, 15, &crate::itx_2d::DCT16_KBP_X4, (base >> 1) + 3),
)
}};
}
macro_rules! d_at {
($m:expr) => {{
let base = ($m) * 8;
_mm256_add_epi32(
madd_pair!(2, 6, &crate::itx_2d::DCT16_KDP_X4, base >> 1),
madd_pair!(10, 14, &crate::itx_2d::DCT16_KDP_X4, (base >> 1) + 1),
)
}};
}
let f0 = madd_pair!(4, 12, &crate::itx_2d::DCT16_KFP_X4, 0);
let f1 = madd_pair!(4, 12, &crate::itx_2d::DCT16_KFP_X4, 1);
let g0 = madd_pair!(0, 8, &crate::itx_2d::DCT16_KGP_X4, 0);
let g1 = madd_pair!(0, 8, &crate::itx_2d::DCT16_KGP_X4, 1);
macro_rules! cc_at {
($k:expr) => {{
match $k {
0 => _mm256_add_epi32(g0, f0),
1 => _mm256_add_epi32(g1, f1),
2 => _mm256_sub_epi32(g1, f1),
_ => _mm256_sub_epi32(g0, f0),
}
}};
}
macro_rules! a_at {
($k:expr) => {{
let k = $k;
if k < 4 {
_mm256_add_epi32(cc_at!(k), d_at!(k))
} else {
_mm256_sub_epi32(cc_at!(7 - k), d_at!(7 - k))
}
}};
}
macro_rules! store_group {
($k:expr) => {{
let k = $k;
let a0 = a_at!(k);
let b0 = b_at!(k);
let l0 = _mm256_add_epi32(a0, b0);
let h0 = _mm256_sub_epi32(a0, b0);
let a1 = a_at!(k + 1);
let b1 = b_at!(k + 1);
let l1 = _mm256_add_epi32(a1, b1);
let h1 = _mm256_sub_epi32(a1, b1);
let a2 = a_at!(k + 2);
let b2 = b_at!(k + 2);
let l2 = _mm256_add_epi32(a2, b2);
let h2 = _mm256_sub_epi32(a2, b2);
let a3 = a_at!(k + 3);
let b3 = b_at!(k + 3);
let l3 = _mm256_add_epi32(a3, b3);
let h3 = _mm256_sub_epi32(a3, b3);
avx2_store4x8_i16_clip256::<STRIDE>(
scratch,
off + k,
l0,
l1,
l2,
l3,
rnd,
sh,
minv,
maxv,
);
avx2_store4x8_i16_clip256::<STRIDE>(
scratch,
off + 8 + 7 - (k + 3),
h3,
h2,
h1,
h0,
rnd,
sh,
minv,
maxv,
);
}};
}
store_group!(0usize);
store_group!(4usize);
}
#[target_feature(enable = "avx2")]
fn avx2_dct16_i16x16_store_quads_from_coeff16_stride_const<
const IS_RECT2: bool,
const COEFF_STRIDE: usize,
const STRIDE: usize,
>(
coeff: &[i16],
coeff_base: usize,
scratch: &mut [i16],
off: usize,
rnd: __m128i,
sh: __m128i,
minv: __m128i,
maxv: __m128i,
) {
debug_assert!(coeff_base + 15 * COEFF_STRIDE + 16 <= coeff.len());
macro_rules! load {
($idx:expr) => {
avx2_load16_i16_coeff_packed_const::<IS_RECT2>(
coeff,
coeff_base + ($idx) * COEFF_STRIDE,
)
};
}
macro_rules! madd_pair {
($a:expr, $b:expr, $tbl:expr, $idx:expr) => {
avx2_madd_i16x16_pair(load!($a), load!($b), avx2_coeff_pair_i16x8($tbl, $idx))
};
}
macro_rules! b_at {
($m:expr) => {{
let base = ($m) * 8;
let mut acc = avx2_i32x16_zero();
acc = avx2_i32x16_add(
acc,
madd_pair!(1, 3, &crate::itx_2d::DCT16_KBP_X4, base >> 1),
);
acc = avx2_i32x16_add(
acc,
madd_pair!(5, 7, &crate::itx_2d::DCT16_KBP_X4, (base >> 1) + 1),
);
acc = avx2_i32x16_add(
acc,
madd_pair!(9, 11, &crate::itx_2d::DCT16_KBP_X4, (base >> 1) + 2),
);
avx2_i32x16_add(
acc,
madd_pair!(13, 15, &crate::itx_2d::DCT16_KBP_X4, (base >> 1) + 3),
)
}};
}
macro_rules! d_at {
($m:expr) => {{
let base = ($m) * 8;
avx2_i32x16_add(
madd_pair!(2, 6, &crate::itx_2d::DCT16_KDP_X4, base >> 1),
madd_pair!(10, 14, &crate::itx_2d::DCT16_KDP_X4, (base >> 1) + 1),
)
}};
}
let f0 = madd_pair!(4, 12, &crate::itx_2d::DCT16_KFP_X4, 0);
let f1 = madd_pair!(4, 12, &crate::itx_2d::DCT16_KFP_X4, 1);
let g0 = madd_pair!(0, 8, &crate::itx_2d::DCT16_KGP_X4, 0);
let g1 = madd_pair!(0, 8, &crate::itx_2d::DCT16_KGP_X4, 1);
macro_rules! cc_at {
($k:expr) => {{
match $k {
0 => avx2_i32x16_add(g0, f0),
1 => avx2_i32x16_add(g1, f1),
2 => avx2_i32x16_sub(g1, f1),
_ => avx2_i32x16_sub(g0, f0),
}
}};
}
macro_rules! a_at {
($k:expr) => {{
let k = $k;
if k < 4 {
avx2_i32x16_add(cc_at!(k), d_at!(k))
} else {
avx2_i32x16_sub(cc_at!(7 - k), d_at!(7 - k))
}
}};
}
macro_rules! store_group {
($k:expr) => {{
let k = $k;
let a0 = a_at!(k);
let b0 = b_at!(k);
let l0 = avx2_i32x16_add(a0, b0);
let h0 = avx2_i32x16_sub(a0, b0);
let a1 = a_at!(k + 1);
let b1 = b_at!(k + 1);
let l1 = avx2_i32x16_add(a1, b1);
let h1 = avx2_i32x16_sub(a1, b1);
let a2 = a_at!(k + 2);
let b2 = b_at!(k + 2);
let l2 = avx2_i32x16_add(a2, b2);
let h2 = avx2_i32x16_sub(a2, b2);
let a3 = a_at!(k + 3);
let b3 = b_at!(k + 3);
let l3 = avx2_i32x16_add(a3, b3);
let h3 = avx2_i32x16_sub(a3, b3);
avx2_store4x16_i16_clip256::<STRIDE>(
scratch,
off + k,
l0,
l1,
l2,
l3,
rnd,
sh,
minv,
maxv,
);
avx2_store4x16_i16_clip256::<STRIDE>(
scratch,
off + 8 + 7 - (k + 3),
h3,
h2,
h1,
h0,
rnd,
sh,
minv,
maxv,
);
}};
}
store_group!(0usize);
store_group!(4usize);
}
#[target_feature(enable = "avx2")]
fn avx2_dct16_i16x4_coeff_rows_to_scratch<const IS_RECT2: bool, const COEFF_STRIDE: usize>(
coeff: &[i16],
scratch: &mut [i16],
mut y: usize,
nrows: usize,
rnd: __m128i,
sh: __m128i,
minv: __m128i,
maxv: __m128i,
) -> usize {
while y + 16 <= nrows {
avx2_dct16_i16x16_store_quads_from_coeff16_stride_const::<IS_RECT2, COEFF_STRIDE, 16>(
coeff,
y,
scratch,
y * 16,
rnd,
sh,
minv,
maxv,
);
y += 16;
}
while y + 8 <= nrows {
avx2_dct16_i16x8_store_quads_from_coeff8_stride_const::<IS_RECT2, COEFF_STRIDE, 16>(
coeff,
y,
scratch,
y * 16,
rnd,
sh,
minv,
maxv,
);
y += 8;
}
if y + 4 <= nrows {
let out = avx2_dct16_i16x4_all_from_coeff4_stride_const::<IS_RECT2, COEFF_STRIDE>(coeff, y);
let row_base = y * 16;
let mut m = 0usize;
while m < 16 {
avx2_store4x4_i16_clip::<16>(
scratch,
row_base + m,
out[m],
out[m + 1],
out[m + 2],
out[m + 3],
rnd,
sh,
minv,
maxv,
);
m += 4;
}
y += 4;
}
y
}
#[target_feature(enable = "avx2")]
fn avx2_dct32_i16x4_coeff_rows_to_scratch<const IS_RECT2: bool, const COEFF_STRIDE: usize>(
coeff: &[i16],
scratch: &mut [i16],
mut y: usize,
nrows: usize,
rnd: __m128i,
sh: __m128i,
minv: __m128i,
maxv: __m128i,
) -> usize {
while y + 16 <= nrows {
let row_base = y * 32;
avx2_dct32_i16x16_store_quads_from_coeff16_stride_const::<IS_RECT2, COEFF_STRIDE, 32>(
coeff, y, scratch, row_base, rnd, sh, minv, maxv,
);
y += 16;
}
while y + 8 <= nrows {
let row_base = y * 32;
avx2_dct32_i16x8_store_quads_from_coeff8_stride_const::<IS_RECT2, COEFF_STRIDE, 32>(
coeff, y, scratch, row_base, rnd, sh, minv, maxv,
);
y += 8;
}
if y + 4 <= nrows {
let out = avx2_dct32_i16x4_all_from_coeff4_stride_const::<IS_RECT2, COEFF_STRIDE>(coeff, y);
let row_base = y * 32;
let mut m = 0usize;
while m < 32 {
avx2_store4x4_i16_clip::<32>(
scratch,
row_base + m,
out[m],
out[m + 1],
out[m + 2],
out[m + 3],
rnd,
sh,
minv,
maxv,
);
m += 4;
}
y += 4;
}
y
}
#[inline]
#[target_feature(enable = "avx2")]
fn idct_dequant_dct_i16_avx2_impl<const N: usize>(
coeff: &mut [i16],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
if is_rect2 {
idct_dequant_dct_i16_avx2_impl_const::<N, true>(
coeff,
tmp,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
)
} else {
idct_dequant_dct_i16_avx2_impl_const::<N, false>(
coeff,
tmp,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
)
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn idct_dequant_dct_i16_avx2_impl_const<const N: usize, const IS_RECT2: bool>(
coeff: &mut [i16],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
debug_assert!(N == 16 || N == 32);
debug_assert!(coeff.len() >= N * N);
let off = usize::from(crate::scan::LAST_EOB_PER_COL.offset[tx]);
let last_eob = &crate::scan::LAST_EOB_PER_COL.table[off..];
let mut ngrp = 0usize;
while ngrp < N / 4 {
ngrp += 1;
if eob <= last_eob[ngrp - 1] as i32 {
break;
}
}
let ncols = ngrp * 4;
let rnd = _mm_set1_epi32((1 << shift0) >> 1);
let sh = _mm_cvtsi32_si128(shift0);
let minv = _mm_set1_epi32(row_clip_min);
let maxv = _mm_set1_epi32(row_clip_max);
with_avx2_itx_i16_scratch(N * N, |scratch| {
let mut y = 0usize;
if N == 16 {
y = avx2_dct16_i16x4_coeff_rows_to_scratch::<IS_RECT2, 16>(
coeff, scratch, y, ncols, rnd, sh, minv, maxv,
);
} else {
y = avx2_dct32_i16x4_coeff_rows_to_scratch::<IS_RECT2, 32>(
coeff, scratch, y, ncols, rnd, sh, minv, maxv,
);
}
debug_assert_eq!(y, ncols);
crate::itx_2d::clear_i16_coeff_active_rows(coeff, N, ncols);
let mut x = 0usize;
while x + 16 <= N {
if N == 16 {
avx2_dct16_i16x16_scratch16_stride_eob_store::<16>(scratch, x, ncols, tmp);
} else {
avx2_dct32_i16x16_scratch16_stride_eob_store::<32>(scratch, x, ncols, tmp);
}
x += 16;
}
while x + 8 <= N {
if N == 16 {
avx2_dct16_i16x8_scratch8_stride_eob_store::<16>(scratch, x, ncols, tmp);
} else {
avx2_dct32_i16x8_scratch8_stride_eob_store::<32>(scratch, x, ncols, tmp);
}
x += 8;
}
while x < N {
if N == 16 {
avx2_dct16_i16x4_scratch4_stride_eob_store::<16>(scratch, x, ncols, tmp);
} else {
avx2_dct32_i16x4_scratch4_stride_eob_store::<32>(scratch, x, ncols, tmp);
}
x += 4;
}
});
}
#[inline]
#[target_feature(enable = "avx2")]
fn idct_dequant_dct_i16_avx2_fused_8bpc_impl_const<const N: usize, const IS_RECT2: bool>(
coeff: &mut [i16],
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
eob: i32,
tx: usize,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
shift1: i32,
) {
debug_assert!(N == 16 || N == 32);
debug_assert!(coeff.len() >= N * N);
let off = usize::from(crate::scan::LAST_EOB_PER_COL.offset[tx]);
let last_eob = &crate::scan::LAST_EOB_PER_COL.table[off..];
let mut ngrp = 0usize;
while ngrp < N / 4 {
ngrp += 1;
if eob <= last_eob[ngrp - 1] as i32 {
break;
}
}
let ncols = ngrp * 4;
let rnd = _mm_set1_epi32((1 << shift0) >> 1);
let sh = _mm_cvtsi32_si128(shift0);
let minv = _mm_set1_epi32(row_clip_min);
let maxv = _mm_set1_epi32(row_clip_max);
with_avx2_itx_i16_scratch(N * N, |scratch| {
let mut y = 0usize;
if N == 16 {
y = avx2_dct16_i16x4_coeff_rows_to_scratch::<IS_RECT2, 16>(
coeff, scratch, y, ncols, rnd, sh, minv, maxv,
);
} else {
y = avx2_dct32_i16x4_coeff_rows_to_scratch::<IS_RECT2, 32>(
coeff, scratch, y, ncols, rnd, sh, minv, maxv,
);
}
debug_assert_eq!(y, ncols);
crate::itx_2d::clear_i16_coeff_active_rows(coeff, N, ncols);
let rnd1 = _mm256_set1_epi32((1 << shift1) >> 1);
let sh1 = _mm_cvtsi32_si128(shift1);
let mut x = 0usize;
while x + 16 <= N {
if N == 16 {
avx2_dct16_i16x16_scratch16_stride_eob_add_u8::<16>(
scratch, x, ncols, dst, dst_off, dst_stride, 16, 16, rnd1, sh1,
);
} else {
avx2_dct32_i16x16_scratch16_stride_eob_add_u8::<32>(
scratch, x, ncols, dst, dst_off, dst_stride, 32, 32, rnd1, sh1,
);
}
x += 16;
}
while x + 8 <= N {
if N == 16 {
avx2_dct16_i16x8_scratch8_stride_eob_add_u8::<16>(
scratch, x, ncols, dst, dst_off, dst_stride, 16, 16, rnd1, sh1,
);
} else {
avx2_dct32_i16x8_scratch8_stride_eob_add_u8::<32>(
scratch, x, ncols, dst, dst_off, dst_stride, 32, 32, rnd1, sh1,
);
}
x += 8;
}
debug_assert_eq!(x, N);
});
}
#[inline]
#[target_feature(enable = "avx2")]
fn tx_dequant_dense_avx2_i32_impl<const W: usize, const H: usize>(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
if is_rect2 {
tx_dequant_dense_avx2_i32_impl_const::<W, H, true>(
coeff,
tmp,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
} else {
tx_dequant_dense_avx2_i32_impl_const::<W, H, false>(
coeff,
tmp,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn tx_dequant_4x4_avx2_i32_impl(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
_eob: i32,
_tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
if is_rect2 {
tx_dequant_4x4_avx2_i32_impl_const::<true>(
coeff,
tmp,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
} else {
tx_dequant_4x4_avx2_i32_impl_const::<false>(
coeff,
tmp,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
}
}
#[inline(never)]
#[target_feature(enable = "avx2")]
fn tx_dequant_4x4_avx2_i32_impl_const<const IS_RECT2: bool>(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
unsafe {
debug_assert!(coeff.len() >= 16);
let z = _mm_setzero_si128();
let rect_mul = _mm_set1_epi32(181);
let rect_rnd = _mm_set1_epi32(128);
let rnd = _mm_set1_epi32((1 << shift0) >> 1);
let sh = _mm_cvtsi32_si128(shift0);
let minv = _mm_set1_epi32(row_clip_min);
let maxv = _mm_set1_epi32(row_clip_max);
macro_rules! load_col {
($j:expr) => {{
let mut v = _mm_loadu_si128(coeff.as_ptr().add(($j) * 4).cast());
if IS_RECT2 {
v = _mm_srai_epi32::<8>(_mm_add_epi32(_mm_mullo_epi32(v, rect_mul), rect_rnd));
}
v
}};
}
let c0 = load_col!(0);
let c1 = load_col!(1);
let c2 = load_col!(2);
let c3 = load_col!(3);
macro_rules! row_pass {
($m:expr) => {{
let mut a = z;
a = _mm_add_epi32(
a,
_mm_mullo_epi32(
c0,
_mm_set1_epi32(avx2_tx_dense_coeff(first_kind, 4, $m, 0)),
),
);
a = _mm_add_epi32(
a,
_mm_mullo_epi32(
c1,
_mm_set1_epi32(avx2_tx_dense_coeff(first_kind, 4, $m, 1)),
),
);
a = _mm_add_epi32(
a,
_mm_mullo_epi32(
c2,
_mm_set1_epi32(avx2_tx_dense_coeff(first_kind, 4, $m, 2)),
),
);
a = _mm_add_epi32(
a,
_mm_mullo_epi32(
c3,
_mm_set1_epi32(avx2_tx_dense_coeff(first_kind, 4, $m, 3)),
),
);
a
}};
}
let row = [row_pass!(0), row_pass!(1), row_pass!(2), row_pass!(3)];
avx2_store4x4_i32_clip(tmp, 0, &row, rnd, sh, minv, maxv);
coeff[..16].fill(0);
let r0 = _mm_loadu_si128(tmp.as_ptr().cast());
let r1 = _mm_loadu_si128(tmp.as_ptr().add(32).cast());
let r2 = _mm_loadu_si128(tmp.as_ptr().add(64).cast());
let r3 = _mm_loadu_si128(tmp.as_ptr().add(96).cast());
macro_rules! col_pass {
($m:expr) => {{
let mut a = z;
a = _mm_add_epi32(
a,
_mm_mullo_epi32(
r0,
_mm_set1_epi32(avx2_tx_dense_coeff(second_kind, 4, $m, 0)),
),
);
a = _mm_add_epi32(
a,
_mm_mullo_epi32(
r1,
_mm_set1_epi32(avx2_tx_dense_coeff(second_kind, 4, $m, 1)),
),
);
a = _mm_add_epi32(
a,
_mm_mullo_epi32(
r2,
_mm_set1_epi32(avx2_tx_dense_coeff(second_kind, 4, $m, 2)),
),
);
a = _mm_add_epi32(
a,
_mm_mullo_epi32(
r3,
_mm_set1_epi32(avx2_tx_dense_coeff(second_kind, 4, $m, 3)),
),
);
a
}};
}
_mm_storeu_si128(tmp.as_mut_ptr().cast(), col_pass!(0));
_mm_storeu_si128(tmp.as_mut_ptr().add(32).cast(), col_pass!(1));
_mm_storeu_si128(tmp.as_mut_ptr().add(64).cast(), col_pass!(2));
_mm_storeu_si128(tmp.as_mut_ptr().add(96).cast(), col_pass!(3));
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn tx_dequant_dense_avx2_i32_impl_const<const W: usize, const H: usize, const IS_RECT2: bool>(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
unsafe {
debug_assert!(W == 4 || W == 8 || W == 16 || W == 32);
debug_assert!(H == 4 || H == 8 || H == 16 || H == 32);
debug_assert!(W * H <= coeff.len());
let off = usize::from(crate::scan::LAST_EOB_PER_COL.offset[tx]);
let last_eob = &crate::scan::LAST_EOB_PER_COL.table[off..];
let mut ngrp = 0usize;
while ngrp < H / 4 {
ngrp += 1;
if eob <= last_eob[ngrp - 1] as i32 {
break;
}
}
let nrows = ngrp * 4;
let z = _mm_setzero_si128();
let z256 = _mm256_setzero_si256();
let rect_mul = _mm_set1_epi32(181);
let rect_rnd = _mm_set1_epi32(128);
let rnd = _mm_set1_epi32((1 << shift0) >> 1);
let sh = _mm_cvtsi32_si128(shift0);
let minv = _mm_set1_epi32(row_clip_min);
let maxv = _mm_set1_epi32(row_clip_max);
let rnd256 = _mm256_set1_epi32((1 << shift0) >> 1);
let minv256 = _mm256_set1_epi32(row_clip_min);
let maxv256 = _mm256_set1_epi32(row_clip_max);
let mut y = 0usize;
if W >= 8 {
while y + 8 <= nrows {
let mut m = 0usize;
while m + 8 <= W {
let g = avx2_tx_dense_i32x8_from_coeff8_const::<IS_RECT2, W, H>(
coeff, y, first_kind, m,
);
avx2_store8x8_i32_clip(tmp, y * 32 + m, &g, rnd256, sh, minv256, maxv256);
m += 8;
}
y += 8;
}
}
while y + 4 <= nrows {
let mut m = 0usize;
while m < W {
let mut a0 = z;
let mut a1 = z;
let mut a2 = z;
let mut a3 = z;
let mut j = 0usize;
while j < W {
let mut v = _mm_loadu_si128(coeff.as_ptr().add(y + j * H).cast());
if IS_RECT2 {
v = _mm_srai_epi32::<8>(_mm_add_epi32(
_mm_mullo_epi32(v, rect_mul),
rect_rnd,
));
}
a0 = _mm_add_epi32(
a0,
_mm_mullo_epi32(
v,
_mm_set1_epi32(avx2_tx_dense_coeff(first_kind, W, m, j)),
),
);
a1 = _mm_add_epi32(
a1,
_mm_mullo_epi32(
v,
_mm_set1_epi32(avx2_tx_dense_coeff(first_kind, W, m + 1, j)),
),
);
a2 = _mm_add_epi32(
a2,
_mm_mullo_epi32(
v,
_mm_set1_epi32(avx2_tx_dense_coeff(first_kind, W, m + 2, j)),
),
);
a3 = _mm_add_epi32(
a3,
_mm_mullo_epi32(
v,
_mm_set1_epi32(avx2_tx_dense_coeff(first_kind, W, m + 3, j)),
),
);
j += 1;
}
let g = [a0, a1, a2, a3];
avx2_store4x4_i32_clip(tmp, y * 32 + m, &g, rnd, sh, minv, maxv);
m += 4;
}
y += 4;
}
while y < H {
tmp[y * 32..y * 32 + W].fill(0);
y += 1;
}
coeff[..W * H].fill(0);
let mut x = 0usize;
if H >= 8 {
while x + 8 <= W {
let mut vin = [z256; H];
let mut j = 0usize;
while j < H {
vin[j] = _mm256_loadu_si256(tmp.as_ptr().add(x + j * 32).cast());
j += 1;
}
let mut m = 0usize;
while m + 8 <= H {
let g = avx2_tx_dense_i32x8_from_tmp8::<H>(&vin, second_kind, m);
avx2_store_i32x8_rows(tmp, x + m * 32, &g);
m += 8;
}
x += 8;
}
}
while x < W {
let mut vin = [z; H];
{
let mut j = 0usize;
while j < H {
vin[j] = _mm_loadu_si128(tmp.as_ptr().add(x + j * 32).cast());
j += 1;
}
}
let mut m = 0usize;
while m < H {
let mut a0 = z;
let mut a1 = z;
let mut a2 = z;
let mut a3 = z;
let mut j = 0usize;
while j < H {
let v = vin[j];
a0 = _mm_add_epi32(
a0,
_mm_mullo_epi32(
v,
_mm_set1_epi32(avx2_tx_dense_coeff(second_kind, H, m, j)),
),
);
a1 = _mm_add_epi32(
a1,
_mm_mullo_epi32(
v,
_mm_set1_epi32(avx2_tx_dense_coeff(second_kind, H, m + 1, j)),
),
);
a2 = _mm_add_epi32(
a2,
_mm_mullo_epi32(
v,
_mm_set1_epi32(avx2_tx_dense_coeff(second_kind, H, m + 2, j)),
),
);
a3 = _mm_add_epi32(
a3,
_mm_mullo_epi32(
v,
_mm_set1_epi32(avx2_tx_dense_coeff(second_kind, H, m + 3, j)),
),
);
j += 1;
}
_mm_storeu_si128(tmp.as_mut_ptr().add(x + m * 32).cast(), a0);
_mm_storeu_si128(tmp.as_mut_ptr().add(x + (m + 1) * 32).cast(), a1);
_mm_storeu_si128(tmp.as_mut_ptr().add(x + (m + 2) * 32).cast(), a2);
_mm_storeu_si128(tmp.as_mut_ptr().add(x + (m + 3) * 32).cast(), a3);
m += 4;
}
x += 4;
}
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn tx_dequant_dense_avx2_i16_impl<const W: usize, const H: usize>(
coeff: &mut [i16],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
if is_rect2 {
tx_dequant_dense_avx2_i16_impl_const::<W, H, true>(
coeff,
tmp,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
} else {
tx_dequant_dense_avx2_i16_impl_const::<W, H, false>(
coeff,
tmp,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
}
}
#[target_feature(enable = "avx2")]
fn avx2_row_pass<const W: usize, const H: usize, const IS_RECT2: bool>(
coeff: &[i16],
scratch: &mut [i16],
nrows: usize,
first_kind: usize,
rnd: __m128i,
sh: __m128i,
minv: __m128i,
maxv: __m128i,
mut y: usize,
) -> usize {
let z = _mm_setzero_si128();
while y + 4 <= nrows {
let mut m = 0usize;
while m < W {
let mut a0 = z;
let mut a1 = z;
let mut a2 = z;
let mut a3 = z;
let mut j = 0usize;
while j < W {
let x0 = avx2_load4_i16_coeff_packed_const::<IS_RECT2>(coeff, y + j * H);
let x1 = avx2_load4_i16_coeff_packed_const::<IS_RECT2>(coeff, y + (j + 1) * H);
let x01 = _mm_unpacklo_epi16(x0, x1);
a0 = _mm_add_epi32(
a0,
_mm_madd_epi16(x01, avx2_tx_dense_coeff_pair(first_kind, W, m, j)),
);
a1 = _mm_add_epi32(
a1,
_mm_madd_epi16(x01, avx2_tx_dense_coeff_pair(first_kind, W, m + 1, j)),
);
a2 = _mm_add_epi32(
a2,
_mm_madd_epi16(x01, avx2_tx_dense_coeff_pair(first_kind, W, m + 2, j)),
);
a3 = _mm_add_epi32(
a3,
_mm_madd_epi16(x01, avx2_tx_dense_coeff_pair(first_kind, W, m + 3, j)),
);
j += 2;
}
avx2_store4x4_i16_clip::<W>(scratch, y * W + m, a0, a1, a2, a3, rnd, sh, minv, maxv);
m += 4;
}
y += 4;
}
y
}
#[target_feature(enable = "avx2")]
fn avx2_row_pass_const<
const W: usize,
const H: usize,
const IS_RECT2: bool,
const FIRST_KIND: usize,
>(
coeff: &[i16],
scratch: &mut [i16],
nrows: usize,
rnd: __m128i,
sh: __m128i,
minv: __m128i,
maxv: __m128i,
mut y: usize,
) -> usize {
let z = _mm_setzero_si128();
while y + 4 <= nrows {
let mut m = 0usize;
while m < W {
let mut a0 = z;
let mut a1 = z;
let mut a2 = z;
let mut a3 = z;
let mut j = 0usize;
while j < W {
let x0 = avx2_load4_i16_coeff_packed_const::<IS_RECT2>(coeff, y + j * H);
let x1 = avx2_load4_i16_coeff_packed_const::<IS_RECT2>(coeff, y + (j + 1) * H);
let x01 = _mm_unpacklo_epi16(x0, x1);
a0 = _mm_add_epi32(
a0,
_mm_madd_epi16(x01, avx2_tx_dense_coeff_pair(FIRST_KIND, W, m, j)),
);
a1 = _mm_add_epi32(
a1,
_mm_madd_epi16(x01, avx2_tx_dense_coeff_pair(FIRST_KIND, W, m + 1, j)),
);
a2 = _mm_add_epi32(
a2,
_mm_madd_epi16(x01, avx2_tx_dense_coeff_pair(FIRST_KIND, W, m + 2, j)),
);
a3 = _mm_add_epi32(
a3,
_mm_madd_epi16(x01, avx2_tx_dense_coeff_pair(FIRST_KIND, W, m + 3, j)),
);
j += 2;
}
avx2_store4x4_i16_clip::<W>(scratch, y * W + m, a0, a1, a2, a3, rnd, sh, minv, maxv);
m += 4;
}
y += 4;
}
y
}
#[target_feature(enable = "avx2")]
fn avx2_col_generic_store_x4<const W: usize, const H: usize>(
scratch: &[i16],
tmp: &mut [i32; ITX_TMP_PIXELS],
x: usize,
second_kind: usize,
) {
let z = _mm_setzero_si128();
let mut m = 0usize;
while m < H {
let mut a0 = z;
let mut a1 = z;
let mut a2 = z;
let mut a3 = z;
let mut j = 0usize;
while j < H {
let x0 = avx2_load4_i16_scratch(scratch, x + j * W);
let x1 = avx2_load4_i16_scratch(scratch, x + (j + 1) * W);
let x01 = _mm_unpacklo_epi16(x0, x1);
a0 = _mm_add_epi32(
a0,
_mm_madd_epi16(x01, avx2_tx_dense_coeff_pair(second_kind, H, m, j)),
);
a1 = _mm_add_epi32(
a1,
_mm_madd_epi16(x01, avx2_tx_dense_coeff_pair(second_kind, H, m + 1, j)),
);
a2 = _mm_add_epi32(
a2,
_mm_madd_epi16(x01, avx2_tx_dense_coeff_pair(second_kind, H, m + 2, j)),
);
a3 = _mm_add_epi32(
a3,
_mm_madd_epi16(x01, avx2_tx_dense_coeff_pair(second_kind, H, m + 3, j)),
);
j += 2;
}
unsafe {
_mm_storeu_si128(tmp.as_mut_ptr().add(x + m * 32).cast(), a0);
_mm_storeu_si128(tmp.as_mut_ptr().add(x + (m + 1) * 32).cast(), a1);
_mm_storeu_si128(tmp.as_mut_ptr().add(x + (m + 2) * 32).cast(), a2);
_mm_storeu_si128(tmp.as_mut_ptr().add(x + (m + 3) * 32).cast(), a3);
}
m += 4;
}
}
#[inline(never)]
#[target_feature(enable = "avx2")]
fn tx_dequant_dense_avx2_i16_impl_const<const W: usize, const H: usize, const IS_RECT2: bool>(
coeff: &mut [i16],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
debug_assert!(W == 4 || W == 8 || W == 16 || W == 32);
debug_assert!(H == 4 || H == 8 || H == 16 || H == 32);
debug_assert!(W * H <= coeff.len());
let off = usize::from(crate::scan::LAST_EOB_PER_COL.offset[tx]);
let last_eob = &crate::scan::LAST_EOB_PER_COL.table[off..];
let mut ngrp = 0usize;
while ngrp < H / 4 {
ngrp += 1;
if eob <= last_eob[ngrp - 1] as i32 {
break;
}
}
let nrows = ngrp * 4;
let rnd = _mm_set1_epi32((1 << shift0) >> 1);
let sh = _mm_cvtsi32_si128(shift0);
let minv = _mm_set1_epi32(row_clip_min);
let maxv = _mm_set1_epi32(row_clip_max);
with_avx2_itx_i16_scratch(W * H, |scratch| {
let mut y = 0usize;
if first_kind == crate::itx_2d::TX_KIND_DCT && W == 16 {
y = avx2_dct16_i16x4_coeff_rows_to_scratch::<IS_RECT2, H>(
coeff, scratch, y, nrows, rnd, sh, minv, maxv,
);
} else if first_kind == crate::itx_2d::TX_KIND_ADST && W == 16 {
y = avx2_tx16_dense_i16_coeff_rows_to_scratch::<
IS_RECT2,
H,
{ crate::itx_2d::TX_KIND_ADST },
>(coeff, scratch, y, nrows, rnd, sh, minv, maxv);
} else if first_kind == crate::itx_2d::TX_KIND_FLIPADST && W == 16 {
y = avx2_tx16_dense_i16_coeff_rows_to_scratch::<
IS_RECT2,
H,
{ crate::itx_2d::TX_KIND_FLIPADST },
>(coeff, scratch, y, nrows, rnd, sh, minv, maxv);
} else if first_kind == crate::itx_2d::TX_KIND_DCT && W == 32 {
y = avx2_dct32_i16x4_coeff_rows_to_scratch::<IS_RECT2, H>(
coeff, scratch, y, nrows, rnd, sh, minv, maxv,
);
}
avx2_row_pass::<W, H, IS_RECT2>(coeff, scratch, nrows, first_kind, rnd, sh, minv, maxv, y);
coeff[..W * H].fill(0);
let mut x = 0usize;
while x + 16 <= W && second_kind == crate::itx_2d::TX_KIND_DCT && (H == 16 || H == 32) {
if H == 16 {
avx2_dct16_i16x16_scratch16_stride_eob_store::<W>(scratch, x, nrows, tmp);
} else {
avx2_dct32_i16x16_scratch16_stride_eob_store::<W>(scratch, x, nrows, tmp);
}
x += 16;
}
while x + 8 <= W && second_kind == crate::itx_2d::TX_KIND_DCT && (H == 16 || H == 32) {
if H == 16 {
avx2_dct16_i16x8_scratch8_stride_eob_store::<W>(scratch, x, nrows, tmp);
} else {
avx2_dct32_i16x8_scratch8_stride_eob_store::<W>(scratch, x, nrows, tmp);
}
x += 8;
}
while x + 16 <= W && second_kind == crate::itx_2d::TX_KIND_ADST && H == 16 {
avx2_tx16_dense_i16x16_scratch_store_quads::<W, { crate::itx_2d::TX_KIND_ADST }>(
scratch, x, tmp,
);
x += 16;
}
while x + 16 <= W && second_kind == crate::itx_2d::TX_KIND_FLIPADST && H == 16 {
avx2_tx16_dense_i16x16_scratch_store_quads::<W, { crate::itx_2d::TX_KIND_FLIPADST }>(
scratch, x, tmp,
);
x += 16;
}
while x + 8 <= W && second_kind == crate::itx_2d::TX_KIND_ADST && H == 16 {
avx2_tx16_dense_i16x8_scratch_store_quads::<W, { crate::itx_2d::TX_KIND_ADST }>(
scratch, x, tmp,
);
x += 8;
}
while x + 8 <= W && second_kind == crate::itx_2d::TX_KIND_FLIPADST && H == 16 {
avx2_tx16_dense_i16x8_scratch_store_quads::<W, { crate::itx_2d::TX_KIND_FLIPADST }>(
scratch, x, tmp,
);
x += 8;
}
while x < W {
if second_kind == crate::itx_2d::TX_KIND_DCT && H == 16 {
avx2_dct16_i16x4_scratch4_stride_eob_store::<W>(scratch, x, nrows, tmp);
} else if second_kind == crate::itx_2d::TX_KIND_DCT && H == 32 {
avx2_dct32_i16x4_scratch4_stride_eob_store::<W>(scratch, x, nrows, tmp);
} else {
avx2_col_generic_store_x4::<W, H>(scratch, tmp, x, second_kind);
}
x += 4;
}
});
}
#[target_feature(enable = "avx2")]
fn avx2_col_generic_writeback_x4<const W: usize, const H: usize>(
scratch: &[i16],
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
x: usize,
second_kind: usize,
rnd1_4: __m128i,
sh1: __m128i,
) {
let z = _mm_setzero_si128();
let mut m = 0usize;
while m < H {
let mut a0 = z;
let mut a1 = z;
let mut a2 = z;
let mut a3 = z;
let mut j = 0usize;
while j < H {
let x0 = avx2_load4_i16_scratch(scratch, x + j * W);
let x1 = avx2_load4_i16_scratch(scratch, x + (j + 1) * W);
let x01 = _mm_unpacklo_epi16(x0, x1);
a0 = _mm_add_epi32(
a0,
_mm_madd_epi16(x01, avx2_tx_dense_coeff_pair(second_kind, H, m, j)),
);
a1 = _mm_add_epi32(
a1,
_mm_madd_epi16(x01, avx2_tx_dense_coeff_pair(second_kind, H, m + 1, j)),
);
a2 = _mm_add_epi32(
a2,
_mm_madd_epi16(x01, avx2_tx_dense_coeff_pair(second_kind, H, m + 2, j)),
);
a3 = _mm_add_epi32(
a3,
_mm_madd_epi16(x01, avx2_tx_dense_coeff_pair(second_kind, H, m + 3, j)),
);
j += 2;
}
avx2_writeback4_i32_u8::<W, H>(
dst, dst_off, dst_stride, out_w, out_h, x, m, a0, rnd1_4, sh1,
);
avx2_writeback4_i32_u8::<W, H>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
x,
m + 1,
a1,
rnd1_4,
sh1,
);
avx2_writeback4_i32_u8::<W, H>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
x,
m + 2,
a2,
rnd1_4,
sh1,
);
avx2_writeback4_i32_u8::<W, H>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
x,
m + 3,
a3,
rnd1_4,
sh1,
);
m += 4;
}
}
#[inline(never)]
#[target_feature(enable = "avx2")]
fn tx_dequant_dense_avx2_i16_fused_8bpc_impl_const<
const W: usize,
const H: usize,
const IS_RECT2: bool,
>(
coeff: &mut [i16],
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
eob: i32,
tx: usize,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
shift1: i32,
first_kind: usize,
second_kind: usize,
) {
debug_assert!(W == 4 || W == 8 || W == 16 || W == 32);
debug_assert!(H == 4 || H == 8 || H == 16 || H == 32);
debug_assert!(W * H <= coeff.len());
let off = usize::from(crate::scan::LAST_EOB_PER_COL.offset[tx]);
let last_eob = &crate::scan::LAST_EOB_PER_COL.table[off..];
let mut ngrp = 0usize;
while ngrp < H / 4 {
ngrp += 1;
if eob <= last_eob[ngrp - 1] as i32 {
break;
}
}
let nrows = ngrp * 4;
let rnd = _mm_set1_epi32((1 << shift0) >> 1);
let sh = _mm_cvtsi32_si128(shift0);
let minv = _mm_set1_epi32(row_clip_min);
let maxv = _mm_set1_epi32(row_clip_max);
with_avx2_itx_i16_scratch(W * H, |scratch| {
let mut y = 0usize;
if first_kind == crate::itx_2d::TX_KIND_IDENTITY {
y = fused_identity_pass::<W, H, IS_RECT2>(
coeff, nrows, rnd, sh, minv, maxv, scratch, y,
);
}
if first_kind == crate::itx_2d::TX_KIND_DCT && W == 16 {
y = avx2_dct16_i16x4_coeff_rows_to_scratch::<IS_RECT2, H>(
coeff, scratch, y, nrows, rnd, sh, minv, maxv,
);
} else if first_kind == crate::itx_2d::TX_KIND_ADST && W == 16 {
y = avx2_tx16_dense_i16_coeff_rows_to_scratch::<
IS_RECT2,
H,
{ crate::itx_2d::TX_KIND_ADST },
>(coeff, scratch, y, nrows, rnd, sh, minv, maxv);
} else if first_kind == crate::itx_2d::TX_KIND_FLIPADST && W == 16 {
y = avx2_tx16_dense_i16_coeff_rows_to_scratch::<
IS_RECT2,
H,
{ crate::itx_2d::TX_KIND_FLIPADST },
>(coeff, scratch, y, nrows, rnd, sh, minv, maxv);
} else if first_kind == crate::itx_2d::TX_KIND_DCT && W == 32 {
y = avx2_dct32_i16x4_coeff_rows_to_scratch::<IS_RECT2, H>(
coeff, scratch, y, nrows, rnd, sh, minv, maxv,
);
}
avx2_row_pass::<W, H, IS_RECT2>(coeff, scratch, nrows, first_kind, rnd, sh, minv, maxv, y);
coeff[..W * H].fill(0);
let rnd1_4 = _mm_set1_epi32((1 << shift1) >> 1);
let rnd1_8 = _mm256_set1_epi32((1 << shift1) >> 1);
let sh1 = _mm_cvtsi32_si128(shift1);
let mut x = 0usize;
if second_kind == crate::itx_2d::TX_KIND_IDENTITY {
x = fused_identity_second_pass::<W, H>(
scratch, dst, dst_off, dst_stride, out_w, out_h, rnd1_4, rnd1_8, sh1, x,
);
}
while x + 16 <= W && second_kind == crate::itx_2d::TX_KIND_DCT && (H == 16 || H == 32) {
if H == 16 {
avx2_dct16_i16x16_scratch16_stride_eob_add_u8::<W>(
scratch, x, nrows, dst, dst_off, dst_stride, out_w, out_h, rnd1_8, sh1,
);
} else {
avx2_dct32_i16x16_scratch16_stride_eob_add_u8::<W>(
scratch, x, nrows, dst, dst_off, dst_stride, out_w, out_h, rnd1_8, sh1,
);
}
x += 16;
}
while x + 8 <= W && second_kind == crate::itx_2d::TX_KIND_DCT && (H == 16 || H == 32) {
if H == 16 {
avx2_dct16_i16x8_scratch8_stride_eob_add_u8::<W>(
scratch, x, nrows, dst, dst_off, dst_stride, out_w, out_h, rnd1_8, sh1,
);
} else {
avx2_dct32_i16x8_scratch8_stride_eob_add_u8::<W>(
scratch, x, nrows, dst, dst_off, dst_stride, out_w, out_h, rnd1_8, sh1,
);
}
x += 8;
}
while x < W {
if second_kind == crate::itx_2d::TX_KIND_DCT && H == 16 {
avx2_dct16_i16x4_scratch4_stride_eob_add_u8::<W>(
scratch, x, nrows, dst, dst_off, dst_stride, out_w, out_h, rnd1_4, sh1,
);
} else if second_kind == crate::itx_2d::TX_KIND_DCT && H == 32 {
avx2_dct32_i16x4_scratch4_stride_eob_add_u8::<W>(
scratch, x, nrows, dst, dst_off, dst_stride, out_w, out_h, rnd1_4, sh1,
);
} else {
avx2_col_generic_writeback_x4::<W, H>(
scratch,
dst,
dst_off,
dst_stride,
out_w,
out_h,
x,
second_kind,
rnd1_4,
sh1,
);
}
x += 4;
}
});
}
#[inline]
#[target_feature(enable = "avx2")]
fn tx_dequant_dense_avx2_i16_fused_8bpc_hot_impl_const<
const W: usize,
const H: usize,
const IS_RECT2: bool,
const FIRST_KIND: usize,
const SECOND_KIND: usize,
>(
coeff: &mut [i16],
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
eob: i32,
tx: usize,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
shift1: i32,
) {
debug_assert!(W == 4 || W == 8 || W == 16 || W == 32);
debug_assert!(H == 4 || H == 8 || H == 16 || H == 32);
debug_assert!(W * H <= coeff.len());
let off = usize::from(crate::scan::LAST_EOB_PER_COL.offset[tx]);
let last_eob = &crate::scan::LAST_EOB_PER_COL.table[off..];
let mut ngrp = 0usize;
while ngrp < H / 4 {
ngrp += 1;
if eob <= last_eob[ngrp - 1] as i32 {
break;
}
}
let nrows = ngrp * 4;
let rnd = _mm_set1_epi32((1 << shift0) >> 1);
let sh = _mm_cvtsi32_si128(shift0);
let minv = _mm_set1_epi32(row_clip_min);
let maxv = _mm_set1_epi32(row_clip_max);
with_avx2_itx_i16_scratch(W * H, |scratch| {
let mut y = 0usize;
if FIRST_KIND == crate::itx_2d::TX_KIND_IDENTITY {
y = fused_identity_pass::<W, H, IS_RECT2>(
coeff, nrows, rnd, sh, minv, maxv, scratch, y,
);
}
#[allow(clippy::if_same_then_else)]
if FIRST_KIND == crate::itx_2d::TX_KIND_DCT && W == 16 {
y = avx2_dct16_i16x4_coeff_rows_to_scratch::<IS_RECT2, H>(
coeff, scratch, y, nrows, rnd, sh, minv, maxv,
);
} else if FIRST_KIND == crate::itx_2d::TX_KIND_ADST && W == 16 {
y = avx2_tx16_dense_i16_coeff_rows_to_scratch::<IS_RECT2, H, FIRST_KIND>(
coeff, scratch, y, nrows, rnd, sh, minv, maxv,
);
} else if FIRST_KIND == crate::itx_2d::TX_KIND_FLIPADST && W == 16 {
y = avx2_tx16_dense_i16_coeff_rows_to_scratch::<IS_RECT2, H, FIRST_KIND>(
coeff, scratch, y, nrows, rnd, sh, minv, maxv,
);
} else if FIRST_KIND == crate::itx_2d::TX_KIND_DCT && W == 32 {
y = avx2_dct32_i16x4_coeff_rows_to_scratch::<IS_RECT2, H>(
coeff, scratch, y, nrows, rnd, sh, minv, maxv,
);
}
avx2_row_pass_const::<W, H, IS_RECT2, FIRST_KIND>(
coeff, scratch, nrows, rnd, sh, minv, maxv, y,
);
coeff[..W * H].fill(0);
let rnd1_4 = _mm_set1_epi32((1 << shift1) >> 1);
let rnd1_8 = _mm256_set1_epi32((1 << shift1) >> 1);
let sh1 = _mm_cvtsi32_si128(shift1);
let mut x = 0usize;
if SECOND_KIND == crate::itx_2d::TX_KIND_IDENTITY {
x = fused_identity_second_pass::<W, H>(
scratch, dst, dst_off, dst_stride, out_w, out_h, rnd1_4, rnd1_8, sh1, x,
);
}
while x + 16 <= W && SECOND_KIND == crate::itx_2d::TX_KIND_DCT && (H == 16 || H == 32) {
if H == 16 {
avx2_dct16_i16x16_scratch16_stride_eob_add_u8::<W>(
scratch, x, nrows, dst, dst_off, dst_stride, out_w, out_h, rnd1_8, sh1,
);
} else {
avx2_dct32_i16x16_scratch16_stride_eob_add_u8::<W>(
scratch, x, nrows, dst, dst_off, dst_stride, out_w, out_h, rnd1_8, sh1,
);
}
x += 16;
}
while x + 8 <= W && SECOND_KIND == crate::itx_2d::TX_KIND_DCT && (H == 16 || H == 32) {
if H == 16 {
avx2_dct16_i16x8_scratch8_stride_eob_add_u8::<W>(
scratch, x, nrows, dst, dst_off, dst_stride, out_w, out_h, rnd1_8, sh1,
);
} else {
avx2_dct32_i16x8_scratch8_stride_eob_add_u8::<W>(
scratch, x, nrows, dst, dst_off, dst_stride, out_w, out_h, rnd1_8, sh1,
);
}
x += 8;
}
while x + 16 <= W && SECOND_KIND == crate::itx_2d::TX_KIND_ADST && H == 16 {
avx2_tx16_dense_i16x16_scratch_add_u8_quads::<W, SECOND_KIND>(
scratch, x, dst, dst_off, dst_stride, out_w, out_h, rnd1_8, sh1,
);
x += 16;
}
while x + 16 <= W && SECOND_KIND == crate::itx_2d::TX_KIND_FLIPADST && H == 16 {
avx2_tx16_dense_i16x16_scratch_add_u8_quads::<W, SECOND_KIND>(
scratch, x, dst, dst_off, dst_stride, out_w, out_h, rnd1_8, sh1,
);
x += 16;
}
while x + 8 <= W && SECOND_KIND == crate::itx_2d::TX_KIND_ADST && H == 16 {
avx2_tx16_dense_i16x8_scratch_add_u8_quads::<W, SECOND_KIND>(
scratch, x, dst, dst_off, dst_stride, out_w, out_h, rnd1_8, sh1,
);
x += 8;
}
while x + 8 <= W && SECOND_KIND == crate::itx_2d::TX_KIND_FLIPADST && H == 16 {
avx2_tx16_dense_i16x8_scratch_add_u8_quads::<W, SECOND_KIND>(
scratch, x, dst, dst_off, dst_stride, out_w, out_h, rnd1_8, sh1,
);
x += 8;
}
while x < W {
if SECOND_KIND == crate::itx_2d::TX_KIND_DCT && H == 16 {
avx2_dct16_i16x4_scratch4_stride_eob_add_u8::<W>(
scratch, x, nrows, dst, dst_off, dst_stride, out_w, out_h, rnd1_4, sh1,
);
} else if SECOND_KIND == crate::itx_2d::TX_KIND_DCT && H == 32 {
avx2_dct32_i16x4_scratch4_stride_eob_add_u8::<W>(
scratch, x, nrows, dst, dst_off, dst_stride, out_w, out_h, rnd1_4, sh1,
);
} else {
avx2_col_generic_writeback_x4::<W, H>(
scratch,
dst,
dst_off,
dst_stride,
out_w,
out_h,
x,
SECOND_KIND,
rnd1_4,
sh1,
);
}
x += 4;
}
});
}
#[inline(never)]
#[target_feature(enable = "avx2")]
fn fused_identity_pass<const W: usize, const H: usize, const IS_RECT2: bool>(
coeff: &mut [i16],
nrows: usize,
rnd: __m128i,
sh: __m128i,
minv: __m128i,
maxv: __m128i,
scratch: &mut [i16],
y: usize,
) -> usize {
let scale = _mm_set1_epi32(avx2_identity_scale(W));
let mut y = y;
while y + 4 <= nrows {
let mut m = 0usize;
while m < W {
let a0 = avx2_identity_i16x4_coeff_to_i32::<IS_RECT2>(coeff, y + (m + 0) * H, scale);
let a1 = avx2_identity_i16x4_coeff_to_i32::<IS_RECT2>(coeff, y + (m + 1) * H, scale);
let a2 = avx2_identity_i16x4_coeff_to_i32::<IS_RECT2>(coeff, y + (m + 2) * H, scale);
let a3 = avx2_identity_i16x4_coeff_to_i32::<IS_RECT2>(coeff, y + (m + 3) * H, scale);
avx2_store4x4_i16_clip::<W>(scratch, y * W + m, a0, a1, a2, a3, rnd, sh, minv, maxv);
m += 4;
}
y += 4;
}
y
}
#[inline(never)]
#[target_feature(enable = "avx2")]
fn fused_identity_second_pass<const W: usize, const H: usize>(
scratch: &[i16],
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
rnd1_4: __m128i,
rnd1_8: __m256i,
sh1: __m128i,
mut x: usize,
) -> usize {
let scale = _mm_set1_epi32(avx2_identity_scale(H));
while x + 8 <= W {
let mut m = 0usize;
while m < H {
let lo = avx2_identity_i16x4_scratch_to_i32(scratch, x + m * W, scale);
let hi = avx2_identity_i16x4_scratch_to_i32(scratch, x + 4 + m * W, scale);
let v = _mm256_set_m128i(hi, lo);
avx2_writeback8_i32_u8::<W, H>(
dst, dst_off, dst_stride, out_w, out_h, x, m, v, rnd1_8, sh1,
);
m += 1;
}
x += 8;
}
while x < W {
let mut m = 0usize;
while m < H {
let a = avx2_identity_i16x4_scratch_to_i32(scratch, x + m * W, scale);
avx2_writeback4_i32_u8::<W, H>(
dst, dst_off, dst_stride, out_w, out_h, x, m, a, rnd1_4, sh1,
);
m += 1;
}
x += 4;
}
x
}
#[inline]
#[target_feature(enable = "avx2")]
fn tx_dequant_dense_avx2_i16_fused_4x4_impl(
coeff: &mut [i16],
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
shift1: i32,
first_kind: usize,
second_kind: usize,
) {
if is_rect2 {
tx_dequant_dense_avx2_i16_fused_4x4_const::<true>(
coeff,
dst,
dst_off,
dst_stride,
out_w,
out_h,
shift0,
row_clip_min,
row_clip_max,
shift1,
first_kind,
second_kind,
)
} else {
tx_dequant_dense_avx2_i16_fused_4x4_const::<false>(
coeff,
dst,
dst_off,
dst_stride,
out_w,
out_h,
shift0,
row_clip_min,
row_clip_max,
shift1,
first_kind,
second_kind,
)
}
}
#[inline(never)]
#[target_feature(enable = "avx2")]
fn tx_dequant_dense_avx2_i16_fused_4x4_const<const IS_RECT2: bool>(
coeff: &mut [i16],
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
shift1: i32,
first_kind: usize,
second_kind: usize,
) {
debug_assert!(coeff.len() >= 16);
let z = _mm_setzero_si128();
let rnd = _mm_set1_epi32((1 << shift0) >> 1);
let sh = _mm_cvtsi32_si128(shift0);
let minv = _mm_set1_epi32(row_clip_min);
let maxv = _mm_set1_epi32(row_clip_max);
let c0 = avx2_load4_i16_coeff_packed_const::<IS_RECT2>(coeff, 0);
let c1 = avx2_load4_i16_coeff_packed_const::<IS_RECT2>(coeff, 4);
let c2 = avx2_load4_i16_coeff_packed_const::<IS_RECT2>(coeff, 8);
let c3 = avx2_load4_i16_coeff_packed_const::<IS_RECT2>(coeff, 12);
let c01 = _mm_unpacklo_epi16(c0, c1);
let c23 = _mm_unpacklo_epi16(c2, c3);
macro_rules! row_pass {
($m:expr) => {{
_mm_add_epi32(
_mm_madd_epi16(c01, avx2_tx_dense_coeff_pair(first_kind, 4, $m, 0)),
_mm_madd_epi16(c23, avx2_tx_dense_coeff_pair(first_kind, 4, $m, 2)),
)
}};
}
macro_rules! clip {
($x:expr) => {{
_mm_min_epi32(
_mm_max_epi32(_mm_sra_epi32(_mm_add_epi32($x, rnd), sh), minv),
maxv,
)
}};
}
let r0 = clip!(row_pass!(0));
let r1 = clip!(row_pass!(1));
let r2 = clip!(row_pass!(2));
let r3 = clip!(row_pass!(3));
let t0 = _mm_unpacklo_epi32(r0, r1);
let t1 = _mm_unpackhi_epi32(r0, r1);
let t2 = _mm_unpacklo_epi32(r2, r3);
let t3 = _mm_unpackhi_epi32(r2, r3);
let s0 = _mm_packs_epi32(_mm_unpacklo_epi64(t0, t2), z);
let s1 = _mm_packs_epi32(_mm_unpackhi_epi64(t0, t2), z);
let s2 = _mm_packs_epi32(_mm_unpacklo_epi64(t1, t3), z);
let s3 = _mm_packs_epi32(_mm_unpackhi_epi64(t1, t3), z);
let s01 = _mm_unpacklo_epi16(s0, s1);
let s23 = _mm_unpacklo_epi16(s2, s3);
macro_rules! col_pass {
($m:expr) => {{
_mm_add_epi32(
_mm_madd_epi16(s01, avx2_tx_dense_coeff_pair(second_kind, 4, $m, 0)),
_mm_madd_epi16(s23, avx2_tx_dense_coeff_pair(second_kind, 4, $m, 2)),
)
}};
}
let rnd1 = _mm_set1_epi32((1 << shift1) >> 1);
let sh1 = _mm_cvtsi32_si128(shift1);
avx2_writeback4_i32_u8::<4, 4>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
0,
0,
col_pass!(0),
rnd1,
sh1,
);
avx2_writeback4_i32_u8::<4, 4>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
0,
1,
col_pass!(1),
rnd1,
sh1,
);
avx2_writeback4_i32_u8::<4, 4>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
0,
2,
col_pass!(2),
rnd1,
sh1,
);
avx2_writeback4_i32_u8::<4, 4>(
dst,
dst_off,
dst_stride,
out_w,
out_h,
0,
3,
col_pass!(3),
rnd1,
sh1,
);
coeff[..16].fill(0);
}
#[inline]
#[target_feature(enable = "avx2")]
fn tx_dequant_dense_avx2_i16_fused_hot_square<const W: usize, const H: usize>(
coeff: &mut [i16],
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
eob: i32,
tx: usize,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
shift1: i32,
first_kind: usize,
second_kind: usize,
) -> bool {
debug_assert_eq!(W, H);
macro_rules! call_pair {
($first:expr, $second:expr) => {{
tx_dequant_dense_avx2_i16_fused_8bpc_hot_impl_const::<
W,
H,
false,
{ $first },
{ $second },
>(
coeff,
dst,
dst_off,
dst_stride,
out_w,
out_h,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
shift1,
);
true
}};
}
match (first_kind, second_kind) {
(crate::itx_2d::TX_KIND_DCT, crate::itx_2d::TX_KIND_DCT) => {
call_pair!(crate::itx_2d::TX_KIND_DCT, crate::itx_2d::TX_KIND_DCT)
}
(crate::itx_2d::TX_KIND_DCT, crate::itx_2d::TX_KIND_ADST) => {
call_pair!(crate::itx_2d::TX_KIND_DCT, crate::itx_2d::TX_KIND_ADST)
}
(crate::itx_2d::TX_KIND_ADST, crate::itx_2d::TX_KIND_DCT) => {
call_pair!(crate::itx_2d::TX_KIND_ADST, crate::itx_2d::TX_KIND_DCT)
}
(crate::itx_2d::TX_KIND_ADST, crate::itx_2d::TX_KIND_ADST) => {
call_pair!(crate::itx_2d::TX_KIND_ADST, crate::itx_2d::TX_KIND_ADST)
}
(crate::itx_2d::TX_KIND_DCT, crate::itx_2d::TX_KIND_FLIPADST) => {
call_pair!(crate::itx_2d::TX_KIND_DCT, crate::itx_2d::TX_KIND_FLIPADST)
}
(crate::itx_2d::TX_KIND_FLIPADST, crate::itx_2d::TX_KIND_DCT) => {
call_pair!(crate::itx_2d::TX_KIND_FLIPADST, crate::itx_2d::TX_KIND_DCT)
}
(crate::itx_2d::TX_KIND_ADST, crate::itx_2d::TX_KIND_FLIPADST) => {
call_pair!(crate::itx_2d::TX_KIND_ADST, crate::itx_2d::TX_KIND_FLIPADST)
}
(crate::itx_2d::TX_KIND_FLIPADST, crate::itx_2d::TX_KIND_ADST) => {
call_pair!(crate::itx_2d::TX_KIND_FLIPADST, crate::itx_2d::TX_KIND_ADST)
}
(crate::itx_2d::TX_KIND_FLIPADST, crate::itx_2d::TX_KIND_FLIPADST) => {
call_pair!(
crate::itx_2d::TX_KIND_FLIPADST,
crate::itx_2d::TX_KIND_FLIPADST
)
}
_ => false,
}
}
#[target_feature(enable = "avx2")]
fn tx_dequant_dense_avx2_i16_fused_8bpc_impl<const W: usize, const H: usize>(
coeff: &mut [i16],
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
shift1: i32,
first_kind: usize,
second_kind: usize,
) {
if is_rect2 {
tx_dequant_dense_avx2_i16_fused_8bpc_impl_const::<W, H, true>(
coeff,
dst,
dst_off,
dst_stride,
out_w,
out_h,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
shift1,
first_kind,
second_kind,
)
} else {
tx_dequant_dense_avx2_i16_fused_8bpc_impl_const::<W, H, false>(
coeff,
dst,
dst_off,
dst_stride,
out_w,
out_h,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
shift1,
first_kind,
second_kind,
)
}
}
macro_rules! avx2_fused_match_body {
($call:ident, $coeff:ident, $dst:ident, $dst_off:ident, $dst_stride:ident, $out_w:ident, $out_h:ident, $eob:ident, $tx:ident, $is_rect2:ident, $shift0:ident, $row_clip_min:ident, $row_clip_max:ident, $shift1:ident, $first_kind:ident, $second_kind:ident) => {{
match $tx {
crate::levels::txsz::TX_8X8 => $call::<8, 8>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::TX_16X16 => $call::<16, 16>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::TX_32X32 => $call::<32, 32>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::TX_64X64 => $call::<32, 32>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_4X8 => $call::<4, 8>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_8X4 => $call::<8, 4>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_8X16 => $call::<8, 16>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_16X8 => $call::<16, 8>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_16X32 => $call::<16, 32>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_32X16 => $call::<32, 16>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_32X64 => $call::<32, 32>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_64X32 => $call::<32, 32>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_4X16 => $call::<4, 16>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_16X4 => $call::<16, 4>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_8X32 => $call::<8, 32>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_32X8 => $call::<32, 8>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_16X64 => $call::<16, 32>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_64X16 => $call::<32, 16>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_4X32 => $call::<4, 32>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_32X4 => $call::<32, 4>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_8X64 => $call::<8, 32>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_64X8 => $call::<32, 8>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_4X64 => $call::<4, 32>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
crate::levels::txsz::RTX_64X4 => $call::<32, 4>(
$coeff,
$dst,
$dst_off,
$dst_stride,
$out_w,
$out_h,
$eob,
$tx,
$is_rect2,
$shift0,
$row_clip_min,
$row_clip_max,
$shift1,
$first_kind,
$second_kind,
),
_ => return false,
}
true
}};
}
#[target_feature(enable = "avx2")]
pub(crate) fn itx_dequant_i16_avx2_fused_8bpc(
coeff: &mut [i16],
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
out_w: usize,
out_h: usize,
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
shift1: i32,
first_kind: usize,
second_kind: usize,
) -> bool {
if tx == crate::levels::txsz::TX_4X4 {
tx_dequant_dense_avx2_i16_fused_4x4_impl(
coeff,
dst,
dst_off,
dst_stride,
out_w,
out_h,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
shift1,
first_kind,
second_kind,
);
return true;
}
if !is_rect2 {
let handled_hot = match tx {
crate::levels::txsz::TX_8X8 => tx_dequant_dense_avx2_i16_fused_hot_square::<8, 8>(
coeff,
dst,
dst_off,
dst_stride,
out_w,
out_h,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
shift1,
first_kind,
second_kind,
),
crate::levels::txsz::TX_16X16 => tx_dequant_dense_avx2_i16_fused_hot_square::<16, 16>(
coeff,
dst,
dst_off,
dst_stride,
out_w,
out_h,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
shift1,
first_kind,
second_kind,
),
_ => false,
};
if handled_hot {
return true;
}
}
avx2_fused_match_body!(
tx_dequant_dense_avx2_i16_fused_8bpc_impl,
coeff,
dst,
dst_off,
dst_stride,
out_w,
out_h,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
shift1,
first_kind,
second_kind
)
}
#[inline]
#[target_feature(enable = "avx2")]
fn tx_dequant_8x8_avx2_i32_impl(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
if is_rect2 {
tx_dequant_8x8_avx2_i32_impl_const::<true>(
coeff,
tmp,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
} else {
tx_dequant_8x8_avx2_i32_impl_const::<false>(
coeff,
tmp,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn tx_dequant_8x8_avx2_i32_impl_const<const IS_RECT2: bool>(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
debug_assert!(coeff.len() >= 64);
let off = usize::from(crate::scan::LAST_EOB_PER_COL.offset[tx]);
let last_eob = &crate::scan::LAST_EOB_PER_COL.table[off..];
let mut ngrp = 0usize;
while ngrp < 2 {
ngrp += 1;
if eob <= last_eob[ngrp - 1] as i32 {
break;
}
}
let ncols = ngrp * 4;
let rnd = _mm_set1_epi32((1 << shift0) >> 1);
let sh = _mm_cvtsi32_si128(shift0);
let minv = _mm_set1_epi32(row_clip_min);
let maxv = _mm_set1_epi32(row_clip_max);
let rnd256 = _mm256_set1_epi32((1 << shift0) >> 1);
let minv256 = _mm256_set1_epi32(row_clip_min);
let maxv256 = _mm256_set1_epi32(row_clip_max);
let mut y = 0usize;
while y + 8 <= ncols {
let out = avx2_tx8_i32x8_from_coeff8_const::<IS_RECT2>(coeff, y, first_kind);
avx2_store8x8_i32_clip(tmp, y * 32, &out, rnd256, sh, minv256, maxv256);
y += 8;
}
if y + 4 <= ncols {
let mut x = 0usize;
while x < 8 {
let g = avx2_tx8_i32x4_from_coeff4_const::<IS_RECT2>(coeff, y, first_kind, x);
avx2_store4x4_i32_clip(tmp, y * 32 + x, &g, rnd, sh, minv, maxv);
x += 4;
}
y += 4;
}
while y < 8 {
tmp[y * 32..y * 32 + 8].fill(0);
y += 1;
}
coeff[..64].fill(0);
let out = avx2_tx8_i32x8_from_tmp8(tmp, 0, second_kind);
avx2_store_i32x8_rows(tmp, 0, &out);
}
#[inline]
#[target_feature(enable = "avx2")]
fn idct_dequant_16x16_avx2_i32_impl(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
if is_rect2 {
idct_dequant_16x16_avx2_i32_impl_const::<true>(
coeff,
tmp,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
)
} else {
idct_dequant_16x16_avx2_i32_impl_const::<false>(
coeff,
tmp,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
)
}
}
#[inline]
#[target_feature(enable = "avx2")]
fn idct_dequant_16x16_avx2_i32_impl_const<const IS_RECT2: bool>(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
debug_assert!(coeff.len() >= 256);
let off = usize::from(crate::scan::LAST_EOB_PER_COL.offset[tx]);
let last_eob = &crate::scan::LAST_EOB_PER_COL.table[off..];
let mut ngrp = 0usize;
while ngrp < 4 {
ngrp += 1;
if eob <= last_eob[ngrp - 1] as i32 {
break;
}
}
let ncols = ngrp * 4;
let rnd = _mm_set1_epi32((1 << shift0) >> 1);
let sh = _mm_cvtsi32_si128(shift0);
let minv = _mm_set1_epi32(row_clip_min);
let maxv = _mm_set1_epi32(row_clip_max);
let rnd256 = _mm256_set1_epi32((1 << shift0) >> 1);
let minv256 = _mm256_set1_epi32(row_clip_min);
let maxv256 = _mm256_set1_epi32(row_clip_max);
let mut y = 0usize;
while y + 8 <= ncols {
let out =
avx2_tx16_i32x8_from_coeff8_const::<IS_RECT2>(coeff, y, 16, crate::itx_2d::TX_KIND_DCT);
avx2_store8x8_i32_clip(
tmp,
y * 32,
array_ref8_i32x8(&out, 0),
rnd256,
sh,
minv256,
maxv256,
);
avx2_store8x8_i32_clip(
tmp,
y * 32 + 8,
array_ref8_i32x8(&out, 8),
rnd256,
sh,
minv256,
maxv256,
);
y += 8;
}
if y + 4 <= ncols {
let out =
avx2_tx16_i32x4_from_coeff4_const::<IS_RECT2>(coeff, y, 16, crate::itx_2d::TX_KIND_DCT);
let mut x = 0usize;
while x < 16 {
let g = [out[x], out[x + 1], out[x + 2], out[x + 3]];
avx2_store4x4_i32_clip(tmp, y * 32 + x, &g, rnd, sh, minv, maxv);
x += 4;
}
y += 4;
}
while y < 16 {
tmp[y * 32..y * 32 + 16].fill(0);
y += 1;
}
coeff[..256].fill(0);
let mut x = 0usize;
while x + 8 <= 16 {
let out = avx2_tx16_i32x8_from_tmp8(tmp, x, crate::itx_2d::TX_KIND_DCT);
let mut m = 0usize;
while m < 16 {
unsafe {
_mm256_storeu_si256(tmp.as_mut_ptr().add(x + m * 32).cast(), out[m]);
}
m += 1;
}
x += 8;
}
}
#[target_feature(enable = "avx2")]
fn idct_dequant_32x32_avx2_i32_impl(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
if is_rect2 {
idct_dequant_32x32_avx2_i32_impl_const::<true>(
coeff,
tmp,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
)
} else {
idct_dequant_32x32_avx2_i32_impl_const::<false>(
coeff,
tmp,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
)
}
}
#[target_feature(enable = "avx2")]
fn idct_dequant_32x32_avx2_i32_impl_const<const IS_RECT2: bool>(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
debug_assert!(coeff.len() >= 1024);
let off = usize::from(crate::scan::LAST_EOB_PER_COL.offset[tx]);
let last_eob = &crate::scan::LAST_EOB_PER_COL.table[off..];
let mut ngrp = 0usize;
while ngrp < 8 {
ngrp += 1;
if eob <= last_eob[ngrp - 1] as i32 {
break;
}
}
let ncols = ngrp * 4;
let rnd = _mm_set1_epi32((1 << shift0) >> 1);
let sh = _mm_cvtsi32_si128(shift0);
let minv = _mm_set1_epi32(row_clip_min);
let maxv = _mm_set1_epi32(row_clip_max);
let rnd256 = _mm256_set1_epi32((1 << shift0) >> 1);
let minv256 = _mm256_set1_epi32(row_clip_min);
let maxv256 = _mm256_set1_epi32(row_clip_max);
let mut y = 0usize;
while y + 8 <= ncols {
let mut x = 0usize;
while x < 32 {
let g = avx2_dct32_i32x8_from_coeff8_const::<IS_RECT2>(coeff, y, x);
avx2_store8x8_i32_clip(tmp, y * 32 + x, &g, rnd256, sh, minv256, maxv256);
x += 8;
}
y += 8;
}
if y + 4 <= ncols {
let mut x = 0usize;
while x < 32 {
let g = avx2_dct32_i32x4_from_coeff4_const::<IS_RECT2>(coeff, y, x);
avx2_store4x4_i32_clip(tmp, y * 32 + x, &g, rnd, sh, minv, maxv);
x += 4;
}
y += 4;
}
while y < 32 {
tmp[y * 32..y * 32 + 32].fill(0);
y += 1;
}
coeff[..1024].fill(0);
let mut x = 0usize;
while x + 8 <= 32 {
let mut stage = core::mem::MaybeUninit::<[i32; 32 * 8]>::uninit();
let stage_ptr = stage.as_mut_ptr().cast::<i32>();
let mut m = 0usize;
while m < 32 {
let g = avx2_dct32_i32x8_from_tmp8(tmp, x, m);
let mut lane = 0usize;
while lane < 8 {
unsafe {
_mm256_storeu_si256(stage_ptr.add((m + lane) * 8).cast(), g[lane]);
}
lane += 1;
}
m += 8;
}
let mut m = 0usize;
while m < 32 {
unsafe {
let row = _mm256_loadu_si256(stage_ptr.add(m * 8).cast());
_mm256_storeu_si256(tmp.as_mut_ptr().add(x + m * 32).cast(), row);
}
m += 1;
}
x += 8;
}
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_4x4_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
tx_dequant_4x4_avx2_i32_impl(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
crate::itx_2d::TX_KIND_DCT,
crate::itx_2d::TX_KIND_DCT,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_8x8_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
tx_dequant_8x8_avx2_i32_impl(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
crate::itx_2d::TX_KIND_DCT,
crate::itx_2d::TX_KIND_DCT,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_16x16_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
idct_dequant_16x16_avx2_i32_impl(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_32x32_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
idct_dequant_32x32_avx2_i32_impl(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_64x64_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
tx_dequant_dense_avx2_i32_impl::<32, 32>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
crate::itx_2d::TX_KIND_DCT,
crate::itx_2d::TX_KIND_DCT,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn iadst_dequant_4x4_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
tx_dequant_4x4_avx2_i32_impl(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn iadst_dequant_8x8_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
tx_dequant_8x8_avx2_i32_impl(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn iadst_dequant_16x16_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
iadst_dequant_16x16_avx2_i32_impl(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_4x8_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
tx_dequant_dense_avx2_i32_impl::<4, 8>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
crate::itx_2d::TX_KIND_DCT,
crate::itx_2d::TX_KIND_DCT,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_8x4_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
tx_dequant_dense_avx2_i32_impl::<8, 4>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
crate::itx_2d::TX_KIND_DCT,
crate::itx_2d::TX_KIND_DCT,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_8x16_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
tx_dequant_dense_avx2_i32_impl::<8, 16>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
crate::itx_2d::TX_KIND_DCT,
crate::itx_2d::TX_KIND_DCT,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_16x8_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
tx_dequant_dense_avx2_i32_impl::<16, 8>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
crate::itx_2d::TX_KIND_DCT,
crate::itx_2d::TX_KIND_DCT,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_16x32_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
tx_dequant_dense_avx2_i32_impl::<16, 32>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
crate::itx_2d::TX_KIND_DCT,
crate::itx_2d::TX_KIND_DCT,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_32x16_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
tx_dequant_dense_avx2_i32_impl::<32, 16>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
crate::itx_2d::TX_KIND_DCT,
crate::itx_2d::TX_KIND_DCT,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_4x16_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
tx_dequant_dense_avx2_i32_impl::<4, 16>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
crate::itx_2d::TX_KIND_DCT,
crate::itx_2d::TX_KIND_DCT,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_16x4_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
tx_dequant_dense_avx2_i32_impl::<16, 4>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
crate::itx_2d::TX_KIND_DCT,
crate::itx_2d::TX_KIND_DCT,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_8x32_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
tx_dequant_dense_avx2_i32_impl::<8, 32>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
crate::itx_2d::TX_KIND_DCT,
crate::itx_2d::TX_KIND_DCT,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_32x8_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
tx_dequant_dense_avx2_i32_impl::<32, 8>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
crate::itx_2d::TX_KIND_DCT,
crate::itx_2d::TX_KIND_DCT,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_4x32_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
tx_dequant_dense_avx2_i32_impl::<4, 32>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
crate::itx_2d::TX_KIND_DCT,
crate::itx_2d::TX_KIND_DCT,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_32x4_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
tx_dequant_dense_avx2_i32_impl::<32, 4>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
crate::itx_2d::TX_KIND_DCT,
crate::itx_2d::TX_KIND_DCT,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn iadst_dequant_4x8_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
tx_dequant_dense_avx2_i32_impl::<4, 8>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn iadst_dequant_8x4_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
tx_dequant_dense_avx2_i32_impl::<8, 4>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn iadst_dequant_8x16_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
tx_dequant_dense_avx2_i32_impl::<8, 16>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn iadst_dequant_16x8_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
tx_dequant_dense_avx2_i32_impl::<16, 8>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn iadst_dequant_4x16_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
tx_dequant_dense_avx2_i32_impl::<4, 16>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn iadst_dequant_16x4_avx2(
coeff: &mut [i32],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
tx_dequant_dense_avx2_i32_impl::<16, 4>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
}
macro_rules! idct_i16_fn {
($pub_fn:ident, $imp:ident, $s:expr) => {
#[target_feature(enable = "avx2")]
pub(crate) fn $pub_fn(
coeff: &mut [i16],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
tx_dequant_dense_avx2_i16_impl::<{ $s }, { $s }>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
crate::itx_2d::TX_KIND_DCT,
crate::itx_2d::TX_KIND_DCT,
)
}
};
}
macro_rules! iadst_i16_fn {
($pub_fn:ident, $imp:ident, $s:expr) => {
#[target_feature(enable = "avx2")]
pub(crate) fn $pub_fn(
coeff: &mut [i16],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
tx_dequant_dense_avx2_i16_impl::<{ $s }, { $s }>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
}
};
}
macro_rules! idct_rect_i16_fn {
($pub_fn:ident, $imp:ident, $w:expr, $h:expr) => {
#[target_feature(enable = "avx2")]
pub(crate) fn $pub_fn(
coeff: &mut [i16],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
tx_dequant_dense_avx2_i16_impl::<{ $w }, { $h }>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
crate::itx_2d::TX_KIND_DCT,
crate::itx_2d::TX_KIND_DCT,
)
}
};
}
macro_rules! iadst_rect_i16_fn {
($pub_fn:ident, $imp:ident, $w:expr, $h:expr) => {
#[target_feature(enable = "avx2")]
pub(crate) fn $pub_fn(
coeff: &mut [i16],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
tx_dequant_dense_avx2_i16_impl::<{ $w }, { $h }>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
}
};
}
idct_i16_fn!(idct_dequant_4x4_i16_avx2, idct_dequant_4x4_i16_avx2_impl, 4);
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_8x8_i16_avx2(
coeff: &mut [i16],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
tx_dequant_dense_avx2_i16_impl::<8, 8>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
crate::itx_2d::TX_KIND_DCT,
crate::itx_2d::TX_KIND_DCT,
)
}
#[inline]
#[target_feature(enable = "avx2")]
fn idct_dequant_dct_i16_avx2_fused_8bpc_impl<const N: usize>(
coeff: &mut [i16],
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
shift1: i32,
) {
if is_rect2 {
idct_dequant_dct_i16_avx2_fused_8bpc_impl_const::<N, true>(
coeff,
dst,
dst_off,
dst_stride,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
shift1,
)
} else {
idct_dequant_dct_i16_avx2_fused_8bpc_impl_const::<N, false>(
coeff,
dst,
dst_off,
dst_stride,
eob,
tx,
shift0,
row_clip_min,
row_clip_max,
shift1,
)
}
}
#[target_feature(enable = "avx2")]
fn idct_dequant_32x32_i16_avx2_fixed_fused_8bpc_impl(
coeff: &mut [i16],
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
shift1: i32,
) {
idct_dequant_dct_i16_avx2_fused_8bpc_impl::<32>(
coeff,
dst,
dst_off,
dst_stride,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
shift1,
)
}
#[target_feature(enable = "avx2")]
fn idct_dequant_32x32_i16_avx2_fixed_impl(
coeff: &mut [i16],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
idct_dequant_dct_i16_avx2_impl::<32>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_16x16_i16_avx2_fused_8bpc(
coeff: &mut [i16],
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
shift1: i32,
) {
idct_dequant_dct_i16_avx2_fused_8bpc_impl::<16>(
coeff,
dst,
dst_off,
dst_stride,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
shift1,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_32x32_i16_avx2_fused_8bpc(
coeff: &mut [i16],
dst: &mut [u8],
dst_off: usize,
dst_stride: usize,
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
shift1: i32,
) {
idct_dequant_32x32_i16_avx2_fixed_fused_8bpc_impl(
coeff,
dst,
dst_off,
dst_stride,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
shift1,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_16x16_i16_avx2(
coeff: &mut [i16],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
idct_dequant_dct_i16_avx2_impl::<16>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn idct_dequant_32x32_i16_avx2(
coeff: &mut [i16],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
) {
idct_dequant_32x32_i16_avx2_fixed_impl(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
)
}
idct_i16_fn!(
idct_dequant_64x64_i16_avx2,
idct_dequant_64x64_i16_avx2_impl,
32
);
iadst_i16_fn!(
iadst_dequant_4x4_i16_avx2,
iadst_dequant_4x4_i16_avx2_impl,
4
);
#[target_feature(enable = "avx2")]
pub(crate) fn iadst_dequant_8x8_i16_avx2(
coeff: &mut [i16],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
tx_dequant_dense_avx2_i16_impl::<8, 8>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
}
#[target_feature(enable = "avx2")]
pub(crate) fn iadst_dequant_16x16_i16_avx2(
coeff: &mut [i16],
tmp: &mut [i32; ITX_TMP_PIXELS],
eob: i32,
tx: usize,
is_rect2: bool,
shift0: i32,
row_clip_min: i32,
row_clip_max: i32,
first_kind: usize,
second_kind: usize,
) {
tx_dequant_dense_avx2_i16_impl::<16, 16>(
coeff,
tmp,
eob,
tx,
is_rect2,
shift0,
row_clip_min,
row_clip_max,
first_kind,
second_kind,
)
}
idct_rect_i16_fn!(
idct_dequant_4x8_i16_avx2,
idct_dequant_4x8_i16_avx2_impl,
4,
8
);
idct_rect_i16_fn!(
idct_dequant_8x4_i16_avx2,
idct_dequant_8x4_i16_avx2_impl,
8,
4
);
idct_rect_i16_fn!(
idct_dequant_8x16_i16_avx2,
idct_dequant_8x16_i16_avx2_impl,
8,
16
);
idct_rect_i16_fn!(
idct_dequant_16x8_i16_avx2,
idct_dequant_16x8_i16_avx2_impl,
16,
8
);
idct_rect_i16_fn!(
idct_dequant_16x32_i16_avx2,
idct_dequant_16x32_i16_avx2_impl,
16,
32
);
idct_rect_i16_fn!(
idct_dequant_32x16_i16_avx2,
idct_dequant_32x16_i16_avx2_impl,
32,
16
);
idct_rect_i16_fn!(
idct_dequant_4x16_i16_avx2,
idct_dequant_4x16_i16_avx2_impl,
4,
16
);
idct_rect_i16_fn!(
idct_dequant_16x4_i16_avx2,
idct_dequant_16x4_i16_avx2_impl,
16,
4
);
idct_rect_i16_fn!(
idct_dequant_8x32_i16_avx2,
idct_dequant_8x32_i16_avx2_impl,
8,
32
);
idct_rect_i16_fn!(
idct_dequant_32x8_i16_avx2,
idct_dequant_32x8_i16_avx2_impl,
32,
8
);
idct_rect_i16_fn!(
idct_dequant_4x32_i16_avx2,
idct_dequant_4x32_i16_avx2_impl,
4,
32
);
idct_rect_i16_fn!(
idct_dequant_32x4_i16_avx2,
idct_dequant_32x4_i16_avx2_impl,
32,
4
);
iadst_rect_i16_fn!(
iadst_dequant_4x8_i16_avx2,
iadst_dequant_4x8_i16_avx2_impl,
4,
8
);
iadst_rect_i16_fn!(
iadst_dequant_8x4_i16_avx2,
iadst_dequant_8x4_i16_avx2_impl,
8,
4
);
iadst_rect_i16_fn!(
iadst_dequant_8x16_i16_avx2,
iadst_dequant_8x16_i16_avx2_impl,
8,
16
);
iadst_rect_i16_fn!(
iadst_dequant_16x8_i16_avx2,
iadst_dequant_16x8_i16_avx2_impl,
16,
8
);
iadst_rect_i16_fn!(
iadst_dequant_4x16_i16_avx2,
iadst_dequant_4x16_i16_avx2_impl,
4,
16
);
iadst_rect_i16_fn!(
iadst_dequant_16x4_i16_avx2,
iadst_dequant_16x4_i16_avx2_impl,
16,
4
);