#![cfg_attr(not(feature = "std"), allow(dead_code))]
#![cfg_attr(not(any(feature = "yuv-planar", feature = "rgb")), allow(dead_code))]
#[cfg_attr(miri, allow(unused_imports))]
use core::arch::x86_64::*;
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
fn hsum128_u32(acc: __m128i) -> u32 {
let hi = _mm_shuffle_epi32::<0b01_00_11_10>(acc);
let s = _mm_add_epi32(acc, hi);
let s2 = _mm_add_epi32(s, _mm_shuffle_epi32::<0b00_00_00_01>(s));
_mm_cvtsi128_si32(s2) as u32
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
fn mac128_u16x8(acc: __m128i, s16: __m128i, w: __m128i) -> __m128i {
let lo = _mm_mullo_epi16(s16, w);
let hi = _mm_mulhi_epu16(s16, w);
let acc = _mm_add_epi32(acc, _mm_unpacklo_epi16(lo, hi));
_mm_add_epi32(acc, _mm_unpackhi_epi16(lo, hi))
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
fn mac512_u16x32(acc: __m512i, s16: __m512i, w: __m512i) -> __m512i {
let lo = _mm512_mullo_epi16(s16, w);
let hi = _mm512_mulhi_epu16(s16, w);
let acc = _mm512_add_epi32(acc, _mm512_unpacklo_epi16(lo, hi));
_mm512_add_epi32(acc, _mm512_unpackhi_epi16(lo, hi))
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
unsafe fn c1_group(acc: __m128i, row: &[u8], base: usize, w8: &[u16]) -> __m128i {
unsafe {
let s8 = if base + 8 <= row.len() {
_mm_loadl_epi64(row.as_ptr().add(base).cast())
} else {
let mut sbuf = [0u8; 8];
let take = row.len() - base;
sbuf[..take].copy_from_slice(&row[base..]);
_mm_loadl_epi64(sbuf.as_ptr().cast())
};
let w = _mm_loadu_si128(w8.as_ptr().cast());
mac128_u16x8(acc, _mm_cvtepu8_epi16(s8), w)
}
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
pub(crate) unsafe fn area_h_reduce_row_c1(
row: &[u8],
starts: &[usize],
w16: &[u16],
w16_off: &[usize],
h_tmp: &mut [u32],
) {
for (j, &start) in starts.iter().enumerate() {
let span = &w16[w16_off[j]..w16_off[j + 1]];
let len = span.len();
unsafe {
let mut acc = _mm512_setzero_si512();
let mut acc128 = _mm_setzero_si128();
let mut t = 0usize;
while t + 32 <= len && start + t + 32 <= row.len() {
let s8 = _mm256_loadu_si256(row.as_ptr().add(start + t).cast());
let s16 = _mm512_cvtepu8_epi16(s8);
let w = _mm512_loadu_si512(span[t..].as_ptr().cast());
acc = mac512_u16x32(acc, s16, w);
t += 32;
}
while t + 8 <= len {
acc128 = c1_group(acc128, row, start + t, &span[t..]);
t += 8;
}
let wide = _mm512_reduce_add_epi32(acc) as u32;
h_tmp[j] = wide.wrapping_add(hsum128_u32(acc128));
}
}
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
fn c3_masks() -> ([__m512i; 3], [__m512i; 3]) {
let m0 = [
_mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 15, 12, 9, 6, 3, 0),
_mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 13, 10, 7, 4, 1),
_mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 14, 11, 8, 5, 2),
];
let m1 = [
_mm_set_epi8(
-1, -1, -1, -1, -1, -1, -1, -1, 13, 10, -1, -1, -1, -1, -1, -1,
),
_mm_set_epi8(
-1, -1, -1, -1, -1, -1, -1, -1, 14, 11, 8, -1, -1, -1, -1, -1,
),
_mm_set_epi8(
-1, -1, -1, -1, -1, -1, -1, -1, 15, 12, 9, -1, -1, -1, -1, -1,
),
];
(
[
_mm512_broadcast_i32x4(m0[0]),
_mm512_broadcast_i32x4(m0[1]),
_mm512_broadcast_i32x4(m0[2]),
],
[
_mm512_broadcast_i32x4(m1[0]),
_mm512_broadcast_i32x4(m1[1]),
_mm512_broadcast_i32x4(m1[2]),
],
)
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
unsafe fn c3_group(
acc: &mut [__m128i; 3],
row: &[u8],
cell: usize,
w8: &[u16],
m0: &[__m512i; 3],
m1: &[__m512i; 3],
) {
let base = cell * 3;
unsafe {
let (v0, v1) = if base + 24 <= row.len() {
(
_mm_loadu_si128(row.as_ptr().add(base).cast()),
_mm_loadu_si128(row.as_ptr().add(base + 8).cast()),
)
} else {
let mut sbuf = [0u8; 24];
let take = row.len() - base;
sbuf[..take].copy_from_slice(&row[base..]);
(
_mm_loadu_si128(sbuf.as_ptr().cast()),
_mm_loadu_si128(sbuf.as_ptr().add(8).cast()),
)
};
let w = _mm_loadu_si128(w8.as_ptr().cast());
for ch in 0..3 {
let m0c = _mm512_castsi512_si128(m0[ch]);
let m1c = _mm512_castsi512_si128(m1[ch]);
let gathered = _mm_or_si128(_mm_shuffle_epi8(v0, m0c), _mm_shuffle_epi8(v1, m1c));
acc[ch] = mac128_u16x8(acc[ch], _mm_cvtepu8_epi16(gathered), w);
}
}
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
pub(crate) unsafe fn area_h_reduce_row_c3(
row: &[u8],
starts: &[usize],
w16: &[u16],
w16_off: &[usize],
h_tmp: &mut [u32],
) {
let (m0, m1) = c3_masks();
let zero = _mm512_setzero_si512();
for (j, &start) in starts.iter().enumerate() {
let span = &w16[w16_off[j]..w16_off[j + 1]];
let len = span.len();
unsafe {
let mut acc = [_mm512_setzero_si512(); 3];
let mut acc128 = [_mm_setzero_si128(); 3];
let mut t = 0usize;
while t + 32 <= len && (start + t + 32) * 3 <= row.len() {
let lo = |g: usize| _mm_loadu_si128(row.as_ptr().add((start + t + g * 8) * 3).cast());
let hi = |g: usize| _mm_loadu_si128(row.as_ptr().add((start + t + g * 8) * 3 + 8).cast());
let v0 = _mm512_inserti32x4::<3>(
_mm512_inserti32x4::<2>(
_mm512_inserti32x4::<1>(_mm512_castsi128_si512(lo(0)), lo(1)),
lo(2),
),
lo(3),
);
let v1 = _mm512_inserti32x4::<3>(
_mm512_inserti32x4::<2>(
_mm512_inserti32x4::<1>(_mm512_castsi128_si512(hi(0)), hi(1)),
hi(2),
),
hi(3),
);
let w = _mm512_loadu_si512(span[t..].as_ptr().cast());
for ch in 0..3 {
let gathered = _mm512_or_si512(
_mm512_shuffle_epi8(v0, m0[ch]),
_mm512_shuffle_epi8(v1, m1[ch]),
);
let s16 = _mm512_unpacklo_epi8(gathered, zero);
acc[ch] = mac512_u16x32(acc[ch], s16, w);
}
t += 32;
}
while t + 8 <= len {
c3_group(&mut acc128, row, start + t, &span[t..], &m0, &m1);
t += 8;
}
for ch in 0..3 {
let wide = _mm512_reduce_add_epi32(acc[ch]) as u32;
h_tmp[j * 3 + ch] = wide.wrapping_add(hsum128_u32(acc128[ch]));
}
}
}
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
pub(crate) unsafe fn area_v_accumulate(acc: &mut [u64], h_tmp: &[u32], w: u32) {
let n = acc.len();
debug_assert!(h_tmp.len() >= n, "h_tmp too short");
let wv = _mm512_set1_epi64(i64::from(w));
let idx_lo = _mm512_set_epi64(11, 3, 10, 2, 9, 1, 8, 0);
let idx_hi = _mm512_set_epi64(15, 7, 14, 6, 13, 5, 12, 4);
let mut i = 0usize;
unsafe {
while i + 16 <= n {
let t = _mm512_loadu_si512(h_tmp.as_ptr().add(i).cast());
let even = _mm512_mul_epu32(t, wv);
let odd = _mm512_mul_epu32(_mm512_srli_epi64::<32>(t), wv);
let lo = _mm512_permutex2var_epi64(even, idx_lo, odd);
let hi = _mm512_permutex2var_epi64(even, idx_hi, odd);
let a_lo = _mm512_loadu_si512(acc.as_ptr().add(i).cast());
let a_hi = _mm512_loadu_si512(acc.as_ptr().add(i + 8).cast());
_mm512_storeu_si512(acc.as_mut_ptr().add(i).cast(), _mm512_add_epi64(a_lo, lo));
_mm512_storeu_si512(
acc.as_mut_ptr().add(i + 8).cast(),
_mm512_add_epi64(a_hi, hi),
);
i += 16;
}
}
for k in i..n {
acc[k] += u64::from(w) * u64::from(h_tmp[k]);
}
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
fn hsum128_u64(acc: __m128i) -> u64 {
let hi = _mm_unpackhi_epi64(acc, acc);
_mm_cvtsi128_si64(_mm_add_epi64(acc, hi)) as u64
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
fn mac128_u16x8_u64(acc: __m128i, s16: __m128i, w: __m128i) -> __m128i {
let lo = _mm_mullo_epi16(s16, w);
let hi = _mm_mulhi_epu16(s16, w);
let p_lo = _mm_unpacklo_epi16(lo, hi);
let p_hi = _mm_unpackhi_epi16(lo, hi);
let acc = _mm_add_epi64(acc, _mm_cvtepu32_epi64(p_lo));
let acc = _mm_add_epi64(acc, _mm_cvtepu32_epi64(_mm_srli_si128::<8>(p_lo)));
let acc = _mm_add_epi64(acc, _mm_cvtepu32_epi64(p_hi));
_mm_add_epi64(acc, _mm_cvtepu32_epi64(_mm_srli_si128::<8>(p_hi)))
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
fn mac512_u16x32_u64(acc: __m512i, s16: __m512i, w: __m512i) -> __m512i {
let lo = _mm512_mullo_epi16(s16, w);
let hi = _mm512_mulhi_epu16(s16, w);
let p_lo = _mm512_unpacklo_epi16(lo, hi);
let p_hi = _mm512_unpackhi_epi16(lo, hi);
let acc = _mm512_add_epi64(acc, _mm512_cvtepu32_epi64(_mm512_castsi512_si256(p_lo)));
let acc = _mm512_add_epi64(
acc,
_mm512_cvtepu32_epi64(_mm512_extracti64x4_epi64::<1>(p_lo)),
);
let acc = _mm512_add_epi64(acc, _mm512_cvtepu32_epi64(_mm512_castsi512_si256(p_hi)));
_mm512_add_epi64(
acc,
_mm512_cvtepu32_epi64(_mm512_extracti64x4_epi64::<1>(p_hi)),
)
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
unsafe fn c1_group_u16(acc: __m128i, row: &[u16], base: usize, w8: &[u16]) -> __m128i {
unsafe {
let s16 = if base + 8 <= row.len() {
_mm_loadu_si128(row.as_ptr().add(base).cast())
} else {
let mut sbuf = [0u16; 8];
let take = row.len() - base;
sbuf[..take].copy_from_slice(&row[base..]);
_mm_loadu_si128(sbuf.as_ptr().cast())
};
let w = _mm_loadu_si128(w8.as_ptr().cast());
mac128_u16x8_u64(acc, s16, w)
}
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
pub(crate) unsafe fn area_h_reduce_row_u16_c1(
row: &[u16],
starts: &[usize],
w16: &[u16],
w16_off: &[usize],
h_tmp: &mut [u64],
) {
for (j, &start) in starts.iter().enumerate() {
let span = &w16[w16_off[j]..w16_off[j + 1]];
let len = span.len();
unsafe {
let mut acc = _mm512_setzero_si512();
let mut acc128 = _mm_setzero_si128();
let mut t = 0usize;
while t + 32 <= len && start + t + 32 <= row.len() {
let s16 = _mm512_loadu_si512(row.as_ptr().add(start + t).cast());
let w = _mm512_loadu_si512(span[t..].as_ptr().cast());
acc = mac512_u16x32_u64(acc, s16, w);
t += 32;
}
while t + 8 <= len {
acc128 = c1_group_u16(acc128, row, start + t, &span[t..]);
t += 8;
}
let wide = _mm512_reduce_add_epi64(acc) as u64;
h_tmp[j] = wide.wrapping_add(hsum128_u64(acc128));
}
}
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
fn c3_masks_u16() -> ([__m128i; 3], [__m128i; 3], [__m128i; 3]) {
let m0: [__m128i; 3] = [
_mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 13, 12, 7, 6, 1, 0),
_mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 15, 14, 9, 8, 3, 2),
_mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 11, 10, 5, 4),
];
let m1: [__m128i; 3] = [
_mm_set_epi8(-1, -1, -1, -1, 15, 14, 9, 8, 3, 2, -1, -1, -1, -1, -1, -1),
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 11, 10, 5, 4, -1, -1, -1, -1, -1, -1),
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 13, 12, 7, 6, 1, 0, -1, -1, -1, -1),
];
let m2: [__m128i; 3] = [
_mm_set_epi8(11, 10, 5, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1),
_mm_set_epi8(13, 12, 7, 6, 1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1),
_mm_set_epi8(15, 14, 9, 8, 3, 2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1),
];
(m0, m1, m2)
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
unsafe fn c3_group_u16(
acc: &mut [__m128i; 3],
row: &[u16],
cell: usize,
w8: &[u16],
m0: &[__m128i; 3],
m1: &[__m128i; 3],
m2: &[__m128i; 3],
) {
let base = cell * 3;
unsafe {
let (v0, v1, v2) = if base + 24 <= row.len() {
(
_mm_loadu_si128(row.as_ptr().add(base).cast()),
_mm_loadu_si128(row.as_ptr().add(base + 8).cast()),
_mm_loadu_si128(row.as_ptr().add(base + 16).cast()),
)
} else {
let mut sbuf = [0u16; 24];
let take = row.len() - base;
sbuf[..take].copy_from_slice(&row[base..]);
(
_mm_loadu_si128(sbuf.as_ptr().cast()),
_mm_loadu_si128(sbuf.as_ptr().add(8).cast()),
_mm_loadu_si128(sbuf.as_ptr().add(16).cast()),
)
};
let w = _mm_loadu_si128(w8.as_ptr().cast());
for ch in 0..3 {
let gathered = _mm_or_si128(
_mm_or_si128(_mm_shuffle_epi8(v0, m0[ch]), _mm_shuffle_epi8(v1, m1[ch])),
_mm_shuffle_epi8(v2, m2[ch]),
);
acc[ch] = mac128_u16x8_u64(acc[ch], gathered, w);
}
}
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
pub(crate) unsafe fn area_h_reduce_row_u16_c3(
row: &[u16],
starts: &[usize],
w16: &[u16],
w16_off: &[usize],
h_tmp: &mut [u64],
) {
let (m0, m1, m2) = c3_masks_u16();
for (j, &start) in starts.iter().enumerate() {
let span = &w16[w16_off[j]..w16_off[j + 1]];
let len = span.len();
unsafe {
let mut acc = [_mm_setzero_si128(); 3];
let mut t = 0usize;
while t + 8 <= len {
c3_group_u16(&mut acc, row, start + t, &span[t..], &m0, &m1, &m2);
t += 8;
}
for ch in 0..3 {
h_tmp[j * 3 + ch] = hsum128_u64(acc[ch]);
}
}
}
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
pub(crate) unsafe fn area_v_accumulate_u16(acc: &mut [u64], h_tmp: &[u64], w: u32) {
let n = acc.len();
debug_assert!(h_tmp.len() >= n, "h_tmp too short");
let wv = _mm512_set1_epi64(i64::from(w));
let mut i = 0usize;
unsafe {
while i + 8 <= n {
let t = _mm512_loadu_si512(h_tmp.as_ptr().add(i).cast());
let prod_lo = _mm512_mul_epu32(t, wv);
let prod_hi = _mm512_mul_epu32(_mm512_srli_epi64::<32>(t), wv);
let prod = _mm512_add_epi64(prod_lo, _mm512_slli_epi64::<32>(prod_hi));
let a = _mm512_loadu_si512(acc.as_ptr().add(i).cast());
_mm512_storeu_si512(acc.as_mut_ptr().add(i).cast(), _mm512_add_epi64(a, prod));
i += 8;
}
}
for k in i..n {
acc[k] += u64::from(w) * h_tmp[k];
}
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
fn widen_w16_f64(w: __m128i) -> __m512d {
_mm512_cvtepi32_pd(_mm256_cvtepu16_epi32(w))
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
fn mac8_f32(acc: __m512d, s: __m256, wf: __m512d) -> __m512d {
let keep = _mm512_cmp_pd_mask::<_CMP_NEQ_OQ>(wf, _mm512_setzero_pd());
let sf = _mm512_maskz_mov_pd(keep, _mm512_cvtps_pd(s));
_mm512_add_pd(acc, _mm512_mul_pd(sf, wf))
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
fn deint3_f32(x: __m128, y: __m128, z: __m128) -> (__m128, __m128, __m128) {
let yz_r = _mm_shuffle_ps::<0b00_01_00_10>(y, z);
let r = _mm_shuffle_ps::<0b10_00_11_00>(x, yz_r);
let xy_g = _mm_shuffle_ps::<0b00_00_00_01>(x, y);
let yz_g = _mm_shuffle_ps::<0b00_10_00_11>(y, z);
let g = _mm_shuffle_ps::<0b10_00_10_00>(xy_g, yz_g);
let xy_b = _mm_shuffle_ps::<0b00_01_00_10>(x, y);
let b = _mm_shuffle_ps::<0b11_00_10_00>(xy_b, z);
(r, g, b)
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
pub(crate) unsafe fn area_h_reduce_row_f32_c1(
row: &[f32],
starts: &[usize],
w16: &[u16],
w16_off: &[usize],
h_tmp: &mut [f64],
) {
for (j, &start) in starts.iter().enumerate() {
let span = &w16[w16_off[j]..w16_off[j + 1]];
unsafe {
let mut acc = _mm512_setzero_pd();
for (ci, chunk) in span.as_chunks::<8>().0.iter().enumerate() {
let base = start + ci * 8;
let s = if base + 8 <= row.len() {
_mm256_loadu_ps(row.as_ptr().add(base))
} else {
let mut sbuf = [0f32; 8];
let take = row.len() - base;
sbuf[..take].copy_from_slice(&row[base..]);
_mm256_loadu_ps(sbuf.as_ptr())
};
let w = _mm_loadu_si128(chunk.as_ptr().cast());
acc = mac8_f32(acc, s, widen_w16_f64(w));
}
h_tmp[j] = _mm512_reduce_add_pd(acc);
}
}
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
pub(crate) unsafe fn area_h_reduce_row_f32_c3(
row: &[f32],
starts: &[usize],
w16: &[u16],
w16_off: &[usize],
h_tmp: &mut [f64],
) {
for (j, &start) in starts.iter().enumerate() {
let span = &w16[w16_off[j]..w16_off[j + 1]];
unsafe {
let mut acc0 = _mm512_setzero_pd();
let mut acc1 = _mm512_setzero_pd();
let mut acc2 = _mm512_setzero_pd();
for (ci, chunk) in span.as_chunks::<8>().0.iter().enumerate() {
let base = (start + ci * 8) * 3;
let mut sbuf = [0f32; 24];
let p = if base + 24 <= row.len() {
row.as_ptr().add(base)
} else {
let take = row.len() - base;
sbuf[..take].copy_from_slice(&row[base..]);
sbuf.as_ptr()
};
let (r0, g0, b0) = deint3_f32(
_mm_loadu_ps(p),
_mm_loadu_ps(p.add(4)),
_mm_loadu_ps(p.add(8)),
);
let (r1, g1, b1) = deint3_f32(
_mm_loadu_ps(p.add(12)),
_mm_loadu_ps(p.add(16)),
_mm_loadu_ps(p.add(20)),
);
let w = _mm_loadu_si128(chunk.as_ptr().cast());
let wf = widen_w16_f64(w);
acc0 = mac8_f32(acc0, _mm256_set_m128(r1, r0), wf);
acc1 = mac8_f32(acc1, _mm256_set_m128(g1, g0), wf);
acc2 = mac8_f32(acc2, _mm256_set_m128(b1, b0), wf);
}
h_tmp[j * 3] = _mm512_reduce_add_pd(acc0);
h_tmp[j * 3 + 1] = _mm512_reduce_add_pd(acc1);
h_tmp[j * 3 + 2] = _mm512_reduce_add_pd(acc2);
}
}
}
#[inline]
#[target_feature(enable = "avx512f,avx512bw")]
pub(crate) unsafe fn area_v_accumulate_f32(acc: &mut [f64], h_tmp: &[f64], w: f64) {
let n = acc.len();
debug_assert!(h_tmp.len() >= n, "h_tmp too short");
let wv = _mm512_set1_pd(w);
let mut i = 0usize;
unsafe {
while i + 8 <= n {
let t = _mm512_loadu_pd(h_tmp.as_ptr().add(i));
let a = _mm512_loadu_pd(acc.as_ptr().add(i));
_mm512_storeu_pd(
acc.as_mut_ptr().add(i),
_mm512_add_pd(a, _mm512_mul_pd(t, wv)),
);
i += 8;
}
}
for k in i..n {
acc[k] += w * h_tmp[k];
}
}