#![cfg_attr(not(feature = "std"), allow(dead_code))]
#![cfg_attr(not(any(feature = "yuv-planar", feature = "rgb")), allow(dead_code))]
#[cfg(target_arch = "x86_64")]
#[cfg_attr(miri, allow(unused_imports))]
use core::arch::x86_64::*;
#[inline]
#[target_feature(enable = "sse4.1")]
fn hsum_u32(acc: __m128i) -> u32 {
let hi = _mm_shuffle_epi32::<0b01_00_11_10>(acc);
let s = _mm_add_epi32(acc, hi);
let s2 = _mm_add_epi32(s, _mm_shuffle_epi32::<0b00_00_00_01>(s));
_mm_cvtsi128_si32(s2) as u32
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn mac_u16x8(acc: __m128i, s16: __m128i, w: __m128i) -> __m128i {
let lo = _mm_mullo_epi16(s16, w);
let hi = _mm_mulhi_epu16(s16, w);
let acc = _mm_add_epi32(acc, _mm_unpacklo_epi16(lo, hi));
_mm_add_epi32(acc, _mm_unpackhi_epi16(lo, hi))
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn area_h_reduce_row_c1(
row: &[u8],
starts: &[usize],
w16: &[u16],
w16_off: &[usize],
h_tmp: &mut [u32],
) {
for (j, &start) in starts.iter().enumerate() {
let span = &w16[w16_off[j]..w16_off[j + 1]];
unsafe {
let mut acc = _mm_setzero_si128();
for (ci, chunk) in span.as_chunks::<8>().0.iter().enumerate() {
let base = start + ci * 8;
let s8 = if base + 8 <= row.len() {
_mm_loadl_epi64(row.as_ptr().add(base).cast())
} else {
let mut sbuf = [0u8; 8];
let take = row.len() - base;
sbuf[..take].copy_from_slice(&row[base..]);
_mm_loadl_epi64(sbuf.as_ptr().cast())
};
let s16 = _mm_cvtepu8_epi16(s8);
let w = _mm_loadu_si128(chunk.as_ptr().cast());
acc = mac_u16x8(acc, s16, w);
}
h_tmp[j] = hsum_u32(acc);
}
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn area_h_reduce_row_c3(
row: &[u8],
starts: &[usize],
w16: &[u16],
w16_off: &[usize],
h_tmp: &mut [u32],
) {
let (m0, m1): ([__m128i; 3], [__m128i; 3]) = (
[
_mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 15, 12, 9, 6, 3, 0),
_mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 13, 10, 7, 4, 1),
_mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 14, 11, 8, 5, 2),
],
[
_mm_set_epi8(
-1, -1, -1, -1, -1, -1, -1, -1, 13, 10, -1, -1, -1, -1, -1, -1,
),
_mm_set_epi8(
-1, -1, -1, -1, -1, -1, -1, -1, 14, 11, 8, -1, -1, -1, -1, -1,
),
_mm_set_epi8(
-1, -1, -1, -1, -1, -1, -1, -1, 15, 12, 9, -1, -1, -1, -1, -1,
),
],
);
for (j, &start) in starts.iter().enumerate() {
let span = &w16[w16_off[j]..w16_off[j + 1]];
unsafe {
let mut acc = [_mm_setzero_si128(); 3];
for (ci, chunk) in span.as_chunks::<8>().0.iter().enumerate() {
let base = (start + ci * 8) * 3;
let mut sbuf = [0u8; 24];
let (v0, v1) = if base + 24 <= row.len() {
(
_mm_loadu_si128(row.as_ptr().add(base).cast()),
_mm_loadu_si128(row.as_ptr().add(base + 8).cast()),
)
} else {
let take = row.len() - base;
sbuf[..take].copy_from_slice(&row[base..]);
(
_mm_loadu_si128(sbuf.as_ptr().cast()),
_mm_loadu_si128(sbuf.as_ptr().add(8).cast()),
)
};
let w = _mm_loadu_si128(chunk.as_ptr().cast());
for ch in 0..3 {
let gathered = _mm_or_si128(_mm_shuffle_epi8(v0, m0[ch]), _mm_shuffle_epi8(v1, m1[ch]));
acc[ch] = mac_u16x8(acc[ch], _mm_cvtepu8_epi16(gathered), w);
}
}
for ch in 0..3 {
h_tmp[j * 3 + ch] = hsum_u32(acc[ch]);
}
}
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn area_v_accumulate(acc: &mut [u64], h_tmp: &[u32], w: u32) {
let n = acc.len();
debug_assert!(h_tmp.len() >= n, "h_tmp too short");
let wv = _mm_set1_epi64x(i64::from(w));
let mut i = 0usize;
unsafe {
while i + 4 <= n {
let t = _mm_loadu_si128(h_tmp.as_ptr().add(i).cast());
let even = _mm_mul_epu32(t, wv);
let odd = _mm_mul_epu32(_mm_srli_epi64::<32>(t), wv);
let a01 = _mm_loadu_si128(acc.as_ptr().add(i).cast());
let a23 = _mm_loadu_si128(acc.as_ptr().add(i + 2).cast());
let p01 = _mm_unpacklo_epi64(even, odd);
let p23 = _mm_unpackhi_epi64(even, odd);
_mm_storeu_si128(acc.as_mut_ptr().add(i).cast(), _mm_add_epi64(a01, p01));
_mm_storeu_si128(acc.as_mut_ptr().add(i + 2).cast(), _mm_add_epi64(a23, p23));
i += 4;
}
}
for k in i..n {
acc[k] += u64::from(w) * u64::from(h_tmp[k]);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn hsum_u64(acc: __m128i) -> u64 {
let hi = _mm_unpackhi_epi64(acc, acc);
_mm_cvtsi128_si64(_mm_add_epi64(acc, hi)) as u64
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn mac_u16x8_u64(acc: __m128i, s16: __m128i, w: __m128i) -> __m128i {
let lo = _mm_mullo_epi16(s16, w);
let hi = _mm_mulhi_epu16(s16, w);
let p_lo = _mm_unpacklo_epi16(lo, hi);
let p_hi = _mm_unpackhi_epi16(lo, hi);
let acc = _mm_add_epi64(acc, _mm_cvtepu32_epi64(p_lo));
let acc = _mm_add_epi64(acc, _mm_cvtepu32_epi64(_mm_srli_si128::<8>(p_lo)));
let acc = _mm_add_epi64(acc, _mm_cvtepu32_epi64(p_hi));
_mm_add_epi64(acc, _mm_cvtepu32_epi64(_mm_srli_si128::<8>(p_hi)))
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn area_h_reduce_row_u16_c1(
row: &[u16],
starts: &[usize],
w16: &[u16],
w16_off: &[usize],
h_tmp: &mut [u64],
) {
for (j, &start) in starts.iter().enumerate() {
let span = &w16[w16_off[j]..w16_off[j + 1]];
unsafe {
let mut acc = _mm_setzero_si128();
for (ci, chunk) in span.as_chunks::<8>().0.iter().enumerate() {
let base = start + ci * 8;
let s16 = if base + 8 <= row.len() {
_mm_loadu_si128(row.as_ptr().add(base).cast())
} else {
let mut sbuf = [0u16; 8];
let take = row.len() - base;
sbuf[..take].copy_from_slice(&row[base..]);
_mm_loadu_si128(sbuf.as_ptr().cast())
};
let w = _mm_loadu_si128(chunk.as_ptr().cast());
acc = mac_u16x8_u64(acc, s16, w);
}
h_tmp[j] = hsum_u64(acc);
}
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn area_h_reduce_row_u16_c3(
row: &[u16],
starts: &[usize],
w16: &[u16],
w16_off: &[usize],
h_tmp: &mut [u64],
) {
let m0: [__m128i; 3] = [
_mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 13, 12, 7, 6, 1, 0),
_mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 15, 14, 9, 8, 3, 2),
_mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 11, 10, 5, 4),
];
let m1: [__m128i; 3] = [
_mm_set_epi8(-1, -1, -1, -1, 15, 14, 9, 8, 3, 2, -1, -1, -1, -1, -1, -1),
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 11, 10, 5, 4, -1, -1, -1, -1, -1, -1),
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 13, 12, 7, 6, 1, 0, -1, -1, -1, -1),
];
let m2: [__m128i; 3] = [
_mm_set_epi8(11, 10, 5, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1),
_mm_set_epi8(13, 12, 7, 6, 1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1),
_mm_set_epi8(15, 14, 9, 8, 3, 2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1),
];
for (j, &start) in starts.iter().enumerate() {
let span = &w16[w16_off[j]..w16_off[j + 1]];
unsafe {
let mut acc = [_mm_setzero_si128(); 3];
for (ci, chunk) in span.as_chunks::<8>().0.iter().enumerate() {
let base = (start + ci * 8) * 3;
let mut sbuf = [0u16; 24];
let (v0, v1, v2) = if base + 24 <= row.len() {
(
_mm_loadu_si128(row.as_ptr().add(base).cast()),
_mm_loadu_si128(row.as_ptr().add(base + 8).cast()),
_mm_loadu_si128(row.as_ptr().add(base + 16).cast()),
)
} else {
let take = row.len() - base;
sbuf[..take].copy_from_slice(&row[base..]);
(
_mm_loadu_si128(sbuf.as_ptr().cast()),
_mm_loadu_si128(sbuf.as_ptr().add(8).cast()),
_mm_loadu_si128(sbuf.as_ptr().add(16).cast()),
)
};
let w = _mm_loadu_si128(chunk.as_ptr().cast());
for ch in 0..3 {
let gathered = _mm_or_si128(
_mm_or_si128(_mm_shuffle_epi8(v0, m0[ch]), _mm_shuffle_epi8(v1, m1[ch])),
_mm_shuffle_epi8(v2, m2[ch]),
);
acc[ch] = mac_u16x8_u64(acc[ch], gathered, w);
}
}
for ch in 0..3 {
h_tmp[j * 3 + ch] = hsum_u64(acc[ch]);
}
}
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn area_v_accumulate_u16(acc: &mut [u64], h_tmp: &[u64], w: u32) {
let n = acc.len();
debug_assert!(h_tmp.len() >= n, "h_tmp too short");
let wv = _mm_set1_epi64x(i64::from(w));
let mut i = 0usize;
unsafe {
while i + 2 <= n {
let t = _mm_loadu_si128(h_tmp.as_ptr().add(i).cast());
let prod_lo = _mm_mul_epu32(t, wv);
let prod_hi = _mm_mul_epu32(_mm_srli_epi64::<32>(t), wv);
let prod = _mm_add_epi64(prod_lo, _mm_slli_epi64::<32>(prod_hi));
let a = _mm_loadu_si128(acc.as_ptr().add(i).cast());
_mm_storeu_si128(acc.as_mut_ptr().add(i).cast(), _mm_add_epi64(a, prod));
i += 2;
}
}
for k in i..n {
acc[k] += u64::from(w) * h_tmp[k];
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn hsum_pd(v: __m128d) -> f64 {
_mm_cvtsd_f64(_mm_add_sd(v, _mm_unpackhi_pd(v, v)))
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn widen_w16_f64(w: __m128i) -> (__m128d, __m128d, __m128d, __m128d) {
let w_lo = _mm_cvtepu16_epi32(w);
let w_hi = _mm_cvtepu16_epi32(_mm_srli_si128::<8>(w));
(
_mm_cvtepi32_pd(w_lo),
_mm_cvtepi32_pd(_mm_unpackhi_epi64(w_lo, w_lo)),
_mm_cvtepi32_pd(w_hi),
_mm_cvtepi32_pd(_mm_unpackhi_epi64(w_hi, w_hi)),
)
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn mac8_f32(
acc: __m128d,
s_lo: __m128,
s_hi: __m128,
wf0: __m128d,
wf1: __m128d,
wf2: __m128d,
wf3: __m128d,
) -> __m128d {
let a = _mm_add_pd(acc, _mm_mul_pd(mask_pd(_mm_cvtps_pd(s_lo), wf0), wf0));
let a = _mm_add_pd(
a,
_mm_mul_pd(mask_pd(_mm_cvtps_pd(_mm_movehl_ps(s_lo, s_lo)), wf1), wf1),
);
let a = _mm_add_pd(a, _mm_mul_pd(mask_pd(_mm_cvtps_pd(s_hi), wf2), wf2));
_mm_add_pd(
a,
_mm_mul_pd(mask_pd(_mm_cvtps_pd(_mm_movehl_ps(s_hi, s_hi)), wf3), wf3),
)
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn mask_pd(sf: __m128d, wf: __m128d) -> __m128d {
_mm_and_pd(sf, _mm_cmpneq_pd(wf, _mm_setzero_pd()))
}
#[inline]
#[target_feature(enable = "sse4.1")]
fn deint3_f32(x: __m128, y: __m128, z: __m128) -> (__m128, __m128, __m128) {
let yz_r = _mm_shuffle_ps::<0b00_01_00_10>(y, z);
let r = _mm_shuffle_ps::<0b10_00_11_00>(x, yz_r);
let xy_g = _mm_shuffle_ps::<0b00_00_00_01>(x, y);
let yz_g = _mm_shuffle_ps::<0b00_10_00_11>(y, z);
let g = _mm_shuffle_ps::<0b10_00_10_00>(xy_g, yz_g);
let xy_b = _mm_shuffle_ps::<0b00_01_00_10>(x, y);
let b = _mm_shuffle_ps::<0b11_00_10_00>(xy_b, z);
(r, g, b)
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn area_h_reduce_row_f32_c1(
row: &[f32],
starts: &[usize],
w16: &[u16],
w16_off: &[usize],
h_tmp: &mut [f64],
) {
for (j, &start) in starts.iter().enumerate() {
let span = &w16[w16_off[j]..w16_off[j + 1]];
unsafe {
let mut acc = _mm_setzero_pd();
for (ci, chunk) in span.as_chunks::<8>().0.iter().enumerate() {
let base = start + ci * 8;
let (s_lo, s_hi) = if base + 8 <= row.len() {
(
_mm_loadu_ps(row.as_ptr().add(base)),
_mm_loadu_ps(row.as_ptr().add(base + 4)),
)
} else {
let mut sbuf = [0f32; 8];
let take = row.len() - base;
sbuf[..take].copy_from_slice(&row[base..]);
(
_mm_loadu_ps(sbuf.as_ptr()),
_mm_loadu_ps(sbuf.as_ptr().add(4)),
)
};
let w = _mm_loadu_si128(chunk.as_ptr().cast());
let (wf0, wf1, wf2, wf3) = widen_w16_f64(w);
acc = mac8_f32(acc, s_lo, s_hi, wf0, wf1, wf2, wf3);
}
h_tmp[j] = hsum_pd(acc);
}
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn area_h_reduce_row_f32_c3(
row: &[f32],
starts: &[usize],
w16: &[u16],
w16_off: &[usize],
h_tmp: &mut [f64],
) {
for (j, &start) in starts.iter().enumerate() {
let span = &w16[w16_off[j]..w16_off[j + 1]];
unsafe {
let mut acc0 = _mm_setzero_pd();
let mut acc1 = _mm_setzero_pd();
let mut acc2 = _mm_setzero_pd();
for (ci, chunk) in span.as_chunks::<8>().0.iter().enumerate() {
let base = (start + ci * 8) * 3;
let mut sbuf = [0f32; 24];
let p = if base + 24 <= row.len() {
row.as_ptr().add(base)
} else {
let take = row.len() - base;
sbuf[..take].copy_from_slice(&row[base..]);
sbuf.as_ptr()
};
let (r0, g0, b0) = deint3_f32(
_mm_loadu_ps(p),
_mm_loadu_ps(p.add(4)),
_mm_loadu_ps(p.add(8)),
);
let (r1, g1, b1) = deint3_f32(
_mm_loadu_ps(p.add(12)),
_mm_loadu_ps(p.add(16)),
_mm_loadu_ps(p.add(20)),
);
let w = _mm_loadu_si128(chunk.as_ptr().cast());
let (wf0, wf1, wf2, wf3) = widen_w16_f64(w);
acc0 = mac8_f32(acc0, r0, r1, wf0, wf1, wf2, wf3);
acc1 = mac8_f32(acc1, g0, g1, wf0, wf1, wf2, wf3);
acc2 = mac8_f32(acc2, b0, b1, wf0, wf1, wf2, wf3);
}
h_tmp[j * 3] = hsum_pd(acc0);
h_tmp[j * 3 + 1] = hsum_pd(acc1);
h_tmp[j * 3 + 2] = hsum_pd(acc2);
}
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn area_v_accumulate_f32(acc: &mut [f64], h_tmp: &[f64], w: f64) {
let n = acc.len();
debug_assert!(h_tmp.len() >= n, "h_tmp too short");
let wv = _mm_set1_pd(w);
let mut i = 0usize;
unsafe {
while i + 2 <= n {
let t = _mm_loadu_pd(h_tmp.as_ptr().add(i));
let a = _mm_loadu_pd(acc.as_ptr().add(i));
_mm_storeu_pd(acc.as_mut_ptr().add(i), _mm_add_pd(a, _mm_mul_pd(t, wv)));
i += 2;
}
}
for k in i..n {
acc[k] += w * h_tmp[k];
}
}