use crate::aq_blur::{BlurRows, RADIUS, WINDOW};
use std::arch::x86_64::*;
#[target_feature(enable = "avx2")]
pub(crate) fn blur_horizontal_avx2(src: &[f32], k: &[f32; WINDOW], dst: &mut [f32]) -> usize {
assert!(src.len() >= dst.len() + 2 * RADIUS);
let done = horizontal::<4>(src, k, dst);
done + horizontal::<1>(&src[done..], k, &mut dst[done..])
}
#[inline]
#[target_feature(enable = "avx2")]
fn horizontal<const N: usize>(src: &[f32], k: &[f32; WINDOW], dst: &mut [f32]) -> usize {
const LANES: usize = 8;
let center = _mm256_set1_ps(k[RADIUS]);
let mut done = 0;
let (vectors, _) = dst.as_chunks_mut::<LANES>();
let (blocks, _) = vectors.as_chunks_mut::<N>();
for (block, dst) in blocks.iter_mut().enumerate() {
let dst = dst.as_flattened_mut();
let x = block * N * LANES;
unsafe {
let mut acc: [__m256; N] = std::array::from_fn(|lane| {
_mm256_mul_ps(
_mm256_loadu_ps(src.as_ptr().add(x + lane * LANES + RADIUS)),
center,
)
});
for i in 0..RADIUS {
let coeff = _mm256_set1_ps(k[i]);
for (lane, acc) in acc.iter_mut().enumerate() {
let a = _mm256_loadu_ps(src.as_ptr().add(x + lane * LANES + i));
let b = _mm256_loadu_ps(src.as_ptr().add(x + lane * LANES + WINDOW - 1 - i));
*acc = _mm256_add_ps(*acc, _mm256_mul_ps(_mm256_add_ps(a, b), coeff));
}
}
for (lane, acc) in acc.into_iter().enumerate() {
_mm256_storeu_ps(dst.as_mut_ptr().add(lane * LANES), acc);
}
}
done += N * LANES;
}
done
}
#[target_feature(enable = "avx2")]
pub(crate) fn blur_vertical_avx2(rows: &BlurRows<'_>, k: &[f32; WINDOW], dst: &mut [f32]) -> usize {
assert!(rows.iter().all(|row| row.len() >= dst.len()));
let done = vertical::<4>(rows, k, dst);
done + vertical::<1>(&rows.map(|row| &row[done..]), k, &mut dst[done..])
}
#[inline]
#[target_feature(enable = "avx2")]
fn vertical<const N: usize>(rows: &BlurRows<'_>, k: &[f32; WINDOW], dst: &mut [f32]) -> usize {
const LANES: usize = 8;
let center = _mm256_set1_ps(k[RADIUS]);
let mut done = 0;
let (vectors, _) = dst.as_chunks_mut::<LANES>();
let (blocks, _) = vectors.as_chunks_mut::<N>();
for (block, dst) in blocks.iter_mut().enumerate() {
let dst = dst.as_flattened_mut();
let x = block * N * LANES;
unsafe {
let mut acc: [__m256; N] = std::array::from_fn(|lane| {
_mm256_mul_ps(
_mm256_loadu_ps(rows[RADIUS].as_ptr().add(x + lane * LANES)),
center,
)
});
for i in 0..RADIUS {
let coeff = _mm256_set1_ps(k[i]);
for (lane, acc) in acc.iter_mut().enumerate() {
let a = _mm256_loadu_ps(rows[i].as_ptr().add(x + lane * LANES));
let b = _mm256_loadu_ps(rows[WINDOW - 1 - i].as_ptr().add(x + lane * LANES));
*acc = _mm256_add_ps(*acc, _mm256_mul_ps(_mm256_add_ps(a, b), coeff));
}
}
for (lane, acc) in acc.into_iter().enumerate() {
_mm256_storeu_ps(dst.as_mut_ptr().add(lane * LANES), acc);
}
}
done += N * LANES;
}
done
}