use std::arch::x86_64::*;
#[inline]
#[target_feature(enable = "avx2")]
fn reduce_transposed_4x8(a: __m256, b: __m256, c: __m256, d: __m256) -> [f32; 4] {
let ab_lo = _mm256_unpacklo_ps(a, b);
let ab_hi = _mm256_unpackhi_ps(a, b);
let cd_lo = _mm256_unpacklo_ps(c, d);
let cd_hi = _mm256_unpackhi_ps(c, d);
let lane_0_4 = _mm256_shuffle_ps::<0x44>(ab_lo, cd_lo);
let lane_1_5 = _mm256_shuffle_ps::<0xee>(ab_lo, cd_lo);
let lane_2_6 = _mm256_shuffle_ps::<0x44>(ab_hi, cd_hi);
let lane_3_7 = _mm256_shuffle_ps::<0xee>(ab_hi, cd_hi);
let sums_0_3_and_4_7 = _mm256_add_ps(
_mm256_add_ps(lane_0_4, lane_1_5),
_mm256_add_ps(lane_2_6, lane_3_7),
);
let low = _mm256_castps256_ps128(sums_0_3_and_4_7);
let high = _mm256_extractf128_ps::<1>(sums_0_3_and_4_7);
let sums = _mm_add_ps(low, high);
let mut out = [0.0f32; 4];
unsafe { _mm_storeu_ps(out.as_mut_ptr(), sums) };
out
}
#[target_feature(enable = "avx2,fma")]
pub(crate) fn cfl_regression_avx2(
y: &[f32; 64],
x: &[f32; 64],
b: &[f32; 64],
qm_x: &[f32; 64],
qm_b: &[f32; 64],
) -> [f32; 4] {
let inv_color = _mm256_set1_ps(1.0 / 84.0);
let mut ca_x = _mm256_setzero_ps();
let mut cb_x = _mm256_setzero_ps();
let mut ca_b = _mm256_setzero_ps();
let mut cb_b = _mm256_setzero_ps();
let y_chunks = y.as_chunks::<8>().0;
let x_chunks = x.as_chunks::<8>().0;
let b_chunks = b.as_chunks::<8>().0;
let qm_x_chunks = qm_x.as_chunks::<8>().0;
let qm_b_chunks = qm_b.as_chunks::<8>().0;
for ((((y, x), b), qm_x), qm_b) in y_chunks
.iter()
.zip(x_chunks)
.zip(b_chunks)
.zip(qm_x_chunks)
.zip(qm_b_chunks)
{
let yv = unsafe { _mm256_loadu_ps(y.as_ptr()) };
let xv = unsafe { _mm256_loadu_ps(x.as_ptr()) };
let bv = unsafe { _mm256_loadu_ps(b.as_ptr()) };
let qx = unsafe { _mm256_loadu_ps(qm_x.as_ptr()) };
let qb = unsafe { _mm256_loadu_ps(qm_b.as_ptr()) };
let mx = _mm256_mul_ps(yv, qx);
let sx = _mm256_mul_ps(xv, qx);
let ax = _mm256_mul_ps(inv_color, mx);
ca_x = _mm256_fmadd_ps(ax, ax, ca_x);
cb_x = _mm256_fnmadd_ps(ax, sx, cb_x);
let mb = _mm256_mul_ps(yv, qb);
let sb = _mm256_mul_ps(bv, qb);
let ab = _mm256_mul_ps(inv_color, mb);
let residual_b = _mm256_sub_ps(mb, sb);
ca_b = _mm256_fmadd_ps(ab, ab, ca_b);
cb_b = _mm256_fmadd_ps(ab, residual_b, cb_b);
}
reduce_transposed_4x8(ca_x, cb_x, ca_b, cb_b)
}
#[target_feature(enable = "avx2,fma")]
pub(crate) fn apply_cfl_avx2(x: &mut [f32], y: &[f32], b: &mut [f32], cmap_factor: [f32; 3]) {
assert_eq!(x.len(), y.len());
assert_eq!(x.len(), b.len());
let cx = _mm256_set1_ps(-cmap_factor[0]);
let cb = _mm256_set1_ps(-cmap_factor[2]);
let (x_chunks, x_tail) = x.as_chunks_mut::<8>();
let (y_chunks, y_tail) = y.as_chunks::<8>();
let (b_chunks, b_tail) = b.as_chunks_mut::<8>();
let (x_groups, x_chunks_tail) = x_chunks.as_chunks_mut::<4>();
let (y_groups, y_chunks_tail) = y_chunks.as_chunks::<4>();
let (b_groups, b_chunks_tail) = b_chunks.as_chunks_mut::<4>();
for ((x, y), b) in x_groups.iter_mut().zip(y_groups).zip(b_groups) {
let y0 = unsafe { _mm256_loadu_ps(y[0].as_ptr()) };
let y1 = unsafe { _mm256_loadu_ps(y[1].as_ptr()) };
let y2 = unsafe { _mm256_loadu_ps(y[2].as_ptr()) };
let y3 = unsafe { _mm256_loadu_ps(y[3].as_ptr()) };
let x0 = unsafe { _mm256_loadu_ps(x[0].as_ptr()) };
let x1 = unsafe { _mm256_loadu_ps(x[1].as_ptr()) };
let x2 = unsafe { _mm256_loadu_ps(x[2].as_ptr()) };
let x3 = unsafe { _mm256_loadu_ps(x[3].as_ptr()) };
let b0 = unsafe { _mm256_loadu_ps(b[0].as_ptr()) };
let b1 = unsafe { _mm256_loadu_ps(b[1].as_ptr()) };
let b2 = unsafe { _mm256_loadu_ps(b[2].as_ptr()) };
let b3 = unsafe { _mm256_loadu_ps(b[3].as_ptr()) };
let x0 = _mm256_fmadd_ps(cx, y0, x0);
let x1 = _mm256_fmadd_ps(cx, y1, x1);
let x2 = _mm256_fmadd_ps(cx, y2, x2);
let x3 = _mm256_fmadd_ps(cx, y3, x3);
let b0 = _mm256_fmadd_ps(cb, y0, b0);
let b1 = _mm256_fmadd_ps(cb, y1, b1);
let b2 = _mm256_fmadd_ps(cb, y2, b2);
let b3 = _mm256_fmadd_ps(cb, y3, b3);
unsafe {
_mm256_storeu_ps(x[0].as_mut_ptr(), x0);
_mm256_storeu_ps(x[1].as_mut_ptr(), x1);
_mm256_storeu_ps(x[2].as_mut_ptr(), x2);
_mm256_storeu_ps(x[3].as_mut_ptr(), x3);
_mm256_storeu_ps(b[0].as_mut_ptr(), b0);
_mm256_storeu_ps(b[1].as_mut_ptr(), b1);
_mm256_storeu_ps(b[2].as_mut_ptr(), b2);
_mm256_storeu_ps(b[3].as_mut_ptr(), b3);
}
}
for ((x, y), b) in x_chunks_tail
.iter_mut()
.zip(y_chunks_tail)
.zip(b_chunks_tail)
{
let yv = unsafe { _mm256_loadu_ps(y.as_ptr()) };
let xv = unsafe { _mm256_loadu_ps(x.as_ptr()) };
let bv = unsafe { _mm256_loadu_ps(b.as_ptr()) };
unsafe {
_mm256_storeu_ps(x.as_mut_ptr(), _mm256_fmadd_ps(cx, yv, xv));
_mm256_storeu_ps(b.as_mut_ptr(), _mm256_fmadd_ps(cb, yv, bv));
}
}
for ((x, &y), b) in x_tail.iter_mut().zip(y_tail).zip(b_tail) {
*x = (-cmap_factor[0]).mul_add(y, *x);
*b = (-cmap_factor[2]).mul_add(y, *b);
}
}
#[cfg(test)]
mod tests {
#[test]
fn apply_cfl_avx2_matches_scalar() {
if !is_x86_feature_detected!("avx2") || !is_x86_feature_detected!("fma") {
return;
}
for size in [0, 1, 7, 8, 15, 32, 64, 1024, 4096] {
let y: Vec<f32> = (0..size)
.map(|i| ((i * 37 % 101) as f32 - 50.0) * 0.03125)
.collect();
let mut x: Vec<f32> = (0..size).map(|i| i as f32 * 0.015625 - 2.0).collect();
let mut b: Vec<f32> = (0..size).map(|i| 3.0 - i as f32 * 0.0078125).collect();
let mut want_x = x.clone();
let mut want_b = b.clone();
let factors: [f32; 3] = [0.3125, 0.0, -0.1875];
for i in 0..size {
want_x[i] = (-factors[0]).mul_add(y[i], want_x[i]);
want_b[i] = (-factors[2]).mul_add(y[i], want_b[i]);
}
unsafe { super::apply_cfl_avx2(&mut x, &y, &mut b, factors) };
assert_eq!(x, want_x, "X mismatch at size {size}");
assert_eq!(b, want_b, "B mismatch at size {size}");
}
}
}