use fearless_simd::{Level, Simd, prelude::*};
use realfft::num_complex::Complex;
use std::sync::OnceLock;
pub fn level() -> Level {
static LEVEL: OnceLock<Level> = OnceLock::new();
*LEVEL.get_or_init(Level::new)
}
#[inline(always)]
fn load_i16<S: Simd, const U: usize>(simd: S, q: &[i16]) -> [S::f32s; U] {
let n = S::f32s::LEN;
let mut out = [S::f32s::splat(simd, 0.0); U];
for p in 0..U / 2 {
let (lo, hi) = S::i16s::from_slice(simd, &q[p * 2 * n..(p + 1) * 2 * n]).widen();
out[2 * p] = lo.to_float::<S::f32s>();
out[2 * p + 1] = hi.to_float::<S::f32s>();
}
if U % 2 == 1 {
let tail = &q[(U - 1) * n..U * n];
out[U - 1] = S::f32s::from_fn(simd, |i| tail[i] as f32);
}
out
}
#[inline(always)]
pub fn fma_rows_multi<S: Simd, const T: usize, const U: usize>(
simd: S,
out: &mut [f32],
oc: usize,
w: &[i16],
scales: &[f32],
rows: &[usize],
xs: &[f32],
) {
let n = S::f32s::LEN;
let mut j = 0;
while j + U * n <= oc {
fma_block::<S, T, U>(simd, out, oc, w, scales, rows, xs, j);
j += U * n;
}
while j + n <= oc {
fma_block::<S, T, 1>(simd, out, oc, w, scales, rows, xs, j);
j += n;
}
for jj in j..oc {
for t in 0..T {
let mut acc = 0.0f32;
for (&r, xr) in rows.iter().zip(xs.as_chunks::<T>().0) {
acc += w[r * oc + jj] as f32 * xr[t];
}
out[t * oc + jj] += acc * scales[jj];
}
}
}
#[inline(always)]
fn fma_block<S: Simd, const T: usize, const U: usize>(
simd: S,
out: &mut [f32],
oc: usize,
w: &[i16],
scales: &[f32],
rows: &[usize],
xs: &[f32],
j: usize,
) {
let n = S::f32s::LEN;
let zero = S::f32s::splat(simd, 0.0);
let mut acc = [[zero; U]; T];
for (&r, xr) in rows.iter().zip(xs.as_chunks::<T>().0) {
let start = r * oc + j;
let wr = &w[start..start + U * n];
let w_v = load_i16::<S, U>(simd, wr);
for t in 0..T {
let x_vec = S::f32s::splat(simd, xr[t]);
for u in 0..U {
acc[t][u] = w_v[u].mul_add(x_vec, acc[t][u]);
}
}
}
for u in 0..U {
let scale_v = S::f32s::from_slice(simd, &scales[j + u * n..j + (u + 1) * n]);
for (t, acc_t) in acc.iter().enumerate() {
let start = t * oc + j + u * n;
let out_v = S::f32s::from_slice(simd, &out[start..start + n]);
acc_t[u]
.mul_add(scale_v, out_v)
.store_slice(&mut out[start..start + n]);
}
}
}
const UNROLL: usize = 8;
#[inline(always)]
pub fn fma_rows<S: Simd>(
simd: S,
out: &mut [f32],
w: &[i16],
scales: &[f32],
rows: &[(usize, f32)],
) {
let n = S::f32s::LEN;
let len = out.len();
let zero = S::f32s::splat(simd, 0.0);
let mut j = 0;
while j + UNROLL * n <= len {
let mut acc = [zero; UNROLL];
for &(off, x) in rows {
let x_vec = S::f32s::splat(simd, x);
let w_v = load_i16::<S, UNROLL>(simd, &w[off + j..off + j + UNROLL * n]);
for u in 0..UNROLL {
acc[u] = w_v[u].mul_add(x_vec, acc[u]);
}
}
for (u, acc_u) in acc.iter().enumerate() {
let range = j + u * n..j + (u + 1) * n;
let scale_v = S::f32s::from_slice(simd, &scales[range.clone()]);
let out_v = S::f32s::from_slice(simd, &out[range.clone()]);
acc_u.mul_add(scale_v, out_v).store_slice(&mut out[range]);
}
j += UNROLL * n;
}
while j + n <= len {
let mut acc = zero;
for &(off, x) in rows {
let x_vec = S::f32s::splat(simd, x);
let [w_v] = load_i16::<S, 1>(simd, &w[off + j..off + j + n]);
acc = w_v.mul_add(x_vec, acc);
}
let scale_v = S::f32s::from_slice(simd, &scales[j..j + n]);
let out_v = S::f32s::from_slice(simd, &out[j..j + n]);
acc.mul_add(scale_v, out_v).store_slice(&mut out[j..j + n]);
j += n;
}
for jj in j..len {
let mut acc = 0.0f32;
for &(off, x) in rows {
acc += w[off + jj] as f32 * x;
}
out[jj] += acc * scales[jj];
}
}
#[inline(always)]
pub fn mul_slices<S: Simd>(simd: S, out: &mut [f32], a: &[f32], b: &[f32]) {
let n = S::f32s::LEN;
let len = out.len();
let (a, b) = (&a[..len], &b[..len]);
let mut j = 0;
while j + n <= len {
let a_v = S::f32s::from_slice(simd, &a[j..j + n]);
let b_v = S::f32s::from_slice(simd, &b[j..j + n]);
(a_v * b_v).store_slice(&mut out[j..j + n]);
j += n;
}
for i in j..len {
out[i] = a[i] * b[i];
}
}
#[inline(always)]
pub fn complex_norm<S: Simd>(simd: S, out: &mut [f32], input: &[Complex<f32>]) {
let n = S::f32s::LEN;
let len = out.len();
let input = &input[..len];
let mut j = 0;
while j + n <= len {
let c = &input[j..j + n];
let re = S::f32s::from_fn(simd, |i| c[i].re);
let im = S::f32s::from_fn(simd, |i| c[i].im);
re.mul_add(re, im * im)
.sqrt()
.store_slice(&mut out[j..j + n]);
j += n;
}
for i in j..len {
out[i] = input[i].norm();
}
}