use std::marker::PhantomData;
use std::sync::LazyLock;
use rayon::prelude::*;
use rayon::ThreadPool;
pub fn avx2_fma_available() -> bool {
#[cfg(target_arch = "x86_64")]
{
std::arch::is_x86_feature_detected!("avx2") && std::arch::is_x86_feature_detected!("fma")
}
#[cfg(not(target_arch = "x86_64"))]
{
false
}
}
static POOL: LazyLock<ThreadPool> = LazyLock::new(|| {
let threads = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1)
.max(1);
rayon::ThreadPoolBuilder::new()
.num_threads(threads)
.thread_name(|i| format!("joshua-matmul-{i}"))
.build()
.expect("failed to build the joshua matmul thread pool")
});
pub fn for_each_row(n: usize, f: impl Fn(usize) + Send + Sync) {
let pool = &*POOL;
let threads = pool.current_num_threads();
run_chunks(pool, threads, n, |rows| {
for &row in rows {
f(row);
}
});
}
pub fn for_each_row_chunks(n: usize, f: impl Fn(&[usize]) + Send + Sync) {
let pool = &*POOL;
let threads = pool.current_num_threads();
run_chunks(pool, threads, n, f);
}
#[cfg(test)]
pub fn for_each_row_with_threads(threads: usize, n: usize, f: impl Fn(usize) + Send + Sync) {
if threads <= 1 || n < 8 {
for row in 0..n {
f(row);
}
return;
}
let pool = rayon::ThreadPoolBuilder::new()
.num_threads(threads)
.build()
.expect("test thread pool");
run_chunks(&pool, threads, n, |rows| {
for &row in rows {
f(row);
}
});
}
fn run_chunks(
pool: &ThreadPool,
threads: usize,
n: usize,
f: impl Fn(&[usize]) + Send + Sync,
) {
if threads <= 1 || n < 8 {
let rows: Vec<usize> = (0..n).collect();
f(&rows);
return;
}
let chunk = (n / (threads * 4)).max(1);
pool.install(|| {
(0..n).into_par_iter().chunks(chunk).for_each(|rows| f(&rows));
});
}
pub struct DstPtr<'a> {
ptr: *mut f32,
_marker: PhantomData<&'a mut [f32]>,
}
unsafe impl Send for DstPtr<'_> {}
unsafe impl Sync for DstPtr<'_> {}
impl<'a> DstPtr<'a> {
pub fn new(dst: &'a mut [f32]) -> Self {
Self {
ptr: dst.as_mut_ptr(),
_marker: PhantomData,
}
}
#[inline(always)]
pub unsafe fn write(&self, idx: usize, val: f32) {
*self.ptr.add(idx) = val;
}
#[inline(always)]
pub unsafe fn add(&self, idx: usize, val: f32) {
*self.ptr.add(idx) += val;
}
}
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx2,fma")]
pub(crate) unsafe fn hsum256(v: std::arch::x86_64::__m256) -> f32 {
use std::arch::x86_64::*;
let hi = _mm256_extractf128_ps(v, 1);
let lo = _mm256_castps256_ps128(v);
let s = _mm_add_ps(lo, hi);
let s = _mm_add_ps(s, _mm_movehl_ps(s, s));
let s = _mm_add_ps(s, _mm_shuffle_ps(s, s, 1));
_mm_cvtss_f32(s)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn row_parallel_matches_serial_exactly() {
let n = 1000;
let mut out = vec![0f32; n];
let work = |dst: &DstPtr, row: usize| {
let v = (row as f32) * 1.0000001 + 0.5;
unsafe { dst.write(row, v) };
};
{
let dst = DstPtr::new(&mut out);
for_each_row_with_threads(1, n, |row| work(&dst, row));
}
let serial = out.clone();
out.fill(0.0);
let dst = DstPtr::new(&mut out);
for_each_row_with_threads(8, n, |row| work(&dst, row));
assert_eq!(serial, out, "parallel rows must match serial bit-for-bit");
}
}