use super::budget::Budget;
use super::mem;
use super::rate::max_rate;
use super::report::CpuMetrics;
use std::time::Duration;
const N_MT: usize = if cfg!(debug_assertions) { 256 } else { 1024 };
const N_1T: usize = if cfg!(debug_assertions) { 128 } else { 512 };
const WINDOW: Duration = Duration::from_millis(300);
macro_rules! gemm {
($name:ident, $t:ty, $mm:path) => {
pub(crate) fn $name(a: &[$t], b: &[$t], c: &mut [$t], n: usize) {
assert!(a.len() >= n * n && b.len() >= n * n && c.len() >= n * n);
let s = n as isize;
unsafe {
$mm(
n,
n,
n,
1.0,
a.as_ptr(),
s,
1,
b.as_ptr(),
s,
1,
0.0,
c.as_mut_ptr(),
s,
1,
);
}
}
};
}
gemm!(sgemm, f32, matrixmultiply::sgemm);
gemm!(dgemm, f64, matrixmultiply::dgemm);
struct Mats<T> {
a: Vec<T>,
b: Vec<T>,
c: Vec<T>,
n: usize,
}
macro_rules! mats {
($name:ident, $t:ty, $kernel:ident) => {
fn $name(n: usize) -> impl Fn(usize) -> Mats<$t> + Sync {
move |_| Mats {
a: vec![1.0; n * n],
b: vec![0.5; n * n],
c: vec![0.0; n * n],
n,
}
}
impl Mats<$t> {
fn run_once(&mut self) -> f64 {
$kernel(&self.a, &self.b, &mut self.c, self.n);
std::hint::black_box(&self.c);
2.0 * (self.n as f64).powi(3)
}
}
};
}
mats!(sgemm_mats, f32, sgemm);
mats!(dgemm_mats, f64, dgemm);
pub struct CpuProbe {
pub metrics: CpuMetrics,
pub notes: Vec<String>,
}
fn panic_note(phase: &str, panicked: usize) -> Option<String> {
(panicked > 0).then(|| format!("cpu: {panicked} worker(s) panicked during {phase}"))
}
fn gflops<T>(
b: &Budget,
threads: usize,
phase: &str,
notes: &mut Vec<String>,
setup: impl Fn(usize) -> Mats<T> + Sync,
run: impl Fn(&mut Mats<T>) -> f64 + Sync,
) -> f64 {
let o = max_rate(b, threads, WINDOW, setup, run);
notes.extend(panic_note(phase, o.panicked));
o.rate.map_or(0.0, |r| r / 1e9)
}
pub fn probe(b: &Budget) -> Result<CpuProbe, String> {
if b.cancelled() {
return Err("cancelled".into());
}
let threads = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1);
let mut notes = vec![];
let gflops_f32 = gflops(
&b.slice(0.35),
threads,
"fp32 GEMM",
&mut notes,
sgemm_mats(N_MT),
Mats::<f32>::run_once,
);
if b.cancelled() {
return Err("cancelled".into());
}
let gflops_f64 = gflops(
&b.slice(0.4),
threads,
"fp64 GEMM",
&mut notes,
dgemm_mats(N_MT),
Mats::<f64>::run_once,
);
if b.cancelled() {
return Err("cancelled".into());
}
let gflops_f32_1t = gflops(
&b.slice(0.5),
1,
"single-thread GEMM",
&mut notes,
sgemm_mats(N_1T),
Mats::<f32>::run_once,
);
let triad = mem::triad_gbps(b)?;
notes.extend(triad.note);
Ok(CpuProbe {
metrics: CpuMetrics {
gflops_f32,
gflops_f64,
gflops_f32_1t,
mem_bw_gbps: triad.gbps,
},
notes,
})
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::AtomicBool;
use std::sync::Arc;
use std::time::{Duration, Instant};
#[test]
fn sgemm_is_numerically_correct() {
let a = vec![1.0f32, 2., 3., 4., 5., 6., 7., 8., 9.];
let id = vec![1.0f32, 0., 0., 0., 1., 0., 0., 0., 1.];
let mut c = vec![0.0f32; 9];
sgemm(&a, &id, &mut c, 3);
assert_eq!(c, a);
sgemm(&a, &a, &mut c, 3);
assert_eq!(c, vec![30.0, 36., 42., 66., 81., 96., 102., 126., 150.]);
let a64: Vec<f64> = a.iter().map(|&x| x as f64).collect();
let mut c64 = vec![0.0f64; 9];
dgemm(&a64, &a64, &mut c64, 3);
assert_eq!(c64[0], 30.0);
assert_eq!(c64[8], 150.0);
}
#[test]
fn a_panicked_worker_is_named_in_a_note() {
assert_eq!(panic_note("fp32 GEMM", 0), None);
let n = panic_note("fp32 GEMM", 2).unwrap();
assert!(
n.contains("2 worker(s) panicked") && n.contains("fp32 GEMM"),
"{n}"
);
}
#[test]
fn cpu_probe_reports_positive_gflops_within_budget() {
let b = Budget::new(Duration::from_secs(6), Arc::new(AtomicBool::new(false)));
let t = Instant::now();
let m = probe(&b).unwrap().metrics;
assert!(
m.gflops_f32 > 0.0
&& m.gflops_f64 > 0.0
&& m.gflops_f32_1t > 0.0
&& m.mem_bw_gbps > 0.0
);
assert!(t.elapsed() < Duration::from_secs(12));
}
#[test]
fn cpu_probe_returns_early_when_cancelled() {
let c = Arc::new(AtomicBool::new(true));
let t = Instant::now();
let _ = probe(&Budget::new(Duration::from_secs(60), c));
assert!(t.elapsed() < Duration::from_secs(2));
}
}