use crate::error::Result;
use std::time::Duration;
pub mod harness;
pub mod operations;
pub mod report;
#[derive(Debug, Clone)]
pub struct BenchmarkConfig {
pub warmup_iterations: usize,
pub measurement_iterations: usize,
pub min_duration: Duration,
pub compare_cuda: bool,
pub precision: Precision,
pub random_seed: u64,
}
const DEFAULT_MIN_BENCH_DURATION_SECS: u64 = 5;
impl Default for BenchmarkConfig {
fn default() -> Self {
Self {
warmup_iterations: 10,
measurement_iterations: 100,
min_duration: Duration::from_secs(DEFAULT_MIN_BENCH_DURATION_SECS),
compare_cuda: true,
precision: Precision::FP32,
random_seed: 42,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Precision {
FP16,
FP32,
FP64,
INT8,
}
#[derive(Debug, Clone)]
pub struct BenchmarkResult {
pub operation: String,
pub hardware: String,
pub framework: Framework,
pub median_time: Duration,
pub mean_time: Duration,
pub std_dev: Duration,
pub min_time: Duration,
pub max_time: Duration,
pub throughput: f64,
pub bandwidth_gbps: f64,
pub tflops: f64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Framework {
BarraCuda,
CUDA,
PyTorchCUDA,
TensorFlowCUDA,
}
impl std::fmt::Display for Framework {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::BarraCuda => write!(f, "BarraCuda"),
Self::CUDA => write!(f, "CUDA"),
Self::PyTorchCUDA => write!(f, "PyTorch+CUDA"),
Self::TensorFlowCUDA => write!(f, "TensorFlow+CUDA"),
}
}
}
#[derive(Debug, Clone)]
pub struct ComparisonResult {
pub operation: String,
pub hardware: String,
pub barracuda: BenchmarkResult,
pub cuda: Option<BenchmarkResult>,
pub speedup: f64,
pub parity_percent: f64,
}
impl ComparisonResult {
#[must_use]
pub fn new(barracuda: BenchmarkResult, cuda: Option<BenchmarkResult>) -> Self {
let operation = barracuda.operation.clone();
let hardware = barracuda.hardware.clone();
let (speedup, parity_percent) = if let Some(ref cuda_result) = cuda {
let barracuda_secs = barracuda.median_time.as_secs_f64();
let cuda_secs = cuda_result.median_time.as_secs_f64();
let speedup = cuda_secs / barracuda_secs;
let parity = (cuda_secs / barracuda_secs) * 100.0;
(speedup, parity)
} else {
(0.0, 100.0)
};
Self {
operation,
hardware,
barracuda,
cuda,
speedup,
parity_percent,
}
}
#[must_use]
pub fn achieves_parity(&self, target_percent: f64) -> bool {
self.parity_percent >= target_percent
}
}
pub struct BenchmarkSuite {
config: BenchmarkConfig,
results: Vec<ComparisonResult>,
}
impl BenchmarkSuite {
#[must_use]
pub fn new(config: BenchmarkConfig) -> Self {
Self {
config,
results: Vec::new(),
}
}
pub async fn run_all(&mut self) -> Result<()> {
tracing::info!("Starting BarraCuda vs CUDA Benchmark Suite");
let hardware = self.discover_hardware().await?;
tracing::info!("Discovered {} compute device(s)", hardware.len());
for hw in &hardware {
tracing::info!(" {hw}");
}
self.benchmark_matrix_operations().await?;
self.benchmark_activations().await?;
self.benchmark_reductions().await?;
self.benchmark_convolutions().await?;
Ok(())
}
async fn discover_hardware(&self) -> Result<Vec<String>> {
use crate::device::WgpuDevice;
let mut hardware = vec!["CPU".to_string()];
if let Ok(device) = WgpuDevice::new().await {
let info = device.adapter_info();
hardware.push(format!("{} ({:?})", info.name, info.device_type));
} else {
}
Ok(hardware)
}
async fn benchmark_matrix_operations(&mut self) -> Result<()> {
tracing::info!("Matrix Operations");
let quick_sizes: &[(usize, usize, usize)] =
&[(128, 128, 128), (256, 256, 256), (512, 512, 512)];
for &(m, n, k) in quick_sizes {
let (barracuda, cuda) = operations::benchmark_matmul(&self.config, m, n, k).await?;
self.results.push(ComparisonResult::new(barracuda, cuda));
}
Ok(())
}
async fn benchmark_activations(&mut self) -> Result<()> {
tracing::info!("Activation Functions");
let activation_ops = ["ReLU", "GELU", "SiLU", "Sigmoid", "Tanh"];
let sizes = [10_000, 100_000, 1_000_000];
for op in &activation_ops {
for &size in &sizes {
let result = operations::benchmark_activation(&self.config, op, size).await?;
self.results.push(ComparisonResult::new(result, None));
}
}
Ok(())
}
async fn benchmark_reductions(&mut self) -> Result<()> {
tracing::info!("Reduction Operations");
let sizes = [10_000usize, 100_000, 1_000_000, 10_000_000];
for &size in &sizes {
let mut times = Vec::new();
let data: Vec<f32> = (0..size).map(|i| (i % 1000) as f32 * 0.001).collect();
for _ in 0..self.config.warmup_iterations {
let _: f32 = data.iter().sum();
}
for _ in 0..self.config.measurement_iterations {
let start = std::time::Instant::now();
let _sum: f32 = data.iter().sum();
times.push(start.elapsed());
}
times.sort();
let median_time = times[times.len() / 2];
let min_time = times[0];
let max_time = times[times.len() - 1];
let sum: Duration = times.iter().sum();
let mean_time = sum / times.len() as u32;
let mean_nanos = mean_time.as_nanos() as f64;
let variance: f64 = times
.iter()
.map(|t| {
let diff = t.as_nanos() as f64 - mean_nanos;
diff * diff
})
.sum::<f64>()
/ times.len() as f64;
let std_dev = Duration::from_nanos(variance.sqrt() as u64);
let throughput = size as f64 / median_time.as_secs_f64();
let bandwidth_gbps = (size as f64 * 4.0) / median_time.as_secs_f64() / 1e9;
let result = BenchmarkResult {
operation: format!("Sum [size={size}]"),
hardware: "CPU".to_string(),
framework: Framework::BarraCuda,
median_time,
min_time,
max_time,
mean_time,
std_dev,
throughput,
bandwidth_gbps,
tflops: throughput / 1e12,
};
self.results.push(ComparisonResult::new(result, None));
}
Ok(())
}
async fn benchmark_convolutions(&mut self) -> Result<()> {
tracing::info!("Convolution Operations");
let input_sizes = [1024usize, 4096, 16_384];
let kernel_size = 7;
for &input_size in &input_sizes {
let mut times = Vec::new();
let input: Vec<f32> = (0..input_size).map(|i| (i % 256) as f32 / 256.0).collect();
let kernel: Vec<f32> = (0..kernel_size)
.map(|i| i as f32 / kernel_size as f32)
.collect();
for _ in 0..self.config.warmup_iterations {
let _ = cpu_conv1d(&input, &kernel);
}
for _ in 0..self.config.measurement_iterations {
let start = std::time::Instant::now();
let _ = cpu_conv1d(&input, &kernel);
times.push(start.elapsed());
}
times.sort();
let median_time = times[times.len() / 2];
let min_time = times[0];
let max_time = times[times.len() - 1];
let sum: Duration = times.iter().sum();
let mean_time = sum / times.len() as u32;
let mean_nanos = mean_time.as_nanos() as f64;
let variance: f64 = times
.iter()
.map(|t| {
let diff = t.as_nanos() as f64 - mean_nanos;
diff * diff
})
.sum::<f64>()
/ times.len() as f64;
let std_dev = Duration::from_nanos(variance.sqrt() as u64);
let output_size = input_size - kernel_size + 1;
let flops = (output_size * kernel_size) as f64; let throughput = flops / median_time.as_secs_f64();
let bandwidth_gbps = ((input_size + kernel_size + output_size) as f64 * 4.0)
/ median_time.as_secs_f64()
/ 1e9;
let result = BenchmarkResult {
operation: format!("Conv1D [in={input_size}, k={kernel_size}]"),
hardware: "CPU".to_string(),
framework: Framework::BarraCuda,
median_time,
min_time,
max_time,
mean_time,
std_dev,
throughput,
bandwidth_gbps,
tflops: throughput / 1e12,
};
self.results.push(ComparisonResult::new(result, None));
}
Ok(())
}
#[must_use]
pub fn results(&self) -> &[ComparisonResult] {
&self.results
}
#[must_use]
pub fn summary(&self) -> BenchmarkSummary {
let total_ops = self.results.len();
let ops_with_parity_90 = self
.results
.iter()
.filter(|r| r.achieves_parity(90.0))
.count();
let ops_with_parity_95 = self
.results
.iter()
.filter(|r| r.achieves_parity(95.0))
.count();
let ops_with_parity_98 = self
.results
.iter()
.filter(|r| r.achieves_parity(98.0))
.count();
let mean_parity = if total_ops > 0 {
self.results.iter().map(|r| r.parity_percent).sum::<f64>() / total_ops as f64
} else {
0.0
};
BenchmarkSummary {
total_operations: total_ops,
ops_at_90_parity: ops_with_parity_90,
ops_at_95_parity: ops_with_parity_95,
ops_at_98_parity: ops_with_parity_98,
mean_parity_percent: mean_parity,
}
}
}
#[derive(Debug, Clone)]
pub struct BenchmarkSummary {
pub total_operations: usize,
pub ops_at_90_parity: usize,
pub ops_at_95_parity: usize,
pub ops_at_98_parity: usize,
pub mean_parity_percent: f64,
}
impl std::fmt::Display for BenchmarkSummary {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
writeln!(f, "📊 Benchmark Summary")?;
writeln!(f, "━━━━━━━━━━━━━━━━━━━━━━━━━━━━")?;
writeln!(f, "Total Operations: {}", self.total_operations)?;
writeln!(
f,
"≥90% Parity: {} ({:.1}%)",
self.ops_at_90_parity,
(self.ops_at_90_parity as f64 / self.total_operations as f64) * 100.0
)?;
writeln!(
f,
"≥95% Parity: {} ({:.1}%)",
self.ops_at_95_parity,
(self.ops_at_95_parity as f64 / self.total_operations as f64) * 100.0
)?;
writeln!(
f,
"≥98% Parity: {} ({:.1}%)",
self.ops_at_98_parity,
(self.ops_at_98_parity as f64 / self.total_operations as f64) * 100.0
)?;
writeln!(f, "Mean Parity: {:.2}%", self.mean_parity_percent)?;
Ok(())
}
}
#[inline(never)]
fn cpu_conv1d(input: &[f32], kernel: &[f32]) -> Vec<f32> {
let n = input.len();
let k = kernel.len();
if k > n {
return vec![];
}
let output_size = n - k + 1;
let mut output = vec![0.0f32; output_size];
for i in 0..output_size {
let mut sum = 0.0f32;
for j in 0..k {
sum = input[i + j].mul_add(kernel[j], sum);
}
output[i] = sum;
}
output
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Duration;
fn make_result(op: &str, hw: &str, framework: Framework, median_ms: f64) -> BenchmarkResult {
let median = Duration::from_secs_f64(median_ms / 1000.0);
BenchmarkResult {
operation: op.to_string(),
hardware: hw.to_string(),
framework,
median_time: median,
mean_time: median,
std_dev: Duration::from_millis(1),
min_time: median,
max_time: median,
throughput: 1000.0 / median_ms,
bandwidth_gbps: 0.0,
tflops: 0.0,
}
}
#[test]
fn test_benchmark_config_default() {
let config = BenchmarkConfig::default();
assert_eq!(config.warmup_iterations, 10);
assert_eq!(config.measurement_iterations, 100);
assert_eq!(config.precision, Precision::FP32);
}
#[test]
fn test_framework_display() {
assert_eq!(Framework::BarraCuda.to_string(), "BarraCuda");
assert_eq!(Framework::CUDA.to_string(), "CUDA");
}
#[test]
fn test_comparison_result_new_with_cuda() {
let barracuda = make_result("MatMul", "CPU", Framework::BarraCuda, 10.0);
let cuda = make_result("MatMul", "CPU", Framework::CUDA, 5.0);
let cmp = ComparisonResult::new(barracuda, Some(cuda));
assert!((cmp.speedup - 0.5).abs() < 1e-6);
assert!((cmp.parity_percent - 50.0).abs() < 1e-6);
}
#[test]
fn test_comparison_result_new_without_cuda() {
let barracuda = make_result("MatMul", "CPU", Framework::BarraCuda, 10.0);
let cmp = ComparisonResult::new(barracuda, None);
assert_eq!(cmp.parity_percent, 100.0);
assert_eq!(cmp.speedup, 0.0);
}
#[test]
fn test_achieves_parity() {
let barracuda = make_result("MatMul", "CPU", Framework::BarraCuda, 10.0);
let cuda_95 = make_result("MatMul", "CPU", Framework::CUDA, 9.5);
let cuda_80 = make_result("MatMul", "CPU", Framework::CUDA, 8.0);
let cmp_95 = ComparisonResult::new(barracuda.clone(), Some(cuda_95));
let cmp_80 = ComparisonResult::new(barracuda, Some(cuda_80));
assert!(cmp_95.achieves_parity(90.0));
assert!(!cmp_80.achieves_parity(90.0));
}
#[test]
fn test_benchmark_suite_new_and_summary() {
let config = BenchmarkConfig::default();
let suite = BenchmarkSuite::new(config);
let summary = suite.summary();
assert_eq!(summary.total_operations, 0);
assert_eq!(summary.ops_at_90_parity, 0);
assert_eq!(summary.ops_at_95_parity, 0);
assert_eq!(summary.ops_at_98_parity, 0);
assert!((summary.mean_parity_percent - 0.0).abs() < 1e-6);
}
#[test]
fn test_benchmark_summary_display() {
let summary = BenchmarkSummary {
total_operations: 5,
ops_at_90_parity: 4,
ops_at_95_parity: 3,
ops_at_98_parity: 2,
mean_parity_percent: 92.5,
};
let s = summary.to_string();
assert!(s.contains("Benchmark Summary"));
assert!(s.contains("Total Operations: 5"));
assert!(s.contains("≥90% Parity"));
assert!(s.contains("Mean Parity: 92.50%"));
}
#[test]
fn test_precision_variants() {
assert_eq!(Precision::FP16, Precision::FP16);
assert_eq!(Precision::FP32, Precision::FP32);
assert_eq!(Precision::FP64, Precision::FP64);
assert_eq!(Precision::INT8, Precision::INT8);
assert!(Precision::FP32 != Precision::FP64);
}
#[test]
fn test_cpu_conv1d() {
let input = vec![1.0f32, 2.0, 3.0, 4.0, 5.0];
let kernel = vec![1.0f32, 0.0, 1.0];
let out = super::cpu_conv1d(&input, &kernel);
assert_eq!(out.len(), 3);
assert!((out[0] - 4.0).abs() < 1e-6);
assert!((out[1] - 6.0).abs() < 1e-6);
assert!((out[2] - 8.0).abs() < 1e-6);
}
}