use crate::device::WgpuDevice;
use crate::device::capabilities::{DeviceCapabilities, Fp64Strategy};
use crate::error::{BarracudaError, Result};
use bytemuck::{Pod, Zeroable};
use std::sync::Arc;
const WGSL_DF64_CORE: &str = include_str!("../../shaders/math/df64_core.wgsl");
const GEMM_SHADER_DF64: &str = include_str!("../../shaders/linalg/gemm_df64.wgsl");
#[repr(C)]
#[derive(Copy, Clone, Pod, Zeroable)]
struct GemmParams {
m: u32,
k: u32,
n: u32,
batch_size: u32,
alpha_lo: u32, alpha_hi: u32,
beta_lo: u32,
beta_hi: u32,
trans_a: u32, trans_b: u32, _pad0: u32,
_pad1: u32,
}
impl GemmParams {
fn new(
m: u32,
k: u32,
n: u32,
batch_size: u32,
alpha: f64,
beta: f64,
trans_a: bool,
trans_b: bool,
) -> Self {
let alpha_bits = alpha.to_bits();
let beta_bits = beta.to_bits();
Self {
m,
k,
n,
batch_size,
alpha_lo: alpha_bits as u32,
alpha_hi: (alpha_bits >> 32) as u32,
beta_lo: beta_bits as u32,
beta_hi: (beta_bits >> 32) as u32,
trans_a: u32::from(trans_a),
trans_b: u32::from(trans_b),
_pad0: 0,
_pad1: 0,
}
}
}
pub struct GemmF64;
impl GemmF64 {
pub const WGSL: &'static str = include_str!("../../shaders/linalg/gemm_f64.wgsl");
fn wgsl_shader() -> &'static str {
Self::WGSL
}
fn wgsl_shader_for_device(device: &WgpuDevice) -> String {
let caps = DeviceCapabilities::from_device(device);
let strategy = caps.fp64_strategy();
tracing::info!(?strategy, "GEMM: using {:?} FP64 strategy", strategy);
match strategy {
Fp64Strategy::Sovereign | Fp64Strategy::Native | Fp64Strategy::Concurrent => {
Self::wgsl_shader().to_string()
}
Fp64Strategy::Hybrid => format!("{WGSL_DF64_CORE}\n{GEMM_SHADER_DF64}"),
}
}
pub fn execute(
device: Arc<WgpuDevice>,
a: &[f64],
b: &[f64],
m: usize,
k: usize,
n: usize,
batch_size: usize,
) -> Result<Vec<f64>> {
Self::execute_gemm(device, a, b, m, k, n, batch_size, 1.0, 0.0)
}
pub fn execute_gemm(
device: Arc<WgpuDevice>,
a: &[f64],
b: &[f64],
m: usize,
k: usize,
n: usize,
batch_size: usize,
alpha: f64,
beta: f64,
) -> Result<Vec<f64>> {
Self::execute_gemm_ex(device, a, b, m, k, n, batch_size, alpha, beta, false, false)
}
pub fn execute_gemm_ex(
device: Arc<WgpuDevice>,
a: &[f64],
b: &[f64],
m: usize,
k: usize,
n: usize,
batch_size: usize,
alpha: f64,
beta: f64,
trans_a: bool,
trans_b: bool,
) -> Result<Vec<f64>> {
let expected_a = batch_size * m * k;
let expected_b = batch_size * k * n;
if a.len() != expected_a {
return Err(BarracudaError::InvalidInput {
message: format!("A: expected {} elements, got {}", expected_a, a.len()),
});
}
if b.len() != expected_b {
return Err(BarracudaError::InvalidInput {
message: format!("B: expected {} elements, got {}", expected_b, b.len()),
});
}
let c_size = batch_size * m * n;
let a_bytes: &[u8] = bytemuck::cast_slice(a);
let a_buffer = device
.device
.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some("GEMM A f64"),
contents: a_bytes,
usage: wgpu::BufferUsages::STORAGE,
});
let b_bytes: &[u8] = bytemuck::cast_slice(b);
let b_buffer = device
.device
.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some("GEMM B f64"),
contents: b_bytes,
usage: wgpu::BufferUsages::STORAGE,
});
let c_buffer = device.device.create_buffer(&wgpu::BufferDescriptor {
label: Some("GEMM C f64"),
size: (c_size * 8) as u64,
usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC,
mapped_at_creation: false,
});
let params = GemmParams::new(
m as u32,
k as u32,
n as u32,
batch_size as u32,
alpha,
beta,
trans_a,
trans_b,
);
let params_buffer = device.create_uniform_buffer("GEMM Params", ¶ms);
use crate::device::compute_pipeline::ComputeDispatch;
let src = Self::wgsl_shader_for_device(&device);
let wg_x = (n as u32).div_ceil(16);
let wg_y = (m as u32).div_ceil(16);
let wg_z = batch_size as u32;
ComputeDispatch::new(&device, "gemm_f64")
.shader(&src, "gemm_f64")
.f64()
.uniform(0, ¶ms_buffer)
.storage_read(1, &a_buffer)
.storage_read(2, &b_buffer)
.storage_rw(3, &c_buffer)
.dispatch(wg_x, wg_y, wg_z)
.submit()?;
device.read_f64_buffer(&c_buffer, c_size)
}
}
pub struct GemmCachedF64 {
device: Arc<WgpuDevice>,
pipeline: Arc<wgpu::ComputePipeline>,
bgl: Arc<wgpu::BindGroupLayout>,
b_buffer: Arc<wgpu::Buffer>,
pub k: usize,
pub n: usize,
pub batch_size: usize,
}
impl GemmCachedF64 {
pub fn new(
device: Arc<WgpuDevice>,
b: &[f64],
k: usize,
n: usize,
batch_size: usize,
) -> Result<Self> {
if b.len() != batch_size * k * n {
return Err(BarracudaError::InvalidInput {
message: format!(
"GemmCachedF64: B has {} elements, expected batch×K×N={}",
b.len(),
batch_size * k * n
),
});
}
let dev = &device;
let b_buffer = Arc::new(
dev.device
.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some("GemmCached B (weight matrix)"),
contents: bytemuck::cast_slice(b),
usage: wgpu::BufferUsages::STORAGE,
}),
);
let bgl = Arc::new(
dev.device
.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("GemmCached BGL"),
entries: &[
bgl_storage_ro(0),
bgl_storage_ro(1),
bgl_storage_rw(2),
bgl_uniform(3),
],
}),
);
let pl = dev
.device
.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("GemmCached PL"),
bind_group_layouts: &[&bgl],
immediate_size: 0,
});
let src = GemmF64::wgsl_shader_for_device(dev);
let shader = dev.compile_shader_f64(&src, Some("GemmCached"));
let pipeline = Arc::new(dev.device.create_compute_pipeline(
&wgpu::ComputePipelineDescriptor {
label: Some("GemmCached Pipeline"),
layout: Some(&pl),
module: &shader,
entry_point: Some("gemm_f64"),
cache: None,
compilation_options: Default::default(),
},
));
Ok(Self {
device,
pipeline,
bgl,
b_buffer,
k,
n,
batch_size,
})
}
pub fn multiply(&self, a: &[f64], m: usize) -> Result<Vec<f64>> {
let c_buf = self.execute_to_buffer(a, m)?;
let c_size = self.batch_size * m * self.n;
self.device.read_f64_buffer(&c_buf, c_size)
}
pub fn execute_to_buffer(&self, a: &[f64], m: usize) -> Result<wgpu::Buffer> {
let k = self.k;
let n = self.n;
let batch_size = self.batch_size;
if a.len() != batch_size * m * k {
return Err(BarracudaError::InvalidInput {
message: format!(
"GemmCachedF64::execute_to_buffer: A has {} elements, expected batch×M×K={}",
a.len(),
batch_size * m * k
),
});
}
let dev = &self.device;
let c_size = batch_size * m * n;
let params = GemmParams::new(
m as u32,
k as u32,
n as u32,
batch_size as u32,
1.0,
0.0,
false,
false,
);
let params_buf = dev
.device
.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some("GemmCached Params"),
contents: bytemuck::bytes_of(¶ms),
usage: wgpu::BufferUsages::UNIFORM,
});
let a_buf = dev
.device
.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some("GemmCached A"),
contents: bytemuck::cast_slice(a),
usage: wgpu::BufferUsages::STORAGE,
});
let c_buf = dev.device.create_buffer(&wgpu::BufferDescriptor {
label: Some("GemmCached C"),
size: (c_size * std::mem::size_of::<f64>()) as u64,
usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC,
mapped_at_creation: false,
});
let bg = dev.device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some("GemmCached BG"),
layout: &self.bgl,
entries: &[
wgpu::BindGroupEntry {
binding: 0,
resource: a_buf.as_entire_binding(),
},
wgpu::BindGroupEntry {
binding: 1,
resource: self.b_buffer.as_entire_binding(),
},
wgpu::BindGroupEntry {
binding: 2,
resource: c_buf.as_entire_binding(),
},
wgpu::BindGroupEntry {
binding: 3,
resource: params_buf.as_entire_binding(),
},
],
});
let tile = 16u32;
let wg_x = (m as u32).div_ceil(tile);
let wg_y = (n as u32).div_ceil(tile);
let wg_z = batch_size as u32;
let mut encoder = dev.create_encoder_guarded(&wgpu::CommandEncoderDescriptor {
label: Some("GemmCached Encoder"),
});
{
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("GemmCached Pass"),
timestamp_writes: None,
});
pass.set_pipeline(&self.pipeline);
pass.set_bind_group(0, Some(&bg), &[]);
pass.dispatch_workgroups(wg_x, wg_y, wg_z);
}
dev.submit_commands(Some(encoder.finish()));
Ok(c_buf)
}
}
fn bgl_storage_ro(idx: u32) -> wgpu::BindGroupLayoutEntry {
wgpu::BindGroupLayoutEntry {
binding: idx,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Storage { read_only: true },
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
}
}
fn bgl_storage_rw(idx: u32) -> wgpu::BindGroupLayoutEntry {
wgpu::BindGroupLayoutEntry {
binding: idx,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Storage { read_only: false },
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
}
}
fn bgl_uniform(idx: u32) -> wgpu::BindGroupLayoutEntry {
wgpu::BindGroupLayoutEntry {
binding: idx,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Uniform,
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
}
}
#[cfg(test)]
mod tests {
use super::*;
const GEMM_DF64_SHADER: &str = include_str!("../../shaders/linalg/gemm_df64.wgsl");
fn approx_eq(a: f64, b: f64, tol: f64) -> bool {
(a - b).abs() < tol
}
#[test]
fn gemm_params_layout() {
assert_eq!(std::mem::size_of::<GemmParams>(), 48);
}
#[test]
fn gemm_f64_shader_source_valid() {
let source = GemmF64::WGSL;
assert!(!source.is_empty());
assert!(source.contains("fn main") || source.contains("@compute"));
}
#[test]
fn gemm_df64_shader_source_valid() {
assert!(!GEMM_DF64_SHADER.is_empty());
assert!(GEMM_DF64_SHADER.contains("fn main") || GEMM_DF64_SHADER.contains("@compute"));
}
#[tokio::test]
async fn test_gemm_2x2() {
let Some(device) = crate::device::test_pool::get_test_device_if_f64_gpu_available().await
else {
return;
};
let a = vec![1.0_f64, 2.0, 3.0, 4.0];
let b = vec![5.0_f64, 6.0, 7.0, 8.0];
let c = GemmF64::execute(device, &a, &b, 2, 2, 2, 1).unwrap();
assert_eq!(c.len(), 4);
assert!(approx_eq(c[0], 19.0, 1e-10));
assert!(approx_eq(c[1], 22.0, 1e-10));
assert!(approx_eq(c[2], 43.0, 1e-10));
assert!(approx_eq(c[3], 50.0, 1e-10));
}
#[tokio::test]
async fn test_gemm_transpose_a() {
let Some(device) = crate::device::test_pool::get_test_device_if_f64_gpu_available().await
else {
return;
};
let a_stored = vec![1.0_f64, 3.0, 2.0, 4.0]; let b = vec![5.0_f64, 6.0, 7.0, 8.0];
let c = GemmF64::execute_gemm_ex(device, &a_stored, &b, 2, 2, 2, 1, 1.0, 0.0, true, false)
.unwrap();
assert_eq!(c.len(), 4);
assert!(approx_eq(c[0], 19.0, 1e-10));
assert!(approx_eq(c[1], 22.0, 1e-10));
assert!(approx_eq(c[2], 43.0, 1e-10));
assert!(approx_eq(c[3], 50.0, 1e-10));
}
#[tokio::test]
async fn test_gemm_transpose_b() {
let Some(device) = crate::device::test_pool::get_test_device_if_f64_gpu_available().await
else {
return;
};
let a = vec![1.0_f64, 2.0, 3.0, 4.0];
let b_stored = vec![5.0_f64, 7.0, 6.0, 8.0];
let c = GemmF64::execute_gemm_ex(device, &a, &b_stored, 2, 2, 2, 1, 1.0, 0.0, false, true)
.unwrap();
assert_eq!(c.len(), 4);
assert!(approx_eq(c[0], 19.0, 1e-10));
assert!(approx_eq(c[1], 22.0, 1e-10));
assert!(approx_eq(c[2], 43.0, 1e-10));
assert!(approx_eq(c[3], 50.0, 1e-10));
}
#[tokio::test]
async fn test_gemm_batched() {
let Some(device) = crate::device::test_pool::get_test_device_if_f64_gpu_available().await
else {
return;
};
let a = vec![
1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 2.0, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 2.0, ];
let b = vec![
1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, ];
let c = GemmF64::execute(device, &a, &b, 3, 3, 3, 2).unwrap();
assert_eq!(c.len(), 18);
assert!(approx_eq(c[0], 1.0, 1e-10));
assert!(approx_eq(c[4], 5.0, 1e-10));
assert!(approx_eq(c[9], 2.0, 1e-10));
assert!(approx_eq(c[13], 10.0, 1e-10));
}
}