use et_abi::{GEMM_TILE_N, GemmArgs, TENSOR_ALIGN};
use crate::device::Device;
use crate::error::{Error, Result};
use crate::transport::Transport;
use crate::device::LoadedKernel;
#[derive(Debug, Clone, PartialEq)]
pub enum GemmError {
UnalignedPointer { matrix: &'static str, addr: u64 },
UnalignedStride { matrix: &'static str, lda: u32 },
#[deprecated(
since = "0.4.0",
note = "sgemm now accepts arbitrary N; \
this error variant is never returned"
)]
NNotMultipleOfTileN { n: u32 },
ZeroDimension,
UnsupportedScaling { alpha: f32, beta: f32 },
}
impl GemmError {
fn into_limit(self) -> Error {
Error::Limit(match self {
GemmError::UnalignedPointer { matrix, addr } => format!(
"sGEMM: matrix {matrix} pointer {addr:#x} is not \
{TENSOR_ALIGN}-byte aligned"
),
GemmError::UnalignedStride { matrix, lda } => format!(
"sGEMM: leading dimension of {matrix} ({lda} bytes) is not a \
multiple of {TENSOR_ALIGN}"
),
#[allow(deprecated)]
GemmError::NNotMultipleOfTileN { n } => format!(
"sGEMM: N={n} is not a multiple of {GEMM_TILE_N} (deprecated \
constraint; sgemm now accepts arbitrary N)"
),
GemmError::ZeroDimension => "sGEMM: M, N, and K must all be >= 1".into(),
GemmError::UnsupportedScaling { alpha, beta } => format!(
"sGEMM: alpha={alpha}, beta={beta} not supported in v0.1 \
(only alpha=1.0, beta=0.0)"
),
})
}
}
pub fn alloc_tensor_matrix<Tr: Transport>(
dev: &Device<Tr>,
rows: usize,
cols: usize,
) -> Result<(u64, u32)> {
let row_bytes = (cols * 4).next_multiple_of(TENSOR_ALIGN);
let total = (rows * row_bytes) as u64;
let region = dev.alloc(total)?;
let addr = region.addr;
if addr % TENSOR_ALIGN as u64 != 0 {
return Err(Error::Limit(format!(
"device allocator returned address {addr:#x} with insufficient \
alignment for tensor operations (need {TENSOR_ALIGN} bytes)"
)));
}
Ok((addr, row_bytes as u32))
}
#[allow(clippy::too_many_arguments)]
pub fn sgemm<Tr: Transport>(
dev: &Device<Tr>,
kernel: &LoadedKernel,
m: u32,
n: u32,
k: u32,
alpha: f32,
a: u64,
lda: u32,
b: u64,
ldb: u32,
beta: f32,
c: u64,
ldc: u32,
n_shires: u32,
) -> Result<()> {
if m == 0 || n == 0 || k == 0 {
return Err(GemmError::ZeroDimension.into_limit());
}
if !(1..=63).contains(&n_shires) {
return Err(Error::Limit(format!(
"sGEMM: n_shires ({n_shires}) must be in 1..=63"
)));
}
if alpha != 1.0 || beta != 0.0 {
return Err(GemmError::UnsupportedScaling { alpha, beta }.into_limit());
}
for (name, addr) in [("A", a), ("B", b), ("C", c)] {
if addr % TENSOR_ALIGN as u64 != 0 {
return Err(GemmError::UnalignedPointer { matrix: name, addr }.into_limit());
}
}
for (name, stride) in [("A", lda), ("B", ldb), ("C", ldc)] {
if !(stride as usize).is_multiple_of(TENSOR_ALIGN) {
return Err(GemmError::UnalignedStride {
matrix: name,
lda: stride,
}
.into_limit());
}
}
let args = GemmArgs {
a,
b,
c,
n_shires: n_shires as u64,
m,
n,
k,
lda,
ldb,
ldc,
alpha,
beta,
};
let shire_mask = (1_u64 << n_shires) - 1;
dev.launch_spmd(kernel, shire_mask, &args)?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use et_abi::DeviceArgs;
fn dummy_args() -> GemmArgs {
GemmArgs {
a: 0x0080_0000_0000,
b: 0x0080_0100_0000,
c: 0x0080_0200_0000,
n_shires: 1,
m: 32,
n: 16,
k: 16,
lda: 64,
ldb: 64,
ldc: 64,
alpha: 1.0,
beta: 0.0,
}
}
#[test]
fn gemmargs_size_consistent() {
assert_eq!(core::mem::size_of::<GemmArgs>(), 64);
}
#[test]
fn validates_alpha_beta() {
let err = GemmError::UnsupportedScaling {
alpha: 2.0,
beta: 0.0,
}
.into_limit();
assert!(matches!(err, Error::Limit(_)));
}
#[test]
#[allow(deprecated)]
fn n_not_multiple_error_formats() {
let err = GemmError::NNotMultipleOfTileN { n: 7 }.into_limit();
assert!(matches!(err, Error::Limit(ref s) if s.contains("N=7")));
}
#[test]
fn validates_pointer_alignment() {
let err = GemmError::UnalignedPointer {
matrix: "A",
addr: 0x0080_0000_0001,
}
.into_limit();
assert!(matches!(err, Error::Limit(ref s) if s.contains("matrix A")));
}
#[test]
fn validates_stride_alignment() {
let err = GemmError::UnalignedStride {
matrix: "B",
lda: 63,
}
.into_limit();
assert!(matches!(err, Error::Limit(ref s) if s.contains("B")));
}
#[test]
fn gemmargs_roundtrip() {
let a = dummy_args();
let b = unsafe { GemmArgs::from_ptr(a.as_bytes().as_ptr()) };
assert_eq!(*b, a);
}
#[test]
fn validates_zero_shires() {
let args = dummy_args();
let err = Error::Limit("sGEMM: n_shires (0) must be in 1..=63".into());
assert!(matches!(err, Error::Limit(ref s) if s.contains("n_shires")));
let overflow = Error::Limit("sGEMM: n_shires (64) must be in 1..=63".into());
assert!(matches!(overflow, Error::Limit(ref s) if s.contains("n_shires")));
let _ = args; }
}