pub mod cuda;
pub mod dynlink;
pub mod metal;
pub mod rocm;
use crate::error::TptpResult;
use crate::memory::GpuBuffer;
#[derive(Debug, Clone)]
pub enum VendorBackend {
Cuda(cuda::CudaBackend),
Rocm(rocm::RocmBackend),
Metal(metal::MetalBackend),
None,
}
impl VendorBackend {
pub fn detect() -> Self {
if let Ok(backend) = cuda::CudaBackend::new() {
return VendorBackend::Cuda(backend);
}
if let Ok(backend) = rocm::RocmBackend::new() {
return VendorBackend::Rocm(backend);
}
#[cfg(target_os = "macos")]
{
if let Ok(backend) = metal::MetalBackend::new() {
return VendorBackend::Metal(backend);
}
}
VendorBackend::None
}
pub fn is_available(&self) -> bool {
!matches!(self, VendorBackend::None)
}
pub fn name(&self) -> &str {
match self {
VendorBackend::Cuda(_) => "CUDA",
VendorBackend::Rocm(_) => "ROCm",
VendorBackend::Metal(_) => "Metal",
VendorBackend::None => "None",
}
}
pub fn supports_gemm(&self) -> bool {
matches!(
self,
VendorBackend::Cuda(_) | VendorBackend::Rocm(_) | VendorBackend::Metal(_)
)
}
pub fn supports_attention(&self) -> bool {
matches!(self, VendorBackend::Cuda(_) | VendorBackend::Rocm(_))
}
pub fn supports_conv2d(&self) -> bool {
matches!(self, VendorBackend::Cuda(_) | VendorBackend::Rocm(_))
}
pub fn supports_conv3d(&self) -> bool {
matches!(self, VendorBackend::Cuda(_) | VendorBackend::Rocm(_))
}
pub fn supports_int8_gemm(&self) -> bool {
false
}
#[allow(clippy::too_many_arguments)]
pub fn quant_gemm(
&self,
_a_packed: &GpuBuffer<i8>,
_b_f32: &GpuBuffer<f32>,
_output: &mut GpuBuffer<f32>,
_scales: &GpuBuffer<f32>,
_zero_points: &GpuBuffer<i8>,
_m: usize,
_n: usize,
_k: usize,
_bits: u8,
) -> TptpResult<()> {
Err(crate::error::TptpError::unsupported(
"int8 GEMM vendor dispatch not yet implemented for this backend",
))
}
}
pub trait VendorLibrary: Send + Sync {
fn name(&self) -> &str;
fn is_available(&self) -> bool;
#[allow(clippy::too_many_arguments)]
fn gemm(
&self,
a: &GpuBuffer<f32>,
b: &GpuBuffer<f32>,
c: &mut GpuBuffer<f32>,
alpha: f32,
beta: f32,
m: usize,
n: usize,
k: usize,
) -> TptpResult<()>;
#[allow(clippy::too_many_arguments)]
fn attention(
&self,
q: &GpuBuffer<f32>,
k: &GpuBuffer<f32>,
v: &GpuBuffer<f32>,
output: &mut GpuBuffer<f32>,
scale: f32,
seq_len: usize,
d_k: usize,
) -> TptpResult<()>;
fn conv2d(
&self,
input: &GpuBuffer<f32>,
filter: &GpuBuffer<f32>,
output: &mut GpuBuffer<f32>,
strides: [u32; 2],
padding: [u32; 2],
) -> TptpResult<()>;
fn conv3d(
&self,
input: &GpuBuffer<f32>,
filter: &GpuBuffer<f32>,
output: &mut GpuBuffer<f32>,
strides: [u32; 3],
padding: [u32; 3],
) -> TptpResult<()>;
}
impl VendorLibrary for VendorBackend {
fn name(&self) -> &str {
self.name()
}
fn is_available(&self) -> bool {
self.is_available()
}
fn gemm(
&self,
a: &GpuBuffer<f32>,
b: &GpuBuffer<f32>,
c: &mut GpuBuffer<f32>,
alpha: f32,
beta: f32,
m: usize,
n: usize,
k: usize,
) -> TptpResult<()> {
match self {
VendorBackend::Cuda(backend) => backend.gemm(a, b, c, alpha, beta, m, n, k),
VendorBackend::Rocm(backend) => backend.gemm(a, b, c, alpha, beta, m, n, k),
VendorBackend::Metal(backend) => backend.gemm(a, b, c, alpha, beta, m, n, k),
VendorBackend::None => Err(crate::error::TptpError::vendor_unavailable(
"no vendor backend",
)),
}
}
fn attention(
&self,
q: &GpuBuffer<f32>,
k: &GpuBuffer<f32>,
v: &GpuBuffer<f32>,
output: &mut GpuBuffer<f32>,
scale: f32,
seq_len: usize,
d_k: usize,
) -> TptpResult<()> {
match self {
VendorBackend::Cuda(backend) => backend.attention(q, k, v, output, scale, seq_len, d_k),
VendorBackend::Rocm(backend) => backend.attention(q, k, v, output, scale, seq_len, d_k),
_ => Err(crate::error::TptpError::unsupported(
"attention not supported on this backend",
)),
}
}
fn conv2d(
&self,
input: &GpuBuffer<f32>,
filter: &GpuBuffer<f32>,
output: &mut GpuBuffer<f32>,
strides: [u32; 2],
padding: [u32; 2],
) -> TptpResult<()> {
match self {
VendorBackend::Cuda(backend) => backend.conv2d(input, filter, output, strides, padding),
VendorBackend::Rocm(backend) => backend.conv2d(input, filter, output, strides, padding),
_ => Err(crate::error::TptpError::unsupported(
"conv2d not supported on this backend",
)),
}
}
fn conv3d(
&self,
input: &GpuBuffer<f32>,
filter: &GpuBuffer<f32>,
output: &mut GpuBuffer<f32>,
strides: [u32; 3],
padding: [u32; 3],
) -> TptpResult<()> {
match self {
VendorBackend::Cuda(backend) => backend.conv3d(input, filter, output, strides, padding),
VendorBackend::Rocm(backend) => backend.conv3d(input, filter, output, strides, padding),
_ => Err(crate::error::TptpError::unsupported(
"conv3d not supported on this backend",
)),
}
}
}