use crate::Tensor;
use std::error::Error;
use std::fmt;
const DISABLED_MESSAGE: &str =
"GPU support is not compiled in; rebuild with `--features gpu` to use the GPU backend";
#[derive(Debug)]
pub enum GpuError {
Initialization(String),
InvalidShape(String),
Map(String),
Synchronization(String),
}
impl fmt::Display for GpuError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Initialization(message) => {
write!(formatter, "GPU initialization failed: {message}")
}
Self::InvalidShape(message) => write!(formatter, "invalid GPU tensor shape: {message}"),
Self::Map(message) => write!(formatter, "GPU readback failed: {message}"),
Self::Synchronization(message) => {
write!(formatter, "GPU synchronization failed: {message}")
}
}
}
}
impl Error for GpuError {}
fn disabled() -> GpuError {
GpuError::Initialization(DISABLED_MESSAGE.into())
}
#[derive(Clone)]
pub struct GpuContext {
_private: (),
}
#[derive(Clone)]
pub struct GpuTensor {
_private: (),
}
impl GpuTensor {
pub fn shape(&self) -> &[usize] {
&[]
}
pub fn len(&self) -> usize {
0
}
pub fn is_empty(&self) -> bool {
true
}
pub fn to_vec(&self) -> Result<Vec<f32>, GpuError> {
Err(disabled())
}
pub(crate) fn reshape(&self, _shape: Vec<usize>) -> Self {
self.clone()
}
pub fn to_tensor(&self, _requires_grad: bool) -> Result<Tensor, GpuError> {
Err(disabled())
}
}
#[derive(Clone)]
pub struct GpuIndexTensor {
_private: (),
}
#[derive(Clone)]
pub struct GpuMatmulTensor {
_private: (),
}
#[derive(Clone)]
pub struct GpuQuantizedLinearWeight {
_private: (),
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum GpuFloatFormat {
#[default]
Fp32,
Fp16,
Bf16,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct GpuFloatFormatSupport {
pub fp32: bool,
pub fp16: bool,
pub bf16: bool,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct GpuTiming {
pub kernel: Option<std::time::Duration>,
pub host_submission: std::time::Duration,
pub synchronization: std::time::Duration,
}
impl GpuTiming {
pub fn host_total(&self) -> std::time::Duration {
self.host_submission + self.synchronization
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct GpuAdapterLimits {
pub max_buffer_size: u64,
pub max_storage_buffer_binding_size: u64,
pub max_compute_workgroup_storage_size: u32,
pub max_compute_invocations_per_workgroup: u32,
pub max_compute_workgroup_size_x: u32,
pub max_bind_groups: u32,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct GpuMemoryStats {
pub live_tensor_buffers: usize,
pub live_tensor_bytes: usize,
pub peak_tensor_bytes: usize,
pub pooled_staging_buffers: usize,
pub pooled_staging_bytes: usize,
pub pooled_output_buffers: usize,
pub pooled_output_bytes: usize,
pub bind_group_cache_entries: usize,
pub output_buffer_allocations: usize,
pub output_buffer_reuses: usize,
pub bind_group_creations: usize,
pub bind_group_cache_hits: usize,
pub uploads: usize,
pub upload_bytes: usize,
pub readbacks: usize,
pub readback_bytes: usize,
}
pub const LOW_MEMORY_BUDGET_THRESHOLD_BYTES: usize = 8 * 1024 * 1024 * 1024;
const LOW_OUTPUT_POOL_BYTES: usize = 16 * 1024 * 1024;
const LOW_BIND_GROUP_CACHE_ENTRIES: usize = 128;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct GpuCacheLimits {
pub output_buffer_bytes: usize,
pub bind_group_entries: usize,
}
impl Default for GpuCacheLimits {
fn default() -> Self {
Self {
output_buffer_bytes: 64 * 1024 * 1024,
bind_group_entries: 512,
}
}
}
impl GpuCacheLimits {
pub fn for_memory_budget(budget_bytes: Option<usize>) -> Self {
match budget_bytes {
Some(bytes) if bytes > 0 && bytes <= LOW_MEMORY_BUDGET_THRESHOLD_BYTES => Self {
output_buffer_bytes: LOW_OUTPUT_POOL_BYTES,
bind_group_entries: LOW_BIND_GROUP_CACHE_ENTRIES,
},
_ => Self::default(),
}
}
pub fn is_low_memory(&self) -> bool {
*self
== Self {
output_buffer_bytes: LOW_OUTPUT_POOL_BYTES,
bind_group_entries: LOW_BIND_GROUP_CACHE_ENTRIES,
}
}
}
#[derive(Clone)]
pub enum ExecutionDevice {
Cpu { fallback_reason: Option<String> },
Gpu(GpuContext),
}
impl ExecutionDevice {
pub fn auto() -> Self {
Self::from_gpu_result(GpuContext::new())
}
pub fn cpu() -> Self {
Self::Cpu {
fallback_reason: None,
}
}
pub fn from_gpu_result(result: Result<GpuContext, GpuError>) -> Self {
match result {
Ok(context) => Self::Gpu(context),
Err(error) => Self::Cpu {
fallback_reason: Some(error.to_string()),
},
}
}
pub fn device(&self) -> crate::Device {
match self {
Self::Cpu { .. } => crate::Device::Cpu,
Self::Gpu(_) => crate::Device::Gpu,
}
}
pub fn gpu(&self) -> Option<&GpuContext> {
match self {
Self::Cpu { .. } => None,
Self::Gpu(context) => Some(context),
}
}
pub fn fallback_reason(&self) -> Option<&str> {
match self {
Self::Cpu { fallback_reason } => fallback_reason.as_deref(),
Self::Gpu(_) => None,
}
}
}
impl GpuContext {
pub fn global() -> Result<Self, GpuError> {
Err(disabled())
}
pub fn new() -> Result<Self, GpuError> {
Err(disabled())
}
pub fn create_devices(_count: usize) -> Result<Vec<Self>, GpuError> {
Err(disabled())
}
pub fn adapter_name(&self) -> &str {
"gpu-disabled"
}
pub fn set_memory_budget_bytes(&self, _budget_bytes: Option<usize>) {}
pub fn apply_memory_budget(&self, _budget_bytes: Option<usize>) -> Result<(), GpuError> {
Err(disabled())
}
pub fn memory_budget(&self) -> crate::memory::DeviceBudget {
crate::memory::DeviceBudget::cpu(None)
}
pub fn memory_stats(&self) -> Result<GpuMemoryStats, GpuError> {
Err(disabled())
}
pub fn adapter_limits(&self) -> GpuAdapterLimits {
GpuAdapterLimits::default()
}
pub fn max_buffer_bytes(&self) -> usize {
0
}
pub fn float_format_support(&self) -> GpuFloatFormatSupport {
GpuFloatFormatSupport::default()
}
pub fn trim_dispatch_caches(&self) -> Result<(), GpuError> {
Err(disabled())
}
pub fn set_cache_limits(&self, _limits: GpuCacheLimits) -> Result<(), GpuError> {
Err(disabled())
}
}
impl Tensor {
pub fn to_cpu(&self) -> Result<Tensor, GpuError> {
Ok(self.clone())
}
pub fn backward_gpu(&self, _context: &GpuContext) -> Result<(), GpuError> {
Err(disabled())
}
pub fn gpu_scale(&self, _factor: f32, _context: &GpuContext) -> Result<Tensor, GpuError> {
Err(disabled())
}
}