oxidelm 0.2.0

Bare-metal pure-Rust Transformer engine with reverse-mode autograd, BPE tokenization, and wgpu acceleration
Documentation
//! Minimal CPU-only stand-in for [`crate::gpu`] used when the `gpu` Cargo
//! feature is disabled.
//!
//! This module keeps the crate's public type names stable (`GpuContext`,
//! `GpuTensor`, `ExecutionDevice`, ...) so the rest of the codebase does not
//! need `cfg` gates just to name these types. Every constructor that would
//! normally talk to a real GPU adapter fails with [`GpuError::Initialization`],
//! and [`ExecutionDevice`] always resolves to CPU. Actual GPU compute methods
//! on `Tensor` / `CausalLM` / etc. live behind `#[cfg(feature = "gpu")]` next
//! to their CPU counterparts and are simply absent from the binary here.

use crate::Tensor;
use std::error::Error;
use std::fmt;

const DISABLED_MESSAGE: &str =
    "GPU support is not compiled in; rebuild with `--features gpu` to use the GPU backend";

/// Mirrors [`crate::gpu::GpuError`]'s shape so call sites do not need cfg gates.
#[derive(Debug)]
pub enum GpuError {
    Initialization(String),
    InvalidShape(String),
    Map(String),
    Synchronization(String),
}

impl fmt::Display for GpuError {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::Initialization(message) => {
                write!(formatter, "GPU initialization failed: {message}")
            }
            Self::InvalidShape(message) => write!(formatter, "invalid GPU tensor shape: {message}"),
            Self::Map(message) => write!(formatter, "GPU readback failed: {message}"),
            Self::Synchronization(message) => {
                write!(formatter, "GPU synchronization failed: {message}")
            }
        }
    }
}

impl Error for GpuError {}

fn disabled() -> GpuError {
    GpuError::Initialization(DISABLED_MESSAGE.into())
}

/// Opaque handle; never constructible without the `gpu` feature.
#[derive(Clone)]
pub struct GpuContext {
    _private: (),
}

/// Opaque GPU-resident tensor handle (never actually created without `gpu`).
#[derive(Clone)]
pub struct GpuTensor {
    _private: (),
}

impl GpuTensor {
    pub fn shape(&self) -> &[usize] {
        &[]
    }

    pub fn len(&self) -> usize {
        0
    }

    pub fn is_empty(&self) -> bool {
        true
    }

    pub fn to_vec(&self) -> Result<Vec<f32>, GpuError> {
        Err(disabled())
    }

    pub(crate) fn reshape(&self, _shape: Vec<usize>) -> Self {
        self.clone()
    }

    pub fn to_tensor(&self, _requires_grad: bool) -> Result<Tensor, GpuError> {
        Err(disabled())
    }
}

/// Opaque GPU-resident integer tensor handle.
#[derive(Clone)]
pub struct GpuIndexTensor {
    _private: (),
}

/// Opaque typed GPU matmul operand handle.
#[derive(Clone)]
pub struct GpuMatmulTensor {
    _private: (),
}

/// Opaque GPU-resident quantized linear weight handle.
#[derive(Clone)]
pub struct GpuQuantizedLinearWeight {
    _private: (),
}

#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum GpuFloatFormat {
    #[default]
    Fp32,
    Fp16,
    Bf16,
}

#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct GpuFloatFormatSupport {
    pub fp32: bool,
    pub fp16: bool,
    pub bf16: bool,
}

#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct GpuTiming {
    pub kernel: Option<std::time::Duration>,
    pub host_submission: std::time::Duration,
    pub synchronization: std::time::Duration,
}

impl GpuTiming {
    pub fn host_total(&self) -> std::time::Duration {
        self.host_submission + self.synchronization
    }
}

#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct GpuAdapterLimits {
    pub max_buffer_size: u64,
    pub max_storage_buffer_binding_size: u64,
    pub max_compute_workgroup_storage_size: u32,
    pub max_compute_invocations_per_workgroup: u32,
    pub max_compute_workgroup_size_x: u32,
    pub max_bind_groups: u32,
}

#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct GpuMemoryStats {
    pub live_tensor_buffers: usize,
    pub live_tensor_bytes: usize,
    pub peak_tensor_bytes: usize,
    pub pooled_staging_buffers: usize,
    pub pooled_staging_bytes: usize,
    pub pooled_output_buffers: usize,
    pub pooled_output_bytes: usize,
    pub bind_group_cache_entries: usize,
    pub output_buffer_allocations: usize,
    pub output_buffer_reuses: usize,
    pub bind_group_creations: usize,
    pub bind_group_cache_hits: usize,
    pub uploads: usize,
    pub upload_bytes: usize,
    pub readbacks: usize,
    pub readback_bytes: usize,
}

/// Budgets at or below this use smaller dispatch-cache defaults (8 GiB).
pub const LOW_MEMORY_BUDGET_THRESHOLD_BYTES: usize = 8 * 1024 * 1024 * 1024;
const LOW_OUTPUT_POOL_BYTES: usize = 16 * 1024 * 1024;
const LOW_BIND_GROUP_CACHE_ENTRIES: usize = 128;

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct GpuCacheLimits {
    pub output_buffer_bytes: usize,
    pub bind_group_entries: usize,
}

impl Default for GpuCacheLimits {
    fn default() -> Self {
        Self {
            output_buffer_bytes: 64 * 1024 * 1024,
            bind_group_entries: 512,
        }
    }
}

impl GpuCacheLimits {
    pub fn for_memory_budget(budget_bytes: Option<usize>) -> Self {
        match budget_bytes {
            Some(bytes) if bytes > 0 && bytes <= LOW_MEMORY_BUDGET_THRESHOLD_BYTES => Self {
                output_buffer_bytes: LOW_OUTPUT_POOL_BYTES,
                bind_group_entries: LOW_BIND_GROUP_CACHE_ENTRIES,
            },
            _ => Self::default(),
        }
    }

    pub fn is_low_memory(&self) -> bool {
        *self
            == Self {
                output_buffer_bytes: LOW_OUTPUT_POOL_BYTES,
                bind_group_entries: LOW_BIND_GROUP_CACHE_ENTRIES,
            }
    }
}

/// Selected execution backend. Always resolves to CPU without the `gpu` feature.
#[derive(Clone)]
pub enum ExecutionDevice {
    Cpu { fallback_reason: Option<String> },
    Gpu(GpuContext),
}

impl ExecutionDevice {
    pub fn auto() -> Self {
        Self::from_gpu_result(GpuContext::new())
    }

    pub fn cpu() -> Self {
        Self::Cpu {
            fallback_reason: None,
        }
    }

    pub fn from_gpu_result(result: Result<GpuContext, GpuError>) -> Self {
        match result {
            Ok(context) => Self::Gpu(context),
            Err(error) => Self::Cpu {
                fallback_reason: Some(error.to_string()),
            },
        }
    }

    pub fn device(&self) -> crate::Device {
        match self {
            Self::Cpu { .. } => crate::Device::Cpu,
            Self::Gpu(_) => crate::Device::Gpu,
        }
    }

    pub fn gpu(&self) -> Option<&GpuContext> {
        match self {
            Self::Cpu { .. } => None,
            Self::Gpu(context) => Some(context),
        }
    }

    pub fn fallback_reason(&self) -> Option<&str> {
        match self {
            Self::Cpu { fallback_reason } => fallback_reason.as_deref(),
            Self::Gpu(_) => None,
        }
    }
}

impl GpuContext {
    pub fn global() -> Result<Self, GpuError> {
        Err(disabled())
    }

    pub fn new() -> Result<Self, GpuError> {
        Err(disabled())
    }

    pub fn create_devices(_count: usize) -> Result<Vec<Self>, GpuError> {
        Err(disabled())
    }

    pub fn adapter_name(&self) -> &str {
        "gpu-disabled"
    }

    pub fn set_memory_budget_bytes(&self, _budget_bytes: Option<usize>) {}

    pub fn apply_memory_budget(&self, _budget_bytes: Option<usize>) -> Result<(), GpuError> {
        Err(disabled())
    }

    pub fn memory_budget(&self) -> crate::memory::DeviceBudget {
        crate::memory::DeviceBudget::cpu(None)
    }

    pub fn memory_stats(&self) -> Result<GpuMemoryStats, GpuError> {
        Err(disabled())
    }

    pub fn adapter_limits(&self) -> GpuAdapterLimits {
        GpuAdapterLimits::default()
    }

    pub fn max_buffer_bytes(&self) -> usize {
        0
    }

    pub fn float_format_support(&self) -> GpuFloatFormatSupport {
        GpuFloatFormatSupport::default()
    }

    pub fn trim_dispatch_caches(&self) -> Result<(), GpuError> {
        Err(disabled())
    }

    pub fn set_cache_limits(&self, _limits: GpuCacheLimits) -> Result<(), GpuError> {
        Err(disabled())
    }
}

impl Tensor {
    /// Without the `gpu` feature every tensor lives on the CPU already.
    pub fn to_cpu(&self) -> Result<Tensor, GpuError> {
        Ok(self.clone())
    }

    pub fn backward_gpu(&self, _context: &GpuContext) -> Result<(), GpuError> {
        Err(disabled())
    }

    pub fn gpu_scale(&self, _factor: f32, _context: &GpuContext) -> Result<Tensor, GpuError> {
        Err(disabled())
    }
}