use crate::memory::DeviceBudget;
use crate::profile;
use crate::{enable_grad, CheckpointRecompute, Device, IndexTensor, Op, Tensor};
use bytemuck::{Pod, Zeroable};
use half::{bf16, f16};
use std::collections::{HashMap, VecDeque};
use std::error::Error;
use std::fmt;
use std::hint::black_box;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{mpsc, Arc, Mutex, OnceLock};
use std::time::{Duration, Instant};
const MATMUL_SHADER: &str = include_str!("../shaders/matmul.wgsl");
const MATMUL_FP16_SHADER: &str = include_str!("../shaders/matmul_fp16.wgsl");
const MATMUL_BF16_SHADER: &str = include_str!("../shaders/matmul_bf16.wgsl");
const MATMUL_QUANTIZED_SHADER: &str = include_str!("../shaders/matmul_quantized.wgsl");
const MATMUL_VEC4_SHADER: &str = include_str!("../shaders/matmul_vec4.wgsl");
const ELEMENTWISE_SHADER: &str = include_str!("../shaders/elementwise.wgsl");
const ATTENTION_SHADER: &str = include_str!("../shaders/attention.wgsl");
const TILED_ATTENTION_SHADER: &str = include_str!("../shaders/tiled_attention.wgsl");
const BACKWARD_SHADER: &str = include_str!("../shaders/backward.wgsl");
const OPTIMIZER_SHADER: &str = include_str!("../shaders/optimizer.wgsl");
const FUSION_SHADER: &str = include_str!("../shaders/fusion.wgsl");
const INDEX_SHADER: &str = include_str!("../shaders/index.wgsl");
const DEFAULT_OUTPUT_POOL_BYTES: usize = 64 * 1024 * 1024;
const DEFAULT_BIND_GROUP_CACHE_ENTRIES: usize = 512;
pub const LOW_MEMORY_BUDGET_THRESHOLD_BYTES: usize = 8 * 1024 * 1024 * 1024;
const LOW_OUTPUT_POOL_BYTES: usize = 16 * 1024 * 1024;
const LOW_BIND_GROUP_CACHE_ENTRIES: usize = 128;
const TIMESTAMP_QUERY_COUNT: u32 = wgpu::QUERY_SET_MAX_QUERIES;
const MATMUL_AUTOTUNE_SAMPLES: usize = 3;
#[cfg(not(target_arch = "wasm32"))]
static GLOBAL_GPU: OnceLock<GpuContext> = OnceLock::new();
static MATMUL_AUTOTUNE_CACHE: OnceLock<Mutex<HashMap<MatmulAutotuneKey, GpuMatmulConfig>>> =
OnceLock::new();
#[repr(C)]
#[derive(Clone, Copy, Pod, Zeroable)]
struct MatmulParams {
m: u32,
k: u32,
n: u32,
batch: u32,
padding_0: u32,
padding_1: u32,
padding_2: u32,
padding_3: u32,
}
#[repr(C)]
#[derive(Clone, Copy, Pod, Zeroable)]
struct QuantMatmulParams {
m: u32,
k: u32,
n: u32,
bits: u32,
group_size: u32,
scheme: u32,
num_groups: u32,
has_zero_points: u32,
}
#[repr(C)]
#[derive(Clone, Copy, Pod, Zeroable)]
struct ElementParams {
length: u32,
rows: u32,
columns: u32,
epsilon: f32,
}
#[repr(C)]
#[derive(Clone, Copy, Default, Pod, Zeroable)]
struct AttentionParams {
length: u32,
dim_0: u32,
dim_1: u32,
dim_2: u32,
dim_3: u32,
past_length: u32,
position_offset: u32,
theta: f32,
rope_type: u32,
rope_original_context: u32,
rope_factor: f32,
rope_attention_factor: f32,
rope_beta_fast: f32,
rope_beta_slow: f32,
padding_0: u32,
padding_1: u32,
}
#[repr(C)]
#[derive(Clone, Copy, Pod, Zeroable)]
struct TiledAttentionParams {
batch_size: u32,
query_heads: u32,
key_value_heads: u32,
query_length: u32,
key_length: u32,
head_dim: u32,
past_length: u32,
use_attention_mask: u32,
scale: f32,
dropout_probability: f32,
dropout_seed: u32,
padding: u32,
}
#[repr(C)]
#[derive(Clone, Copy, Pod, Zeroable)]
struct BackwardParams {
length: u32,
rows: u32,
columns: u32,
inner: u32,
batch: u32,
padding: u32,
epsilon: f32,
scalar: f32,
}
#[repr(C)]
#[derive(Clone, Copy, Pod, Zeroable)]
pub(crate) struct AdamWParams {
pub(crate) length: u32,
pub(crate) padding_0: u32,
pub(crate) padding_1: u32,
pub(crate) padding_2: u32,
pub(crate) learning_rate: f32,
pub(crate) beta1: f32,
pub(crate) beta2: f32,
pub(crate) epsilon: f32,
pub(crate) weight_decay: f32,
pub(crate) first_correction: f32,
pub(crate) second_correction: f32,
pub(crate) gradient_scale: f32,
}
#[repr(C)]
#[derive(Clone, Copy, Pod, Zeroable)]
struct FusionParams {
length: u32,
rows: u32,
input_width: u32,
output_width: u32,
query_length: u32,
key_length: u32,
past_length: u32,
padding: u32,
epsilon: f32,
scalar_0: f32,
scalar_1: f32,
scalar_2: f32,
}
#[derive(Debug)]
pub enum GpuError {
Initialization(String),
InvalidShape(String),
Map(String),
Synchronization(String),
}
impl fmt::Display for GpuError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Initialization(message) => {
write!(formatter, "GPU initialization failed: {message}")
}
Self::InvalidShape(message) => write!(formatter, "invalid GPU tensor shape: {message}"),
Self::Map(message) => write!(formatter, "GPU readback failed: {message}"),
Self::Synchronization(message) => {
write!(formatter, "GPU synchronization failed: {message}")
}
}
}
}
impl Error for GpuError {}
struct GpuInner {
device: wgpu::Device,
queue: wgpu::Queue,
pipelines: HashMap<&'static str, wgpu::ComputePipeline>,
staging_pool: Mutex<Vec<wgpu::Buffer>>,
output_pool: Arc<Mutex<OutputBufferPool>>,
bind_group_cache: Mutex<BindGroupCache>,
memory: Arc<MemoryCounters>,
adapter_name: String,
adapter_key: GpuAdapterKey,
power_mode: &'static str,
fp16_supported: bool,
timestamp_queries_supported: bool,
timestamp_capture: Mutex<Option<GpuTimestampCapture>>,
max_buffer_bytes: usize,
budget_bytes: Mutex<Option<usize>>,
}
struct GpuTimestampCapture {
query_set: wgpu::QuerySet,
resolve_buffer: wgpu::Buffer,
next_query: u32,
overflowed: bool,
}
struct CompletedTimestampCapture {
readback: Option<wgpu::Buffer>,
query_count: u32,
overflowed: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
struct GpuAdapterKey {
name: String,
vendor: u32,
device: u32,
driver: String,
driver_info: String,
backend: wgpu::Backend,
}
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
struct MatmulAutotuneKey {
adapter: GpuAdapterKey,
rows: usize,
shared: usize,
columns: usize,
batch: usize,
}
#[derive(Default)]
struct MemoryCounters {
live_tensor_buffers: AtomicUsize,
live_tensor_bytes: AtomicUsize,
peak_tensor_bytes: AtomicUsize,
uploads: AtomicUsize,
upload_bytes: AtomicUsize,
readbacks: AtomicUsize,
readback_bytes: AtomicUsize,
next_buffer_id: AtomicUsize,
output_buffer_allocations: AtomicUsize,
output_buffer_reuses: AtomicUsize,
bind_group_creations: AtomicUsize,
bind_group_cache_hits: AtomicUsize,
}
struct TrackedStorageBuffer {
id: usize,
buffer: wgpu::Buffer,
byte_size: usize,
memory: Arc<MemoryCounters>,
recycle_shape: Option<Vec<usize>>,
output_pool: Arc<Mutex<OutputBufferPool>>,
}
impl Drop for TrackedStorageBuffer {
fn drop(&mut self) {
self.memory
.live_tensor_buffers
.fetch_sub(1, Ordering::Relaxed);
self.memory
.live_tensor_bytes
.fetch_sub(self.byte_size, Ordering::Relaxed);
if let Some(shape) = self.recycle_shape.take() {
if let Ok(mut pool) = self.output_pool.lock() {
pool.recycle(self.id, shape, self.buffer.clone());
}
}
}
}
struct PooledOutputBuffer {
id: usize,
buffer: wgpu::Buffer,
serial: u64,
}
struct OutputBufferPool {
by_shape: HashMap<Vec<usize>, VecDeque<PooledOutputBuffer>>,
bytes: usize,
buffers: usize,
max_bytes: usize,
serial: u64,
}
impl OutputBufferPool {
fn new(max_bytes: usize) -> Self {
Self {
by_shape: HashMap::new(),
bytes: 0,
buffers: 0,
max_bytes,
serial: 0,
}
}
fn acquire(&mut self, shape: &[usize]) -> Option<(usize, wgpu::Buffer)> {
let (buffer, empty) = {
let buffers = self.by_shape.get_mut(shape)?;
let buffer = buffers.pop_back()?;
(buffer, buffers.is_empty())
};
if empty {
self.by_shape.remove(shape);
}
self.bytes = self.bytes.saturating_sub(buffer.buffer.size() as usize);
self.buffers = self.buffers.saturating_sub(1);
Some((buffer.id, buffer.buffer))
}
fn recycle(&mut self, id: usize, shape: Vec<usize>, buffer: wgpu::Buffer) {
let byte_size = buffer.size() as usize;
if self.max_bytes == 0 || byte_size > self.max_bytes {
return;
}
while self.bytes.saturating_add(byte_size) > self.max_bytes {
if !self.evict_oldest() {
return;
}
}
self.serial = self.serial.wrapping_add(1);
self.bytes += byte_size;
self.buffers += 1;
self.by_shape
.entry(shape)
.or_default()
.push_back(PooledOutputBuffer {
id,
buffer,
serial: self.serial,
});
}
fn set_max_bytes(&mut self, max_bytes: usize) {
self.max_bytes = max_bytes;
while self.bytes > self.max_bytes {
if !self.evict_oldest() {
break;
}
}
}
fn clear(&mut self) {
self.by_shape.clear();
self.bytes = 0;
self.buffers = 0;
}
fn evict_oldest(&mut self) -> bool {
let Some(shape) = self
.by_shape
.iter()
.filter_map(|(shape, buffers)| buffers.front().map(|buffer| (shape, buffer.serial)))
.min_by_key(|(_, serial)| *serial)
.map(|(shape, _)| shape.clone())
else {
return false;
};
let (removed, empty) = {
let buffers = self.by_shape.get_mut(&shape).unwrap();
let removed = buffers.pop_front().unwrap();
(removed, buffers.is_empty())
};
if empty {
self.by_shape.remove(&shape);
}
self.bytes = self.bytes.saturating_sub(removed.buffer.size() as usize);
self.buffers = self.buffers.saturating_sub(1);
true
}
}
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
struct BindGroupBufferKey {
binding: u32,
buffer_id: usize,
}
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
struct BindGroupCacheKey {
entry: &'static str,
uniform_binding: u32,
uniform_bytes: Vec<u8>,
buffers: Vec<BindGroupBufferKey>,
}
struct CachedBindGroup {
bind_group: wgpu::BindGroup,
_uniform: wgpu::Buffer,
serial: u64,
}
struct BindGroupCache {
entries: HashMap<BindGroupCacheKey, CachedBindGroup>,
max_entries: usize,
serial: u64,
}
impl BindGroupCache {
fn new(max_entries: usize) -> Self {
Self {
entries: HashMap::new(),
max_entries,
serial: 0,
}
}
fn get(&mut self, key: &BindGroupCacheKey) -> Option<wgpu::BindGroup> {
let entry = self.entries.get_mut(key)?;
self.serial = self.serial.wrapping_add(1);
entry.serial = self.serial;
Some(entry.bind_group.clone())
}
fn insert(
&mut self,
key: BindGroupCacheKey,
bind_group: wgpu::BindGroup,
uniform: wgpu::Buffer,
) {
if self.max_entries == 0 {
return;
}
while self.entries.len() >= self.max_entries {
if !self.evict_oldest() {
return;
}
}
self.serial = self.serial.wrapping_add(1);
self.entries.insert(
key,
CachedBindGroup {
bind_group,
_uniform: uniform,
serial: self.serial,
},
);
}
fn set_max_entries(&mut self, max_entries: usize) {
self.max_entries = max_entries;
while self.entries.len() > self.max_entries {
if !self.evict_oldest() {
break;
}
}
}
fn clear(&mut self) {
self.entries.clear();
}
fn evict_oldest(&mut self) -> bool {
let Some(key) = self
.entries
.iter()
.min_by_key(|(_, entry)| entry.serial)
.map(|(key, _)| key.clone())
else {
return false;
};
self.entries.remove(&key);
true
}
}
#[derive(Clone)]
pub struct GpuContext {
inner: Arc<GpuInner>,
}
#[derive(Clone)]
pub struct GpuTensor {
context: GpuContext,
buffer: Arc<TrackedStorageBuffer>,
shape: Vec<usize>,
length: usize,
}
#[derive(Clone)]
pub struct GpuMatmulTensor {
context: GpuContext,
buffer: Arc<TrackedStorageBuffer>,
shape: Vec<usize>,
length: usize,
format: GpuFloatFormat,
}
#[derive(Clone)]
pub(crate) struct GpuQuantizedLinearWeight {
context: GpuContext,
values: Arc<TrackedStorageBuffer>,
scales: Arc<TrackedStorageBuffer>,
zero_points: Arc<TrackedStorageBuffer>,
in_features: usize,
out_features: usize,
scheme: crate::weight_quant::QuantScheme,
}
#[derive(Clone)]
pub struct GpuIndexTensor {
context: GpuContext,
buffer: Arc<TrackedStorageBuffer>,
shape: Vec<usize>,
length: usize,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum GpuMatmulKernel {
Tile16,
ExactTile16,
RegisterBlocked32,
Vectorized8x32,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct GpuMatmulConfig {
pub kernel: GpuMatmulKernel,
pub tile_rows: u32,
pub tile_columns: u32,
pub workgroup_size: [u32; 3],
}
impl GpuMatmulConfig {
const TILE_16: Self = Self {
kernel: GpuMatmulKernel::Tile16,
tile_rows: 16,
tile_columns: 16,
workgroup_size: [16, 16, 1],
};
const EXACT_TILE_16: Self = Self {
kernel: GpuMatmulKernel::ExactTile16,
tile_rows: 16,
tile_columns: 16,
workgroup_size: [16, 16, 1],
};
const REGISTER_BLOCKED_32: Self = Self {
kernel: GpuMatmulKernel::RegisterBlocked32,
tile_rows: 32,
tile_columns: 32,
workgroup_size: [8, 8, 1],
};
const VECTORIZED_8X32: Self = Self {
kernel: GpuMatmulKernel::Vectorized8x32,
tile_rows: 8,
tile_columns: 32,
workgroup_size: [8, 8, 1],
};
const CANDIDATES: [Self; 4] = [
Self::TILE_16,
Self::EXACT_TILE_16,
Self::REGISTER_BLOCKED_32,
Self::VECTORIZED_8X32,
];
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum GpuFloatFormat {
#[default]
Fp32,
Fp16,
Bf16,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct GpuFloatFormatSupport {
pub fp32: bool,
pub fp16: bool,
pub bf16: bool,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct GpuTiming {
pub kernel: Option<Duration>,
pub host_submission: Duration,
pub synchronization: Duration,
}
impl GpuTiming {
pub fn host_total(&self) -> Duration {
self.host_submission + self.synchronization
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct GpuAdapterLimits {
pub max_buffer_size: u64,
pub max_storage_buffer_binding_size: u64,
pub max_compute_workgroup_storage_size: u32,
pub max_compute_invocations_per_workgroup: u32,
pub max_compute_workgroup_size_x: u32,
pub max_bind_groups: u32,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct GpuMemoryStats {
pub live_tensor_buffers: usize,
pub live_tensor_bytes: usize,
pub peak_tensor_bytes: usize,
pub pooled_staging_buffers: usize,
pub pooled_staging_bytes: usize,
pub pooled_output_buffers: usize,
pub pooled_output_bytes: usize,
pub bind_group_cache_entries: usize,
pub output_buffer_allocations: usize,
pub output_buffer_reuses: usize,
pub bind_group_creations: usize,
pub bind_group_cache_hits: usize,
pub uploads: usize,
pub upload_bytes: usize,
pub readbacks: usize,
pub readback_bytes: usize,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct GpuCacheLimits {
pub output_buffer_bytes: usize,
pub bind_group_entries: usize,
}
impl Default for GpuCacheLimits {
fn default() -> Self {
Self {
output_buffer_bytes: DEFAULT_OUTPUT_POOL_BYTES,
bind_group_entries: DEFAULT_BIND_GROUP_CACHE_ENTRIES,
}
}
}
impl GpuCacheLimits {
pub fn for_memory_budget(budget_bytes: Option<usize>) -> Self {
match budget_bytes {
Some(bytes) if bytes > 0 && bytes <= LOW_MEMORY_BUDGET_THRESHOLD_BYTES => Self {
output_buffer_bytes: LOW_OUTPUT_POOL_BYTES,
bind_group_entries: LOW_BIND_GROUP_CACHE_ENTRIES,
},
_ => Self::default(),
}
}
pub fn is_low_memory(&self) -> bool {
*self
== Self {
output_buffer_bytes: LOW_OUTPUT_POOL_BYTES,
bind_group_entries: LOW_BIND_GROUP_CACHE_ENTRIES,
}
}
}
#[derive(Clone)]
pub enum ExecutionDevice {
Cpu {
fallback_reason: Option<String>,
},
Gpu(GpuContext),
}
impl ExecutionDevice {
pub fn auto() -> Self {
Self::from_gpu_result(GpuContext::new())
}
pub fn cpu() -> Self {
Self::Cpu {
fallback_reason: None,
}
}
pub fn from_gpu_result(result: Result<GpuContext, GpuError>) -> Self {
match result {
Ok(context) => Self::Gpu(context),
Err(error) => Self::Cpu {
fallback_reason: Some(error.to_string()),
},
}
}
pub fn device(&self) -> Device {
match self {
Self::Cpu { .. } => Device::Cpu,
Self::Gpu(_) => Device::Gpu,
}
}
pub fn gpu(&self) -> Option<&GpuContext> {
match self {
Self::Cpu { .. } => None,
Self::Gpu(context) => Some(context),
}
}
pub fn fallback_reason(&self) -> Option<&str> {
match self {
Self::Cpu { fallback_reason } => fallback_reason.as_deref(),
Self::Gpu(_) => None,
}
}
}
impl GpuContext {
#[cfg(not(target_arch = "wasm32"))]
pub fn global() -> Result<Self, GpuError> {
if let Some(context) = GLOBAL_GPU.get() {
return Ok(context.clone());
}
let context = Self::new()?;
let _ = GLOBAL_GPU.set(context.clone());
Ok(GLOBAL_GPU.get().cloned().unwrap_or(context))
}
pub fn new() -> Result<Self, GpuError> {
pollster::block_on(Self::new_async())
}
pub async fn new_async() -> Result<Self, GpuError> {
let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle());
let adapter = instance
.request_adapter(&wgpu::RequestAdapterOptions {
power_preference: wgpu::PowerPreference::HighPerformance,
compatible_surface: None,
force_fallback_adapter: false,
apply_limit_buckets: false,
})
.await
.map_err(|error| GpuError::Initialization(error.to_string()))?;
Self::from_adapter_async(adapter, "high_performance", 0).await
}
pub fn create_devices(count: usize) -> Result<Vec<Self>, GpuError> {
pollster::block_on(Self::create_devices_async(count))
}
pub async fn create_devices_async(count: usize) -> Result<Vec<Self>, GpuError> {
if count == 0 {
return Err(GpuError::Initialization(
"GPU device count must be positive".into(),
));
}
let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle());
let adapters = instance.enumerate_adapters(wgpu::Backends::all()).await;
if adapters.is_empty() {
return Err(GpuError::Initialization(
"no GPU adapters available for multi-device data parallel".into(),
));
}
let mut contexts = Vec::with_capacity(count);
for index in 0..count {
let adapter = adapters[index % adapters.len()].clone();
contexts.push(
Self::from_adapter_async(adapter, "high_performance", index).await?,
);
}
Ok(contexts)
}
async fn from_adapter_async(
adapter: wgpu::Adapter,
power_mode: &'static str,
device_index: usize,
) -> Result<Self, GpuError> {
let adapter_info = adapter.get_info();
let adapter_name = adapter_info.name.clone();
let adapter_key = GpuAdapterKey {
name: adapter_info.name,
vendor: adapter_info.vendor,
device: adapter_info.device,
driver: adapter_info.driver,
driver_info: adapter_info.driver_info,
backend: adapter_info.backend,
};
let adapter_limits = adapter.limits();
let adapter_features = adapter.features();
let fp16_supported = adapter_features.contains(wgpu::Features::SHADER_F16);
let timestamp_queries_supported =
adapter_features.contains(wgpu::Features::TIMESTAMP_QUERY);
let mut required_features = wgpu::Features::empty();
required_features.set(wgpu::Features::SHADER_F16, fp16_supported);
required_features.set(wgpu::Features::TIMESTAMP_QUERY, timestamp_queries_supported);
let label = format!("oxidelm-gpu-device-{device_index}");
let (device, queue) = adapter
.request_device(&wgpu::DeviceDescriptor {
label: Some(&label),
required_features,
memory_hints: wgpu::MemoryHints::Performance,
..Default::default()
})
.await
.map_err(|error| GpuError::Initialization(error.to_string()))?;
let max_buffer_bytes = device
.limits()
.max_buffer_size
.min(adapter_limits.max_buffer_size) as usize;
let matmul_module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("matmul-wgsl"),
source: wgpu::ShaderSource::Wgsl(MATMUL_SHADER.into()),
});
let matmul_vec4_module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("matmul-vec4-wgsl"),
source: wgpu::ShaderSource::Wgsl(MATMUL_VEC4_SHADER.into()),
});
let matmul_bf16_module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("matmul-bf16-wgsl"),
source: wgpu::ShaderSource::Wgsl(MATMUL_BF16_SHADER.into()),
});
let matmul_quantized_module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("matmul-quantized-wgsl"),
source: wgpu::ShaderSource::Wgsl(MATMUL_QUANTIZED_SHADER.into()),
});
let elementwise_module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("elementwise-wgsl"),
source: wgpu::ShaderSource::Wgsl(ELEMENTWISE_SHADER.into()),
});
let attention_module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("attention-wgsl"),
source: wgpu::ShaderSource::Wgsl(ATTENTION_SHADER.into()),
});
let tiled_attention_module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("tiled-attention-wgsl"),
source: wgpu::ShaderSource::Wgsl(TILED_ATTENTION_SHADER.into()),
});
let backward_module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("backward-wgsl"),
source: wgpu::ShaderSource::Wgsl(BACKWARD_SHADER.into()),
});
let optimizer_module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("optimizer-wgsl"),
source: wgpu::ShaderSource::Wgsl(OPTIMIZER_SHADER.into()),
});
let fusion_module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("fusion-wgsl"),
source: wgpu::ShaderSource::Wgsl(FUSION_SHADER.into()),
});
let index_module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("index-wgsl"),
source: wgpu::ShaderSource::Wgsl(INDEX_SHADER.into()),
});
let mut pipelines = HashMap::new();
for entry_point in ["matmul", "matmul_exact_16", "matmul_register_32"] {
pipelines.insert(
entry_point,
create_pipeline(&device, &matmul_module, entry_point),
);
}
pipelines.insert(
"matmul_vec4_8x32",
create_pipeline(&device, &matmul_vec4_module, "matmul_vec4_8x32"),
);
pipelines.insert(
"matmul_bf16",
create_pipeline(&device, &matmul_bf16_module, "matmul_bf16"),
);
pipelines.insert(
"matmul_quantized",
create_pipeline(&device, &matmul_quantized_module, "matmul_quantized"),
);
if fp16_supported {
let matmul_fp16_module = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: Some("matmul-fp16-wgsl"),
source: wgpu::ShaderSource::Wgsl(MATMUL_FP16_SHADER.into()),
});
pipelines.insert(
"matmul_fp16",
create_pipeline(&device, &matmul_fp16_module, "matmul_fp16"),
);
}
for entry_point in [
"add", "bias_add", "mul", "scale", "relu", "gelu", "silu", "softmax", "rms_norm",
] {
pipelines.insert(
entry_point,
create_pipeline(&device, &elementwise_module, entry_point),
);
}
for entry_point in [
"rope",
"rope_backward",
"transpose_2d",
"swap_middle",
"swap_last",
"causal_mask",
"causal_mask_backward",
] {
pipelines.insert(
entry_point,
create_pipeline(&device, &attention_module, entry_point),
);
}
for entry_point in [
"tiled_attention",
"tiled_attention_backward_query",
"tiled_attention_backward_key_value",
] {
pipelines.insert(
entry_point,
create_pipeline(&device, &tiled_attention_module, entry_point),
);
}
for entry_point in [
"multiply_left",
"multiply_right",
"silu_backward",
"gelu_backward",
"matmul_left",
"matmul_right",
"softmax_backward",
"rms_input",
"rms_weight",
"squared_norm",
] {
pipelines.insert(
entry_point,
create_pipeline(&device, &backward_module, entry_point),
);
}
pipelines.insert(
"adamw",
create_pipeline(&device, &optimizer_module, "adamw"),
);
for entry_point in ["residual_rms_norm", "swiglu", "masked_softmax"] {
pipelines.insert(
entry_point,
create_pipeline(&device, &fusion_module, entry_point),
);
}
for entry_point in [
"embedding_forward",
"embedding_weight",
"cross_entropy_forward",
"cross_entropy_backward",
] {
pipelines.insert(
entry_point,
create_pipeline(&device, &index_module, entry_point),
);
}
let cache_limits = GpuCacheLimits::default();
Ok(Self {
inner: Arc::new(GpuInner {
device,
queue,
pipelines,
staging_pool: Mutex::new(Vec::new()),
output_pool: Arc::new(Mutex::new(OutputBufferPool::new(
cache_limits.output_buffer_bytes.min(max_buffer_bytes),
))),
bind_group_cache: Mutex::new(BindGroupCache::new(cache_limits.bind_group_entries)),
memory: Arc::new(MemoryCounters::default()),
adapter_name,
adapter_key,
power_mode,
fp16_supported,
timestamp_queries_supported,
timestamp_capture: Mutex::new(None),
max_buffer_bytes: max_buffer_bytes.max(1),
budget_bytes: Mutex::new(None),
}),
})
}
pub fn adapter_name(&self) -> &str {
&self.inner.adapter_name
}
pub fn adapter_backend(&self) -> wgpu::Backend {
self.inner.adapter_key.backend
}
pub fn power_mode(&self) -> &str {
self.inner.power_mode
}
pub fn adapter_metadata(&self) -> crate::perf_regression::GpuAdapterMetadata {
let key = &self.inner.adapter_key;
crate::perf_regression::GpuAdapterMetadata {
name: key.name.clone(),
backend: format!("{:?}", key.backend),
driver: key.driver.clone(),
driver_info: key.driver_info.clone(),
vendor: key.vendor,
device: key.device,
}
}
pub fn matmul_autotune_cache_entries(&self) -> usize {
MATMUL_AUTOTUNE_CACHE
.get()
.and_then(|cache| cache.lock().ok())
.map(|cache| {
cache
.keys()
.filter(|key| key.adapter == self.inner.adapter_key)
.count()
})
.unwrap_or(0)
}
pub fn timestamp_queries_supported(&self) -> bool {
self.inner.timestamp_queries_supported
}
pub fn float_format_support(&self) -> GpuFloatFormatSupport {
GpuFloatFormatSupport {
fp32: true,
fp16: self.inner.fp16_supported,
bf16: true,
}
}
pub fn supports_float_format(&self, format: GpuFloatFormat) -> bool {
let support = self.float_format_support();
match format {
GpuFloatFormat::Fp32 => support.fp32,
GpuFloatFormat::Fp16 => support.fp16,
GpuFloatFormat::Bf16 => support.bf16,
}
}
pub fn max_buffer_bytes(&self) -> usize {
self.inner.max_buffer_bytes
}
pub fn adapter_limits(&self) -> GpuAdapterLimits {
let limits = self.inner.device.limits();
GpuAdapterLimits {
max_buffer_size: limits.max_buffer_size,
max_storage_buffer_binding_size: limits.max_storage_buffer_binding_size,
max_compute_workgroup_storage_size: limits.max_compute_workgroup_storage_size,
max_compute_invocations_per_workgroup: limits.max_compute_invocations_per_workgroup,
max_compute_workgroup_size_x: limits.max_compute_workgroup_size_x,
max_bind_groups: limits.max_bind_groups,
}
}
pub fn set_memory_budget_bytes(&self, budget_bytes: Option<usize>) {
if let Ok(mut guard) = self.inner.budget_bytes.lock() {
*guard = budget_bytes;
}
}
pub fn apply_memory_budget(&self, budget_bytes: Option<usize>) -> Result<(), GpuError> {
self.set_memory_budget_bytes(budget_bytes);
if budget_bytes.is_some() {
self.set_cache_limits(GpuCacheLimits::for_memory_budget(budget_bytes))?;
}
Ok(())
}
pub fn memory_budget(&self) -> DeviceBudget {
let override_budget = self.inner.budget_bytes.lock().ok().and_then(|guard| *guard);
DeviceBudget::gpu(
self.inner.adapter_name.clone(),
self.inner.max_buffer_bytes,
override_budget,
)
}
pub fn synchronize(&self) -> Result<(), GpuError> {
let _span = profile::span("gpu.synchronize");
self.inner
.device
.poll(wgpu::PollType::Wait {
submission_index: None,
timeout: None,
})
.map_err(|error| GpuError::Synchronization(error.to_string()))?;
Ok(())
}
pub fn measure_gpu<R, E>(
&self,
operation: impl FnOnce() -> Result<R, E>,
) -> Result<(R, GpuTiming), E>
where
E: From<GpuError>,
{
self.synchronize().map_err(E::from)?;
self.begin_timestamp_capture().map_err(E::from)?;
let submission_started = Instant::now();
let output = match operation() {
Ok(output) => output,
Err(error) => {
self.cancel_timestamp_capture();
return Err(error);
}
};
let host_submission = submission_started.elapsed();
let capture = self.finish_timestamp_capture().map_err(E::from)?;
let synchronization_started = Instant::now();
self.synchronize().map_err(E::from)?;
let synchronization = synchronization_started.elapsed();
let kernel = capture
.map(|capture| self.read_timestamp_capture(capture))
.transpose()
.map_err(E::from)?
.flatten();
Ok((
output,
GpuTiming {
kernel,
host_submission,
synchronization,
},
))
}
fn begin_timestamp_capture(&self) -> Result<(), GpuError> {
let mut active = self
.inner
.timestamp_capture
.lock()
.map_err(|_| GpuError::Synchronization("timestamp capture lock poisoned".into()))?;
if active.is_some() {
return Err(GpuError::Synchronization(
"nested GPU timestamp captures are not supported".into(),
));
}
if !self.inner.timestamp_queries_supported {
return Ok(());
}
let byte_size = u64::from(TIMESTAMP_QUERY_COUNT) * size_of::<u64>() as u64;
*active = Some(GpuTimestampCapture {
query_set: self
.inner
.device
.create_query_set(&wgpu::QuerySetDescriptor {
label: Some("benchmark-timestamp-queries"),
ty: wgpu::QueryType::Timestamp,
count: TIMESTAMP_QUERY_COUNT,
}),
resolve_buffer: self.inner.device.create_buffer(&wgpu::BufferDescriptor {
label: Some("benchmark-timestamp-resolve"),
size: byte_size,
usage: wgpu::BufferUsages::QUERY_RESOLVE | wgpu::BufferUsages::COPY_SRC,
mapped_at_creation: false,
}),
next_query: 0,
overflowed: false,
});
Ok(())
}
fn cancel_timestamp_capture(&self) {
if let Ok(mut active) = self.inner.timestamp_capture.lock() {
active.take();
}
}
fn finish_timestamp_capture(&self) -> Result<Option<CompletedTimestampCapture>, GpuError> {
let capture = self
.inner
.timestamp_capture
.lock()
.map_err(|_| GpuError::Synchronization("timestamp capture lock poisoned".into()))?
.take();
let Some(capture) = capture else {
return Ok(None);
};
if capture.next_query == 0 {
return Ok(Some(CompletedTimestampCapture {
readback: None,
query_count: 0,
overflowed: capture.overflowed,
}));
}
let byte_size = u64::from(capture.next_query) * size_of::<u64>() as u64;
let readback = self.inner.device.create_buffer(&wgpu::BufferDescriptor {
label: Some("benchmark-timestamp-readback"),
size: byte_size,
usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
mapped_at_creation: false,
});
let mut encoder =
self.inner
.device
.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("benchmark-timestamp-resolve-encoder"),
});
encoder.resolve_query_set(
&capture.query_set,
0..capture.next_query,
&capture.resolve_buffer,
0,
);
encoder.copy_buffer_to_buffer(&capture.resolve_buffer, 0, &readback, 0, byte_size);
self.inner.queue.submit([encoder.finish()]);
Ok(Some(CompletedTimestampCapture {
readback: Some(readback),
query_count: capture.next_query,
overflowed: capture.overflowed,
}))
}
fn read_timestamp_capture(
&self,
capture: CompletedTimestampCapture,
) -> Result<Option<Duration>, GpuError> {
if capture.overflowed {
return Ok(None);
}
let Some(readback) = capture.readback else {
return Ok(Some(Duration::ZERO));
};
let slice = readback.slice(..);
let (sender, receiver) = mpsc::channel();
slice.map_async(wgpu::MapMode::Read, move |result| {
let _ = sender.send(result);
});
self.inner
.device
.poll(wgpu::PollType::Wait {
submission_index: None,
timeout: None,
})
.map_err(|error| GpuError::Synchronization(error.to_string()))?;
receiver
.recv()
.map_err(|error| GpuError::Map(error.to_string()))?
.map_err(|error| GpuError::Map(error.to_string()))?;
let mapped = slice
.get_mapped_range()
.map_err(|error| GpuError::Map(error.to_string()))?;
let timestamps = mapped
.chunks_exact(size_of::<u64>())
.take(capture.query_count as usize)
.map(|bytes| u64::from_ne_bytes(bytes.try_into().expect("timestamp is eight bytes")))
.collect::<Vec<_>>();
drop(mapped);
readback.unmap();
let ticks = timestamps
.chunks_exact(2)
.map(|pair| pair[1].wrapping_sub(pair[0]))
.sum::<u64>();
let nanoseconds = ticks as f64 * f64::from(self.inner.queue.get_timestamp_period());
Ok(Some(Duration::from_secs_f64(nanoseconds / 1_000_000_000.0)))
}
pub fn memory_stats(&self) -> Result<GpuMemoryStats, GpuError> {
let staging_pool = self
.inner
.staging_pool
.lock()
.map_err(|_| GpuError::Synchronization("staging pool lock poisoned".into()))?;
let output_pool = self
.inner
.output_pool
.lock()
.map_err(|_| GpuError::Synchronization("output pool lock poisoned".into()))?;
let bind_group_cache = self
.inner
.bind_group_cache
.lock()
.map_err(|_| GpuError::Synchronization("bind group cache lock poisoned".into()))?;
Ok(GpuMemoryStats {
live_tensor_buffers: self
.inner
.memory
.live_tensor_buffers
.load(Ordering::Relaxed),
live_tensor_bytes: self.inner.memory.live_tensor_bytes.load(Ordering::Relaxed),
peak_tensor_bytes: self.inner.memory.peak_tensor_bytes.load(Ordering::Relaxed),
pooled_staging_buffers: staging_pool.len(),
pooled_staging_bytes: staging_pool.iter().map(wgpu::Buffer::size).sum::<u64>() as usize,
pooled_output_buffers: output_pool.buffers,
pooled_output_bytes: output_pool.bytes,
bind_group_cache_entries: bind_group_cache.entries.len(),
output_buffer_allocations: self
.inner
.memory
.output_buffer_allocations
.load(Ordering::Relaxed),
output_buffer_reuses: self
.inner
.memory
.output_buffer_reuses
.load(Ordering::Relaxed),
bind_group_creations: self
.inner
.memory
.bind_group_creations
.load(Ordering::Relaxed),
bind_group_cache_hits: self
.inner
.memory
.bind_group_cache_hits
.load(Ordering::Relaxed),
uploads: self.inner.memory.uploads.load(Ordering::Relaxed),
upload_bytes: self.inner.memory.upload_bytes.load(Ordering::Relaxed),
readbacks: self.inner.memory.readbacks.load(Ordering::Relaxed),
readback_bytes: self.inner.memory.readback_bytes.load(Ordering::Relaxed),
})
}
pub fn reset_memory_stats(&self) {
self.inner.memory.peak_tensor_bytes.store(
self.inner.memory.live_tensor_bytes.load(Ordering::Relaxed),
Ordering::Relaxed,
);
self.inner.memory.uploads.store(0, Ordering::Relaxed);
self.inner.memory.upload_bytes.store(0, Ordering::Relaxed);
self.inner.memory.readbacks.store(0, Ordering::Relaxed);
self.inner.memory.readback_bytes.store(0, Ordering::Relaxed);
self.inner
.memory
.output_buffer_allocations
.store(0, Ordering::Relaxed);
self.inner
.memory
.output_buffer_reuses
.store(0, Ordering::Relaxed);
self.inner
.memory
.bind_group_creations
.store(0, Ordering::Relaxed);
self.inner
.memory
.bind_group_cache_hits
.store(0, Ordering::Relaxed);
}
pub fn trim_staging_pool(&self) -> Result<(), GpuError> {
self.synchronize()?;
self.inner
.staging_pool
.lock()
.map_err(|_| GpuError::Synchronization("staging pool lock poisoned".into()))?
.clear();
Ok(())
}
pub fn set_cache_limits(&self, limits: GpuCacheLimits) -> Result<(), GpuError> {
self.synchronize()?;
self.inner
.output_pool
.lock()
.map_err(|_| GpuError::Synchronization("output pool lock poisoned".into()))?
.set_max_bytes(limits.output_buffer_bytes.min(self.inner.max_buffer_bytes));
self.inner
.bind_group_cache
.lock()
.map_err(|_| GpuError::Synchronization("bind group cache lock poisoned".into()))?
.set_max_entries(limits.bind_group_entries);
Ok(())
}
pub fn trim_dispatch_caches(&self) -> Result<(), GpuError> {
self.synchronize()?;
self.inner
.bind_group_cache
.lock()
.map_err(|_| GpuError::Synchronization("bind group cache lock poisoned".into()))?
.clear();
self.inner
.output_pool
.lock()
.map_err(|_| GpuError::Synchronization("output pool lock poisoned".into()))?
.clear();
Ok(())
}
pub fn upload(&self, data: &[f32], shape: Vec<usize>) -> Result<GpuTensor, GpuError> {
let _span = profile::span("gpu.upload");
validate_data_shape(data, &shape)?;
let buffer = self.create_storage_buffer(data.len(), "gpu-tensor");
self.inner
.queue
.write_buffer(&buffer.buffer, 0, bytemuck::cast_slice(data));
self.inner.memory.uploads.fetch_add(1, Ordering::Relaxed);
self.inner
.memory
.upload_bytes
.fetch_add(data.len() * std::mem::size_of::<f32>(), Ordering::Relaxed);
Ok(GpuTensor {
context: self.clone(),
buffer,
shape,
length: data.len(),
})
}
pub fn upload_matmul(
&self,
data: &[f32],
shape: Vec<usize>,
requested_format: GpuFloatFormat,
) -> Result<GpuMatmulTensor, GpuError> {
let _span = profile::span("gpu.upload_matmul");
validate_data_shape(data, &shape)?;
let format = if self.supports_float_format(requested_format) {
requested_format
} else {
GpuFloatFormat::Fp32
};
let bytes = match format {
GpuFloatFormat::Fp32 => bytemuck::cast_slice(data).to_vec(),
GpuFloatFormat::Fp16 => {
let mut values = data
.iter()
.map(|value| f16::from_f32(*value).to_bits())
.collect::<Vec<_>>();
if values.len() % 2 != 0 {
values.push(0);
}
bytemuck::cast_slice(&values).to_vec()
}
GpuFloatFormat::Bf16 => pack_bf16(data),
};
let buffer = self.create_storage_buffer_bytes(bytes.len(), "gpu-matmul-tensor");
self.inner.queue.write_buffer(&buffer.buffer, 0, &bytes);
self.inner.memory.uploads.fetch_add(1, Ordering::Relaxed);
self.inner
.memory
.upload_bytes
.fetch_add(bytes.len(), Ordering::Relaxed);
Ok(GpuMatmulTensor {
context: self.clone(),
buffer,
shape,
length: data.len(),
format,
})
}
pub(crate) fn upload_quantized_linear_weight(
&self,
packed_values: &[u32],
scales: &[f32],
zero_points: Option<&[f32]>,
in_features: usize,
out_features: usize,
scheme: crate::weight_quant::QuantScheme,
) -> Result<GpuQuantizedLinearWeight, GpuError> {
scheme
.validate()
.map_err(GpuError::InvalidShape)?;
if in_features == 0 || out_features == 0 {
return Err(GpuError::InvalidShape(
"quantized linear weight dimensions are invalid".into(),
));
}
let expected_scales = scheme.scale_count(in_features, out_features);
if scales.len() != expected_scales {
return Err(GpuError::InvalidShape(format!(
"expected {expected_scales} scales, got {}",
scales.len()
)));
}
if let Some(zp) = zero_points {
if scheme.symmetric {
return Err(GpuError::InvalidShape(
"symmetric quantized weights must not include zero-points".into(),
));
}
if zp.len() != expected_scales {
return Err(GpuError::InvalidShape(format!(
"expected {expected_scales} zero-points, got {}",
zp.len()
)));
}
} else if !scheme.symmetric {
return Err(GpuError::InvalidShape(
"asymmetric quantized weights require zero-points".into(),
));
}
let values_per_word = scheme.format.values_per_word();
let expected_words = (in_features * out_features).div_ceil(values_per_word);
if packed_values.len() != expected_words {
return Err(GpuError::InvalidShape(
"packed quantized weight length does not match its shape".into(),
));
}
let values =
self.create_storage_buffer_bytes(packed_values.len() * 4, "gpu-quantized-weight");
self.inner
.queue
.write_buffer(&values.buffer, 0, bytemuck::cast_slice(packed_values));
let scale_buffer = self.create_storage_buffer(scales.len().max(1), "gpu-quantized-scales");
self.inner
.queue
.write_buffer(&scale_buffer.buffer, 0, bytemuck::cast_slice(scales));
let zp_data = zero_points.unwrap_or(&[]);
let zp_len = zp_data.len().max(1);
let zp_buffer = self.create_storage_buffer(zp_len, "gpu-quantized-zero-points");
if zp_data.is_empty() {
self.inner
.queue
.write_buffer(&zp_buffer.buffer, 0, bytemuck::cast_slice(&[0.0f32]));
} else {
self.inner
.queue
.write_buffer(&zp_buffer.buffer, 0, bytemuck::cast_slice(zp_data));
}
let uploaded_bytes = packed_values.len() * 4 + scales.len() * 4 + zp_len * 4;
self.inner.memory.uploads.fetch_add(3, Ordering::Relaxed);
self.inner
.memory
.upload_bytes
.fetch_add(uploaded_bytes, Ordering::Relaxed);
Ok(GpuQuantizedLinearWeight {
context: self.clone(),
values,
scales: scale_buffer,
zero_points: zp_buffer,
in_features,
out_features,
scheme,
})
}
pub fn upload_indices(
&self,
data: &[usize],
shape: Vec<usize>,
) -> Result<GpuIndexTensor, GpuError> {
let _span = profile::span("gpu.upload");
if shape.is_empty() || shape.iter().any(|dimension| *dimension == 0) {
return Err(GpuError::InvalidShape(
"GPU index tensors need at least one non-zero dimension".into(),
));
}
if shape.iter().product::<usize>() != data.len() {
return Err(GpuError::InvalidShape(
"index data length does not match GPU tensor shape".into(),
));
}
let values = data
.iter()
.map(|index| to_u32(*index, "token ID"))
.collect::<Result<Vec<_>, _>>()?;
let buffer = self.create_storage_buffer(values.len(), "gpu-index-tensor");
self.inner
.queue
.write_buffer(&buffer.buffer, 0, bytemuck::cast_slice(&values));
self.inner.memory.uploads.fetch_add(1, Ordering::Relaxed);
self.inner
.memory
.upload_bytes
.fetch_add(values.len() * std::mem::size_of::<u32>(), Ordering::Relaxed);
Ok(GpuIndexTensor {
context: self.clone(),
buffer,
shape,
length: values.len(),
})
}
pub fn matmul(&self, left: &GpuTensor, right: &GpuTensor) -> Result<GpuTensor, GpuError> {
let config = self.select_matmul_config(left, right)?;
self.matmul_with_kernel(left, right, config.kernel)
}
pub fn matmul_mixed(
&self,
left: &GpuMatmulTensor,
right: &GpuMatmulTensor,
) -> Result<GpuTensor, GpuError> {
self.check_matmul_context(left)?;
self.check_matmul_context(right)?;
if left.format != right.format {
return Err(GpuError::InvalidShape(
"mixed-precision matmul operands must use the same format".into(),
));
}
let dimensions = gpu_matmul_dimensions_from_shapes(&left.shape, &right.shape)?;
let output = self.empty_tensor(dimensions.output_shape, "mixed-matmul-output")?;
let params = MatmulParams {
m: to_u32(dimensions.rows, "M")?,
k: to_u32(dimensions.shared, "K")?,
n: to_u32(dimensions.columns, "N")?,
batch: to_u32(dimensions.batch, "batch count")?,
padding_0: 0,
padding_1: 0,
padding_2: 0,
padding_3: 0,
};
let entry = match left.format {
GpuFloatFormat::Fp32 => "matmul",
GpuFloatFormat::Fp16 => "matmul_fp16",
GpuFloatFormat::Bf16 => "matmul_bf16",
};
self.dispatch_matmul_buffers(
entry,
left.buffer.as_ref(),
right.buffer.as_ref(),
output.buffer.as_ref(),
¶ms,
(
to_u32(dimensions.columns.div_ceil(16), "N workgroups")?,
to_u32(dimensions.rows.div_ceil(16), "M workgroups")?,
to_u32(dimensions.batch, "batch workgroups")?,
),
);
Ok(output)
}
pub(crate) fn quantized_linear(
&self,
input: &GpuTensor,
weight: &GpuQuantizedLinearWeight,
) -> Result<GpuTensor, GpuError> {
self.check_context(input)?;
if !weight.belongs_to(self) {
return Err(GpuError::InvalidShape(
"quantized linear weight belongs to another context".into(),
));
}
if input.shape.len() != 2 || input.shape[1] != weight.in_features {
return Err(GpuError::InvalidShape(
"quantized linear input must have shape [rows, in_features]".into(),
));
}
let rows = input.shape[0];
let output =
self.empty_tensor(vec![rows, weight.out_features], "quantized-linear-output")?;
let (group_size, num_groups) = match weight.scheme.granularity {
crate::weight_quant::QuantGranularity::PerChannel => (0u32, 1u32),
crate::weight_quant::QuantGranularity::PerTensor => (1u32, 1u32),
crate::weight_quant::QuantGranularity::PerGroup { size } => {
let groups = weight.in_features.div_ceil(size.max(1));
(to_u32(size, "quant group size")?, to_u32(groups, "quant groups")?)
}
};
let params = QuantMatmulParams {
m: to_u32(rows, "quantized linear rows")?,
k: to_u32(weight.in_features, "quantized linear input width")?,
n: to_u32(weight.out_features, "quantized linear output width")?,
bits: weight.scheme.format.bits(),
group_size,
scheme: weight.scheme.format.gpu_scheme_id(weight.scheme.symmetric),
num_groups,
has_zero_points: u32::from(!weight.scheme.symmetric),
};
let entry = "matmul_quantized";
let pipeline = &self.inner.pipelines[entry];
let bind_group = self.cached_bind_group(
entry,
"quantized-linear-bind-group",
"quantized-linear-params",
&[
(0, input.buffer.as_ref()),
(1, weight.values.as_ref()),
(2, weight.scales.as_ref()),
(3, weight.zero_points.as_ref()),
(4, output.buffer.as_ref()),
],
5,
bytemuck::bytes_of(¶ms),
);
self.submit(
pipeline,
&bind_group,
to_u32(
weight.out_features.div_ceil(16),
"quantized output workgroups",
)?,
to_u32(rows.div_ceil(16), "quantized row workgroups")?,
1,
);
Ok(output)
}
pub fn select_matmul_kernel(
&self,
left: &GpuTensor,
right: &GpuTensor,
) -> Result<GpuMatmulKernel, GpuError> {
Ok(self.select_matmul_config(left, right)?.kernel)
}
pub fn select_matmul_config(
&self,
left: &GpuTensor,
right: &GpuTensor,
) -> Result<GpuMatmulConfig, GpuError> {
self.check_context(left)?;
self.check_context(right)?;
let dimensions = gpu_matmul_dimensions(left, right)?;
let fallback = if dimensions.shared >= 8
&& dimensions.columns >= 32
&& dimensions.shared % 4 == 0
&& dimensions.columns % 4 == 0
{
GpuMatmulConfig::VECTORIZED_8X32
} else if dimensions.rows % 16 == 0
&& dimensions.shared % 16 == 0
&& dimensions.columns % 16 == 0
{
GpuMatmulConfig::EXACT_TILE_16
} else {
GpuMatmulConfig::TILE_16
};
let key = MatmulAutotuneKey {
adapter: self.inner.adapter_key.clone(),
rows: dimensions.rows,
shared: dimensions.shared,
columns: dimensions.columns,
batch: dimensions.batch,
};
let cache = MATMUL_AUTOTUNE_CACHE.get_or_init(|| Mutex::new(HashMap::new()));
if let Some(config) = cache.lock().ok().and_then(|cache| cache.get(&key).copied()) {
return Ok(config);
}
let candidates = GpuMatmulConfig::CANDIDATES
.into_iter()
.filter(|config| config.supports(&dimensions))
.collect::<Vec<_>>();
let config = self
.autotune_matmul(left, right, &candidates)
.unwrap_or(fallback);
if let Ok(mut cache) = cache.lock() {
cache.insert(key, config);
}
Ok(config)
}
fn autotune_matmul(
&self,
left: &GpuTensor,
right: &GpuTensor,
candidates: &[GpuMatmulConfig],
) -> Result<GpuMatmulConfig, GpuError> {
let _span = profile::span("gpu.matmul_autotune");
for config in candidates {
black_box(self.matmul_with_kernel(left, right, config.kernel)?);
}
self.synchronize()?;
let capture_already_active = self.timestamp_capture_active();
let mut fastest = None;
for config in candidates {
let mut samples = Vec::with_capacity(MATMUL_AUTOTUNE_SAMPLES);
for _ in 0..MATMUL_AUTOTUNE_SAMPLES {
let elapsed = if capture_already_active {
let started = Instant::now();
black_box(self.matmul_with_kernel(left, right, config.kernel)?);
self.synchronize()?;
started.elapsed()
} else {
let (output, timing) =
self.measure_gpu(|| self.matmul_with_kernel(left, right, config.kernel))?;
black_box(output);
timing.kernel.unwrap_or_else(|| timing.host_total())
};
samples.push(elapsed);
}
samples.sort_unstable();
let median = samples[samples.len() / 2];
if fastest.is_none_or(|(_, fastest_time)| median < fastest_time) {
fastest = Some((*config, median));
}
}
fastest
.map(|(config, _)| config)
.ok_or_else(|| GpuError::InvalidShape("no valid GPU matmul configuration".into()))
}
fn timestamp_capture_active(&self) -> bool {
self.inner
.timestamp_capture
.lock()
.is_ok_and(|capture| capture.is_some())
}
pub fn matmul_with_kernel(
&self,
left: &GpuTensor,
right: &GpuTensor,
kernel: GpuMatmulKernel,
) -> Result<GpuTensor, GpuError> {
self.check_context(left)?;
self.check_context(right)?;
let dimensions = gpu_matmul_dimensions(left, right)?;
let (m, k, n, batch) = (
dimensions.rows,
dimensions.shared,
dimensions.columns,
dimensions.batch,
);
if kernel == GpuMatmulKernel::ExactTile16 && (m % 16 != 0 || k % 16 != 0 || n % 16 != 0) {
return Err(GpuError::InvalidShape(
"exact 16x16 matmul requires M, K, and N divisible by 16".into(),
));
}
if kernel == GpuMatmulKernel::Vectorized8x32 && (k % 4 != 0 || n % 4 != 0) {
return Err(GpuError::InvalidShape(
"vectorized matmul requires K and N divisible by four".into(),
));
}
let output = self.empty_tensor(dimensions.output_shape, "matmul-output")?;
let params = MatmulParams {
m: to_u32(m, "M")?,
k: to_u32(k, "K")?,
n: to_u32(n, "N")?,
batch: to_u32(batch, "batch count")?,
padding_0: 0,
padding_1: 0,
padding_2: 0,
padding_3: 0,
};
let (entry, workgroups) = match kernel {
GpuMatmulKernel::Tile16 => (
"matmul",
(
to_u32(n.div_ceil(16), "N workgroups")?,
to_u32(m.div_ceil(16), "M workgroups")?,
to_u32(batch, "batch workgroups")?,
),
),
GpuMatmulKernel::ExactTile16 => (
"matmul_exact_16",
(
to_u32(n / 16, "N workgroups")?,
to_u32(m / 16, "M workgroups")?,
to_u32(batch, "batch workgroups")?,
),
),
GpuMatmulKernel::RegisterBlocked32 => (
"matmul_register_32",
(
to_u32(n.div_ceil(32), "N workgroups")?,
to_u32(m.div_ceil(32), "M workgroups")?,
to_u32(batch, "batch workgroups")?,
),
),
GpuMatmulKernel::Vectorized8x32 => (
"matmul_vec4_8x32",
(
to_u32(n.div_ceil(32), "N workgroups")?,
to_u32(m.div_ceil(8), "M workgroups")?,
to_u32(batch, "batch workgroups")?,
),
),
};
self.dispatch_matmul(entry, left, right, &output, ¶ms, workgroups);
Ok(output)
}
pub fn batched_gemm(&self, left: &GpuTensor, right: &GpuTensor) -> Result<GpuTensor, GpuError> {
if left.shape.len() != 4 || right.shape.len() != 4 {
return Err(GpuError::InvalidShape(
"batched GEMM requires rank-4 [B, H, M, K] tensors".into(),
));
}
if left.shape[0] != right.shape[0] || left.shape[1] != right.shape[1] {
return Err(GpuError::InvalidShape(
"batched GEMM batch and head dimensions must match".into(),
));
}
self.matmul(left, right)
}
pub fn tiled_causal_attention(
&self,
query: &GpuTensor,
key: &GpuTensor,
value: &GpuTensor,
scale: f32,
past_length: usize,
) -> Result<GpuTensor, GpuError> {
let (output, _) = self.tiled_causal_attention_with_logsumexp(
query,
key,
value,
None,
scale,
past_length,
0.0,
0,
)?;
Ok(output)
}
fn tiled_causal_attention_with_logsumexp(
&self,
query: &GpuTensor,
key: &GpuTensor,
value: &GpuTensor,
attention_mask: Option<&GpuIndexTensor>,
scale: f32,
past_length: usize,
dropout_probability: f32,
dropout_seed: u32,
) -> Result<(GpuTensor, GpuTensor), GpuError> {
let params = self.tiled_attention_params(
query,
key,
value,
attention_mask.is_some(),
scale,
past_length,
dropout_probability,
dropout_seed,
)?;
let mask_buffer = attention_mask
.map(|mask| mask.buffer.as_ref())
.unwrap_or(query.buffer.as_ref());
let output = self.empty_tensor(query.shape.clone(), "tiled-attention-output")?;
let logsumexp = self.empty_tensor(
vec![query.shape[0], query.shape[1], query.shape[2]],
"tiled-attention-logsumexp",
)?;
let pipeline = &self.inner.pipelines["tiled_attention"];
let bind_group = self.cached_bind_group(
"tiled_attention",
"tiled-attention-bind-group",
"tiled-attention-params",
&[
(0, query.buffer.as_ref()),
(1, key.buffer.as_ref()),
(2, value.buffer.as_ref()),
(3, output.buffer.as_ref()),
(4, logsumexp.buffer.as_ref()),
(9, mask_buffer),
],
8,
bytemuck::bytes_of(¶ms),
);
self.submit(
pipeline,
&bind_group,
to_u32(query.shape[2], "tiled attention query workgroups")?,
to_u32(query.shape[1], "tiled attention head workgroups")?,
to_u32(query.shape[0], "tiled attention batch workgroups")?,
);
Ok((output, logsumexp))
}
fn backward_tiled_attention(
&self,
query: &GpuTensor,
key: &GpuTensor,
value: &GpuTensor,
output: &GpuTensor,
logsumexp: &GpuTensor,
upstream: &GpuTensor,
attention_mask: Option<&GpuIndexTensor>,
scale: f32,
past_length: usize,
dropout_probability: f32,
dropout_seed: u32,
) -> Result<(GpuTensor, GpuTensor, GpuTensor), GpuError> {
let params = self.tiled_attention_params(
query,
key,
value,
attention_mask.is_some(),
scale,
past_length,
dropout_probability,
dropout_seed,
)?;
let mask_buffer = attention_mask
.map(|mask| mask.buffer.as_ref())
.unwrap_or(query.buffer.as_ref());
self.check_context(output)?;
self.check_context(logsumexp)?;
self.check_context(upstream)?;
if output.shape != query.shape || upstream.shape != query.shape {
return Err(GpuError::InvalidShape(
"tiled attention output gradient shape differs".into(),
));
}
if logsumexp.shape != [query.shape[0], query.shape[1], query.shape[2]] {
return Err(GpuError::InvalidShape(
"tiled attention log-sum-exp shape differs".into(),
));
}
let query_gradient =
self.empty_tensor(query.shape.clone(), "tiled-attention-query-gradient")?;
let key_gradient = self.empty_tensor(key.shape.clone(), "tiled-attention-key-gradient")?;
let value_gradient =
self.empty_tensor(value.shape.clone(), "tiled-attention-value-gradient")?;
let query_pipeline = &self.inner.pipelines["tiled_attention_backward_query"];
let query_bind_group = self.cached_bind_group(
"tiled_attention_backward_query",
"tiled-attention-backward-query-bind-group",
"tiled-attention-backward-params",
&[
(0, query.buffer.as_ref()),
(1, key.buffer.as_ref()),
(2, value.buffer.as_ref()),
(3, output.buffer.as_ref()),
(4, upstream.buffer.as_ref()),
(5, query_gradient.buffer.as_ref()),
(7, logsumexp.buffer.as_ref()),
(9, mask_buffer),
],
8,
bytemuck::bytes_of(¶ms),
);
self.submit(
query_pipeline,
&query_bind_group,
to_u32(query.shape[2], "tiled attention query-gradient workgroups")?,
to_u32(query.shape[1], "tiled attention head workgroups")?,
to_u32(query.shape[0], "tiled attention batch workgroups")?,
);
let key_value_pipeline = &self.inner.pipelines["tiled_attention_backward_key_value"];
let key_value_bind_group = self.cached_bind_group(
"tiled_attention_backward_key_value",
"tiled-attention-backward-key-value-bind-group",
"tiled-attention-backward-params",
&[
(0, query.buffer.as_ref()),
(1, key.buffer.as_ref()),
(2, value.buffer.as_ref()),
(3, output.buffer.as_ref()),
(4, upstream.buffer.as_ref()),
(5, key_gradient.buffer.as_ref()),
(6, value_gradient.buffer.as_ref()),
(7, logsumexp.buffer.as_ref()),
],
8,
bytemuck::bytes_of(¶ms),
);
self.submit(
key_value_pipeline,
&key_value_bind_group,
to_u32(key.shape[2], "tiled attention key-gradient workgroups")?,
to_u32(key.shape[1], "tiled attention head workgroups")?,
to_u32(key.shape[0], "tiled attention batch workgroups")?,
);
Ok((query_gradient, key_gradient, value_gradient))
}
fn tiled_attention_params(
&self,
query: &GpuTensor,
key: &GpuTensor,
value: &GpuTensor,
use_attention_mask: bool,
scale: f32,
past_length: usize,
dropout_probability: f32,
dropout_seed: u32,
) -> Result<TiledAttentionParams, GpuError> {
for tensor in [query, key, value] {
self.check_context(tensor)?;
}
if query.shape.len() != 4 || key.shape.len() != 4 || value.shape != key.shape {
return Err(GpuError::InvalidShape(
"tiled attention requires Q [B,H,Q,D] and matching K/V [B,H,K,D]".into(),
));
}
if query.shape[0] != key.shape[0] || query.shape[3] != key.shape[3] {
return Err(GpuError::InvalidShape(
"tiled attention batch or width dimensions differ".into(),
));
}
if key.shape[1] == 0 || query.shape[1] % key.shape[1] != 0 {
return Err(GpuError::InvalidShape(
"tiled attention query heads must be divisible by key/value heads".into(),
));
}
if past_length + query.shape[2] != key.shape[2] {
return Err(GpuError::InvalidShape(
"cached attention dimensions are inconsistent".into(),
));
}
if query.shape[3] > 256 {
return Err(GpuError::InvalidShape(
"tiled GPU attention supports head widths up to 256".into(),
));
}
if !scale.is_finite() || scale <= 0.0 {
return Err(GpuError::InvalidShape(
"tiled attention scale must be positive".into(),
));
}
if !dropout_probability.is_finite() || !(0.0..1.0).contains(&dropout_probability) {
return Err(GpuError::InvalidShape(
"attention dropout probability must be in [0, 1)".into(),
));
}
Ok(TiledAttentionParams {
batch_size: to_u32(query.shape[0], "tiled attention batch size")?,
query_heads: to_u32(query.shape[1], "tiled attention query head count")?,
key_value_heads: to_u32(key.shape[1], "tiled attention key/value head count")?,
query_length: to_u32(query.shape[2], "tiled attention query length")?,
key_length: to_u32(key.shape[2], "tiled attention key length")?,
head_dim: to_u32(query.shape[3], "tiled attention head width")?,
past_length: to_u32(past_length, "tiled attention past length")?,
use_attention_mask: u32::from(use_attention_mask),
scale,
dropout_probability,
dropout_seed,
padding: 0,
})
}
pub fn add(&self, left: &GpuTensor, right: &GpuTensor) -> Result<GpuTensor, GpuError> {
self.check_context(left)?;
self.check_context(right)?;
if left.shape != right.shape {
return Err(GpuError::InvalidShape(
"GPU add requires equal shapes".into(),
));
}
self.dispatch_elementwise(
"add",
left,
Some(right),
left.shape.clone(),
1,
left.length,
0.0,
)
}
pub fn bias_add(&self, input: &GpuTensor, bias: &GpuTensor) -> Result<GpuTensor, GpuError> {
self.check_context(input)?;
self.check_context(bias)?;
let width = *input.shape.last().ok_or_else(|| {
GpuError::InvalidShape("bias add requires a non-empty input shape".into())
})?;
if bias.shape.as_slice() != [width] {
return Err(GpuError::InvalidShape(format!(
"bias shape {:?} must be [{width}] to match input {:?}",
bias.shape, input.shape
)));
}
self.dispatch_elementwise(
"bias_add",
input,
Some(bias),
input.shape.clone(),
1,
width,
0.0,
)
}
pub fn mul(&self, left: &GpuTensor, right: &GpuTensor) -> Result<GpuTensor, GpuError> {
self.check_context(left)?;
self.check_context(right)?;
if left.shape != right.shape {
return Err(GpuError::InvalidShape(
"GPU multiply requires equal shapes".into(),
));
}
self.dispatch_elementwise(
"mul",
left,
Some(right),
left.shape.clone(),
1,
left.length,
0.0,
)
}
pub fn scale(&self, input: &GpuTensor, factor: f32) -> Result<GpuTensor, GpuError> {
self.check_context(input)?;
self.dispatch_elementwise(
"scale",
input,
None,
input.shape.clone(),
1,
input.length,
factor,
)
}
pub fn rope(
&self,
input: &GpuTensor,
sequence_dim: usize,
theta: f32,
position_offset: usize,
) -> Result<GpuTensor, GpuError> {
self.rope_with_scaling(input, sequence_dim, theta, position_offset, None)
}
pub fn rope_with_scaling(
&self,
input: &GpuTensor,
sequence_dim: usize,
theta: f32,
position_offset: usize,
scaling: Option<&crate::rope::RopeScaling>,
) -> Result<GpuTensor, GpuError> {
self.check_context(input)?;
if input.shape.len() != 4 || sequence_dim != 2 {
return Err(GpuError::InvalidShape(
"GPU RoPE requires [B, H, S, D] with sequence dimension 2".into(),
));
}
if input.shape[3] % 2 != 0 || theta <= 0.0 {
return Err(GpuError::InvalidShape(
"GPU RoPE needs an even head width and positive theta".into(),
));
}
self.dispatch_attention(
"rope",
input,
input.shape.clone(),
AttentionParams {
length: to_u32(input.length, "RoPE element count")?,
dim_0: to_u32(input.shape[0], "batch size")?,
dim_1: to_u32(input.shape[1], "head count")?,
dim_2: to_u32(input.shape[2], "sequence length")?,
dim_3: to_u32(input.shape[3], "head width")?,
past_length: 0,
position_offset: to_u32(position_offset, "position offset")?,
theta,
rope_type: scaling.map_or(0, crate::rope::RopeScaling::kind_id),
rope_original_context: to_u32(
scaling.map_or(0, |scaling| scaling.original_max_position_embeddings),
"RoPE original context",
)?,
rope_factor: scaling.map_or(1.0, |scaling| scaling.factor),
rope_attention_factor: scaling
.map_or(1.0, crate::rope::RopeScaling::attention_factor),
rope_beta_fast: scaling.map_or(32.0, |scaling| scaling.beta_fast),
rope_beta_slow: scaling.map_or(1.0, |scaling| scaling.beta_slow),
padding_0: 0,
padding_1: 0,
},
input.length.div_ceil(2).div_ceil(256),
)
}
pub fn permute(&self, input: &GpuTensor, axes: &[usize]) -> Result<GpuTensor, GpuError> {
self.check_context(input)?;
if input.shape.len() == 2 && axes == [1, 0] {
return self.dispatch_attention(
"transpose_2d",
input,
vec![input.shape[1], input.shape[0]],
AttentionParams {
length: to_u32(input.length, "transpose element count")?,
dim_0: to_u32(input.shape[0], "transpose rows")?,
dim_1: to_u32(input.shape[1], "transpose columns")?,
dim_2: 0,
dim_3: 0,
past_length: 0,
position_offset: 0,
theta: 0.0,
..AttentionParams::default()
},
input.length.div_ceil(256),
);
}
if input.shape.len() != 4 || axes.len() != 4 {
return Err(GpuError::InvalidShape(
"GPU permute supports 2D transpose and attention's 4D permutations".into(),
));
}
let (entry, output_shape) = if axes == [0, 2, 1, 3] {
(
"swap_middle",
vec![
input.shape[0],
input.shape[2],
input.shape[1],
input.shape[3],
],
)
} else if axes == [0, 1, 3, 2] {
(
"swap_last",
vec![
input.shape[0],
input.shape[1],
input.shape[3],
input.shape[2],
],
)
} else {
return Err(GpuError::InvalidShape(
"unsupported GPU attention permutation".into(),
));
};
self.dispatch_attention(
entry,
input,
output_shape,
AttentionParams {
length: to_u32(input.length, "permute element count")?,
dim_0: to_u32(input.shape[0], "dimension 0")?,
dim_1: to_u32(input.shape[1], "dimension 1")?,
dim_2: to_u32(input.shape[2], "dimension 2")?,
dim_3: to_u32(input.shape[3], "dimension 3")?,
past_length: 0,
position_offset: 0,
theta: 0.0,
..AttentionParams::default()
},
input.length.div_ceil(256),
)
}
pub fn causal_mask(
&self,
input: &GpuTensor,
past_length: usize,
) -> Result<GpuTensor, GpuError> {
self.check_context(input)?;
if input.shape.len() != 4 {
return Err(GpuError::InvalidShape(
"GPU causal mask requires [B, H, Q, K]".into(),
));
}
let query_length = input.shape[2];
let key_length = input.shape[3];
if past_length + query_length != key_length {
return Err(GpuError::InvalidShape(
"cached attention dimensions are inconsistent".into(),
));
}
self.dispatch_attention(
"causal_mask",
input,
input.shape.clone(),
AttentionParams {
length: to_u32(input.length, "mask element count")?,
dim_0: to_u32(input.shape[0], "batch size")?,
dim_1: to_u32(query_length, "query length")?,
dim_2: to_u32(key_length, "key length")?,
dim_3: 0,
past_length: to_u32(past_length, "past length")?,
position_offset: 0,
theta: 0.0,
..AttentionParams::default()
},
input.length.div_ceil(256),
)
}
fn causal_mask_backward(
&self,
input: &GpuTensor,
past_length: usize,
) -> Result<GpuTensor, GpuError> {
self.check_context(input)?;
if input.shape.len() != 4 {
return Err(GpuError::InvalidShape(
"GPU causal-mask backward requires [B, H, Q, K]".into(),
));
}
self.dispatch_attention(
"causal_mask_backward",
input,
input.shape.clone(),
AttentionParams {
length: to_u32(input.length, "causal gradient length")?,
dim_0: to_u32(input.shape[0], "causal gradient batch")?,
dim_1: to_u32(input.shape[2], "causal gradient queries")?,
dim_2: to_u32(input.shape[3], "causal gradient keys")?,
dim_3: to_u32(input.shape[1], "causal gradient heads")?,
past_length: to_u32(past_length, "causal past length")?,
position_offset: 0,
theta: 0.0,
..AttentionParams::default()
},
input.length.div_ceil(256),
)
}
pub fn append_sequence(
&self,
existing: Option<&GpuTensor>,
addition: &GpuTensor,
maximum: usize,
) -> Result<GpuTensor, GpuError> {
self.check_context(addition)?;
if addition.shape.len() != 4 || addition.shape[2] > maximum {
return Err(GpuError::InvalidShape(
"KV cache tensors require [B, H, S, D] within capacity".into(),
));
}
let Some(existing) = existing else {
return Ok(addition.clone());
};
self.check_context(existing)?;
if existing.shape[0] != addition.shape[0]
|| existing.shape[1] != addition.shape[1]
|| existing.shape[3] != addition.shape[3]
{
return Err(GpuError::InvalidShape("KV cache shapes differ".into()));
}
let outer = addition.shape[0] * addition.shape[1];
let width = addition.shape[3];
let added_length = addition.shape[2];
let existing_length = existing.shape[2];
let retained_length = existing_length.min(maximum - added_length);
let output_length = retained_length + added_length;
let output = self.empty_tensor(
vec![addition.shape[0], addition.shape[1], output_length, width],
"kv-cache-output",
)?;
let mut encoder =
self.inner
.device
.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("kv-cache-append-encoder"),
});
let element_bytes = std::mem::size_of::<f32>() as u64;
for outer_index in 0..outer {
let output_base = outer_index * output_length * width;
if retained_length > 0 {
let source_base =
(outer_index * existing_length + existing_length - retained_length) * width;
encoder.copy_buffer_to_buffer(
&existing.buffer.buffer,
source_base as u64 * element_bytes,
&output.buffer.buffer,
output_base as u64 * element_bytes,
(retained_length * width) as u64 * element_bytes,
);
}
let addition_base = outer_index * added_length * width;
encoder.copy_buffer_to_buffer(
&addition.buffer.buffer,
addition_base as u64 * element_bytes,
&output.buffer.buffer,
(output_base + retained_length * width) as u64 * element_bytes,
(added_length * width) as u64 * element_bytes,
);
}
self.inner.queue.submit([encoder.finish()]);
Ok(output)
}
pub fn relu(&self, input: &GpuTensor) -> Result<GpuTensor, GpuError> {
self.unary("relu", input)
}
pub fn gelu(&self, input: &GpuTensor) -> Result<GpuTensor, GpuError> {
self.unary("gelu", input)
}
pub fn silu(&self, input: &GpuTensor) -> Result<GpuTensor, GpuError> {
self.unary("silu", input)
}
pub fn softmax(&self, input: &GpuTensor) -> Result<GpuTensor, GpuError> {
self.check_context(input)?;
let columns = *input
.shape
.last()
.ok_or_else(|| GpuError::InvalidShape("softmax needs a feature axis".into()))?;
let rows = input.length / columns;
self.dispatch_elementwise(
"softmax",
input,
None,
input.shape.clone(),
rows,
columns,
0.0,
)
}
pub fn rms_norm(
&self,
input: &GpuTensor,
weight: &GpuTensor,
epsilon: f32,
) -> Result<GpuTensor, GpuError> {
self.check_context(input)?;
self.check_context(weight)?;
let columns = *input
.shape
.last()
.ok_or_else(|| GpuError::InvalidShape("RMSNorm needs a feature axis".into()))?;
if weight.shape != [columns] {
return Err(GpuError::InvalidShape(
"RMSNorm weight must match the final input dimension".into(),
));
}
if epsilon <= 0.0 {
return Err(GpuError::InvalidShape(
"RMSNorm epsilon must be positive".into(),
));
}
let rows = input.length / columns;
self.dispatch_elementwise(
"rms_norm",
input,
Some(weight),
input.shape.clone(),
rows,
columns,
epsilon,
)
}
pub fn fused_residual_rms_norm(
&self,
left: &GpuTensor,
right: &GpuTensor,
weight: &GpuTensor,
epsilon: f32,
) -> Result<(GpuTensor, GpuTensor), GpuError> {
if left.shape != right.shape {
return Err(GpuError::InvalidShape(
"fused residual RMSNorm inputs must have equal shapes".into(),
));
}
let width = *left.shape.last().unwrap();
if weight.shape != [width] {
return Err(GpuError::InvalidShape(
"fused residual RMSNorm weight has wrong width".into(),
));
}
let residual = self.empty_tensor(left.shape.clone(), "fused-residual-output")?;
let normalized = self.empty_tensor(left.shape.clone(), "fused-rms-output")?;
self.dispatch_fusion(
"residual_rms_norm",
[Some(left), Some(right), None, Some(weight), None, None],
[Some(&residual), Some(&normalized), None],
FusionParams {
length: to_u32(left.length, "fusion length")?,
rows: to_u32(left.length / width, "fusion rows")?,
input_width: to_u32(width, "fusion width")?,
output_width: 0,
query_length: 0,
key_length: 0,
past_length: 0,
padding: 0,
epsilon,
scalar_0: 0.0,
scalar_1: 0.0,
scalar_2: 0.0,
},
(to_u32(left.length / width, "fusion workgroups")?, 1, 1),
)?;
Ok((residual, normalized))
}
pub fn fused_swiglu(&self, gate: &GpuTensor, up: &GpuTensor) -> Result<GpuTensor, GpuError> {
if gate.shape != up.shape {
return Err(GpuError::InvalidShape(
"fused SwiGLU inputs must have equal shapes".into(),
));
}
let output = self.empty_tensor(gate.shape.clone(), "fused-swiglu-output")?;
self.dispatch_fusion(
"swiglu",
[Some(gate), Some(up), None, None, None, None],
[Some(&output), None, None],
fusion_params(gate.length)?,
(
to_u32(gate.length.div_ceil(256), "SwiGLU workgroups")?,
1,
1,
),
)?;
Ok(output)
}
pub fn fused_masked_softmax(
&self,
input: &GpuTensor,
past_length: usize,
) -> Result<GpuTensor, GpuError> {
if input.shape.len() != 4 {
return Err(GpuError::InvalidShape(
"fused masked Softmax requires [B, H, Q, K]".into(),
));
}
let query_length = input.shape[2];
let key_length = input.shape[3];
let rows = input.length / key_length;
let output = self.empty_tensor(input.shape.clone(), "fused-masked-softmax-output")?;
self.dispatch_fusion(
"masked_softmax",
[Some(input), None, None, None, None, None],
[Some(&output), None, None],
FusionParams {
length: to_u32(input.length, "masked Softmax length")?,
rows: to_u32(rows, "masked Softmax rows")?,
input_width: 0,
output_width: 0,
query_length: to_u32(query_length, "masked Softmax queries")?,
key_length: to_u32(key_length, "masked Softmax keys")?,
past_length: to_u32(past_length, "masked Softmax past length")?,
padding: 0,
epsilon: 0.0,
scalar_0: 0.0,
scalar_1: 0.0,
scalar_2: 0.0,
},
(to_u32(rows, "masked Softmax workgroups")?, 1, 1),
)?;
Ok(output)
}
pub fn matmul_tensor(&self, left: &Tensor, right: &Tensor) -> Result<Tensor, GpuError> {
let output = self.matmul(&left.gpu_storage(self)?, &right.gpu_storage(self)?)?;
Ok(Tensor::from_gpu_operation(
output,
left.requires_grad() || right.requires_grad(),
Op::MatMul(left.clone(), right.clone()),
))
}
pub fn batched_gemm_tensor(&self, left: &Tensor, right: &Tensor) -> Result<Tensor, GpuError> {
let output = self.batched_gemm(&left.gpu_storage(self)?, &right.gpu_storage(self)?)?;
Ok(Tensor::from_gpu_operation(
output,
left.requires_grad() || right.requires_grad(),
Op::MatMul(left.clone(), right.clone()),
))
}
pub fn tiled_causal_attention_tensor(
&self,
query: &Tensor,
key: &Tensor,
value: &Tensor,
scale: f32,
past_length: usize,
) -> Result<Tensor, GpuError> {
self.tiled_causal_attention_tensor_with_optional_mask(
query,
key,
value,
scale,
past_length,
None,
0.0,
0,
)
}
pub fn tiled_causal_attention_tensor_with_mask(
&self,
query: &Tensor,
key: &Tensor,
value: &Tensor,
scale: f32,
past_length: usize,
attention_mask: &IndexTensor,
) -> Result<Tensor, GpuError> {
self.tiled_causal_attention_tensor_with_optional_mask(
query,
key,
value,
scale,
past_length,
Some(attention_mask),
0.0,
0,
)
}
pub(crate) fn tiled_causal_attention_tensor_with_optional_mask(
&self,
query: &Tensor,
key: &Tensor,
value: &Tensor,
scale: f32,
past_length: usize,
attention_mask: Option<&IndexTensor>,
dropout_probability: f32,
dropout_seed: u32,
) -> Result<Tensor, GpuError> {
let attention_mask_gpu = attention_mask
.map(|mask| {
if mask.shape() != [query.shape()[0], key.shape()[2]] {
return Err(GpuError::InvalidShape(
"attention mask must have shape [B, K]".into(),
));
}
if !mask.data().iter().all(|value| *value <= 1) {
return Err(GpuError::InvalidShape(
"attention mask values must be zero or one".into(),
));
}
mask.to_gpu(self)
})
.transpose()?;
let (output, logsumexp) = self.tiled_causal_attention_with_logsumexp(
&query.gpu_storage(self)?,
&key.gpu_storage(self)?,
&value.gpu_storage(self)?,
attention_mask_gpu.as_ref(),
scale,
past_length,
dropout_probability,
dropout_seed,
)?;
let logsumexp = Tensor::from_gpu_operation(logsumexp, false, Op::Leaf);
Ok(Tensor::from_gpu_operation(
output,
query.requires_grad() || key.requires_grad() || value.requires_grad(),
Op::TiledAttention {
query: query.clone(),
key: key.clone(),
value: value.clone(),
attention_mask: attention_mask.cloned(),
logsumexp,
scale,
past_length,
dropout_probability,
dropout_seed,
},
))
}
pub fn add_tensor(&self, left: &Tensor, right: &Tensor) -> Result<Tensor, GpuError> {
let output = self.add(&left.gpu_storage(self)?, &right.gpu_storage(self)?)?;
Ok(Tensor::from_gpu_operation(
output,
left.requires_grad() || right.requires_grad(),
Op::Add(left.clone(), right.clone()),
))
}
pub fn bias_add_tensor(&self, input: &Tensor, bias: &Tensor) -> Result<Tensor, GpuError> {
let output = self.bias_add(&input.gpu_storage(self)?, &bias.gpu_storage(self)?)?;
Ok(Tensor::from_gpu_operation(
output,
input.requires_grad() || bias.requires_grad(),
Op::Add(input.clone(), bias.clone()),
))
}
pub fn mul_tensor(&self, left: &Tensor, right: &Tensor) -> Result<Tensor, GpuError> {
let output = self.mul(&left.gpu_storage(self)?, &right.gpu_storage(self)?)?;
Ok(Tensor::from_gpu_operation(
output,
left.requires_grad() || right.requires_grad(),
Op::Mul(left.clone(), right.clone()),
))
}
pub fn scale_tensor(&self, input: &Tensor, factor: f32) -> Result<Tensor, GpuError> {
let scalar = Tensor::scalar(factor, false);
let output = self.scale(&input.gpu_storage(self)?, factor)?;
Ok(Tensor::from_gpu_operation(
output,
input.requires_grad(),
Op::Mul(input.clone(), scalar),
))
}
pub fn dropout_tensor(
&self,
input: &Tensor,
probability: f32,
seed: u64,
) -> Result<Tensor, GpuError> {
crate::validate_dropout_probability(probability);
if probability == 0.0 {
return Ok(input.clone());
}
let mask = Tensor::new(
crate::dropout_mask_values(input.numel(), probability, seed),
input.shape(),
false,
)
.to_gpu(self)?;
self.mul_tensor(input, &mask)
}
pub fn rope_tensor(
&self,
input: &Tensor,
sequence_dim: usize,
theta: f32,
position_offset: usize,
) -> Result<Tensor, GpuError> {
self.rope_tensor_with_scaling(input, sequence_dim, theta, position_offset, None)
}
pub fn rope_tensor_with_scaling(
&self,
input: &Tensor,
sequence_dim: usize,
theta: f32,
position_offset: usize,
scaling: Option<&crate::rope::RopeScaling>,
) -> Result<Tensor, GpuError> {
let output = self.rope_with_scaling(
&input.gpu_storage(self)?,
sequence_dim,
theta,
position_offset,
scaling,
)?;
Ok(Tensor::from_gpu_operation(
output,
input.requires_grad(),
Op::Rope {
input: input.clone(),
sequence_dim,
theta,
position_offset,
scaling: scaling.cloned(),
},
))
}
pub fn permute_tensor(&self, input: &Tensor, axes: Vec<usize>) -> Result<Tensor, GpuError> {
let output = self.permute(&input.gpu_storage(self)?, &axes)?;
Ok(Tensor::from_gpu_operation(
output,
input.requires_grad(),
Op::Permute {
input: input.clone(),
axes,
},
))
}
pub fn causal_mask_tensor(
&self,
input: &Tensor,
past_length: usize,
) -> Result<Tensor, GpuError> {
let output = self.causal_mask(&input.gpu_storage(self)?, past_length)?;
Ok(Tensor::from_gpu_operation(
output,
input.requires_grad(),
Op::CausalMask {
input: input.clone(),
past_length,
},
))
}
pub fn append_sequence_tensor(
&self,
existing: Option<&Tensor>,
addition: &Tensor,
maximum: usize,
) -> Result<Tensor, GpuError> {
let existing_gpu = existing
.map(|tensor| tensor.gpu_storage(self))
.transpose()?;
let addition_gpu = addition.gpu_storage(self)?;
let output = self.append_sequence(existing_gpu.as_ref(), &addition_gpu, maximum)?;
Ok(Tensor::from_gpu_operation(output, false, Op::Leaf))
}
pub fn relu_tensor(&self, input: &Tensor) -> Result<Tensor, GpuError> {
self.unary_tensor(input, "relu", Op::Relu(input.clone()))
}
pub fn gelu_tensor(&self, input: &Tensor) -> Result<Tensor, GpuError> {
self.unary_tensor(input, "gelu", Op::Gelu(input.clone()))
}
pub fn silu_tensor(&self, input: &Tensor) -> Result<Tensor, GpuError> {
self.unary_tensor(input, "silu", Op::Silu(input.clone()))
}
pub fn softmax_tensor(&self, input: &Tensor) -> Result<Tensor, GpuError> {
let output = self.softmax(&input.gpu_storage(self)?)?;
Ok(Tensor::from_gpu_operation(
output,
input.requires_grad(),
Op::SoftmaxLast(input.clone()),
))
}
pub fn rms_norm_tensor(
&self,
input: &Tensor,
weight: &Tensor,
epsilon: f32,
) -> Result<Tensor, GpuError> {
let output = self.rms_norm(
&input.gpu_storage(self)?,
&weight.gpu_storage(self)?,
epsilon,
)?;
Ok(Tensor::from_gpu_operation(
output,
input.requires_grad() || weight.requires_grad(),
Op::RmsNormGpu {
input: input.clone(),
weight: weight.clone(),
epsilon,
},
))
}
pub fn residual_rms_norm_tensor(
&self,
left: &Tensor,
right: &Tensor,
weight: &Tensor,
epsilon: f32,
) -> Result<(Tensor, Tensor), GpuError> {
let (residual, normalized) = self.fused_residual_rms_norm(
&left.gpu_storage(self)?,
&right.gpu_storage(self)?,
&weight.gpu_storage(self)?,
epsilon,
)?;
let residual = Tensor::from_gpu_operation(
residual,
left.requires_grad() || right.requires_grad(),
Op::Add(left.clone(), right.clone()),
);
let normalized = Tensor::from_gpu_operation(
normalized,
left.requires_grad() || right.requires_grad() || weight.requires_grad(),
Op::ResidualRmsNormGpu {
left: left.clone(),
right: right.clone(),
weight: weight.clone(),
epsilon,
},
);
Ok((residual, normalized))
}
pub fn swiglu_tensor(&self, gate: &Tensor, up: &Tensor) -> Result<Tensor, GpuError> {
let output = self.fused_swiglu(&gate.gpu_storage(self)?, &up.gpu_storage(self)?)?;
Ok(Tensor::from_gpu_operation(
output,
gate.requires_grad() || up.requires_grad(),
Op::SwiGluGpu {
gate: gate.clone(),
up: up.clone(),
},
))
}
pub fn masked_softmax_tensor(
&self,
input: &Tensor,
past_length: usize,
) -> Result<Tensor, GpuError> {
let output = self.fused_masked_softmax(&input.gpu_storage(self)?, past_length)?;
Ok(Tensor::from_gpu_operation(
output,
input.requires_grad(),
Op::MaskedSoftmaxGpu {
input: input.clone(),
past_length,
},
))
}
pub fn cross_entropy_tensor(
&self,
logits: &Tensor,
targets: &IndexTensor,
) -> Result<Tensor, GpuError> {
self.cross_entropy_tensor_with_ignore_index(logits, targets, None)
}
pub fn cross_entropy_tensor_ignore_index(
&self,
logits: &Tensor,
targets: &IndexTensor,
ignore_index: usize,
) -> Result<Tensor, GpuError> {
self.cross_entropy_tensor_with_ignore_index(logits, targets, Some(ignore_index))
}
fn cross_entropy_tensor_with_ignore_index(
&self,
logits: &Tensor,
targets: &IndexTensor,
ignore_index: Option<usize>,
) -> Result<Tensor, GpuError> {
let logits_gpu = logits.gpu_storage(self)?;
let targets_gpu = targets.to_gpu(self)?;
let shape = logits.shape();
let columns = *shape
.last()
.ok_or_else(|| GpuError::InvalidShape("cross entropy needs classes".into()))?;
let rows = logits.numel() / columns;
if targets.shape() != shape[..shape.len() - 1] || targets.numel() != rows {
return Err(GpuError::InvalidShape(
"cross entropy target shape mismatch".into(),
));
}
let target_data = targets.data();
if !target_data
.iter()
.all(|target| Some(*target) == ignore_index || *target < columns)
{
return Err(GpuError::InvalidShape(
"cross entropy target is out of range".into(),
));
}
let valid_rows = target_data
.iter()
.filter(|target| Some(**target) != ignore_index)
.count();
if valid_rows == 0 {
return Err(GpuError::InvalidShape(
"cross entropy needs at least one non-ignored target".into(),
));
}
let output = self.dispatch_index(
"cross_entropy_forward",
Some(&logits_gpu),
&targets_gpu,
None,
vec![1],
BackwardParams {
length: to_u32(logits.numel(), "logit count")?,
rows: to_u32(rows, "cross entropy rows")?,
columns: to_u32(columns, "class count")?,
inner: u32::from(ignore_index.is_some()),
batch: to_u32(valid_rows, "cross entropy valid rows")?,
padding: to_u32(ignore_index.unwrap_or(0), "cross entropy ignore index")?,
epsilon: 0.0,
scalar: 0.0,
},
1,
)?;
Ok(Tensor::from_gpu_operation(
output,
logits.requires_grad(),
Op::CrossEntropy {
logits: logits.clone(),
targets: targets.clone(),
ignore_index,
},
))
}
pub fn embedding_tensor(
&self,
weight: &Tensor,
indices: &IndexTensor,
) -> Result<Tensor, GpuError> {
let weight_gpu = weight.gpu_storage(self)?;
let indices_gpu = indices.to_gpu(self)?;
let weight_shape = weight.shape();
if weight_shape.len() != 2 {
return Err(GpuError::InvalidShape(
"embedding weight must be two-dimensional".into(),
));
}
indices.validate_upper_bound(weight_shape[0], "embedding token ID");
let width = weight_shape[1];
let mut output_shape = indices.shape();
output_shape.push(width);
let length = indices.numel() * width;
let output = self.dispatch_index(
"embedding_forward",
Some(&weight_gpu),
&indices_gpu,
None,
output_shape,
BackwardParams {
length: to_u32(length, "embedding output length")?,
rows: to_u32(indices.numel(), "embedding positions")?,
columns: to_u32(width, "embedding width")?,
inner: 0,
batch: 0,
padding: 0,
epsilon: 0.0,
scalar: 0.0,
},
length.div_ceil(256),
)?;
Ok(Tensor::from_gpu_operation(
output,
weight.requires_grad(),
Op::Embedding {
weight: weight.clone(),
indices: indices.clone(),
},
))
}
fn backward_matmul(
&self,
left: &GpuTensor,
right: &GpuTensor,
upstream: &GpuTensor,
) -> Result<(GpuTensor, GpuTensor), GpuError> {
let rank = left.shape.len();
let rows = left.shape[rank - 2];
let inner = left.shape[rank - 1];
let columns = right.shape[rank - 1];
let batch = left.shape[..rank - 2].iter().product::<usize>();
let params = BackwardParams {
length: to_u32(upstream.length, "matmul gradient length")?,
rows: to_u32(rows, "matmul rows")?,
columns: to_u32(columns, "matmul columns")?,
inner: to_u32(inner, "matmul inner")?,
batch: to_u32(batch, "matmul batches")?,
padding: 0,
epsilon: 0.0,
scalar: 0.0,
};
let left_gradient = self.dispatch_backward(
"matmul_left",
None,
Some(right),
Some(upstream),
left.shape.clone(),
params,
left.length.div_ceil(256),
)?;
let right_gradient = self.dispatch_backward(
"matmul_right",
Some(left),
None,
Some(upstream),
right.shape.clone(),
params,
right.length.div_ceil(256),
)?;
Ok((left_gradient, right_gradient))
}
fn backward_multiply(
&self,
left: &GpuTensor,
right: &GpuTensor,
upstream: &GpuTensor,
) -> Result<(GpuTensor, GpuTensor), GpuError> {
let params = simple_backward_params(upstream.length)?;
let left_gradient = self.dispatch_backward(
"multiply_left",
None,
Some(right),
Some(upstream),
upstream.shape.clone(),
params,
upstream.length.div_ceil(256),
)?;
let right_gradient = self.dispatch_backward(
"multiply_right",
Some(left),
None,
Some(upstream),
upstream.shape.clone(),
params,
upstream.length.div_ceil(256),
)?;
Ok((left_gradient, right_gradient))
}
fn backward_unary(
&self,
entry: &'static str,
input: &GpuTensor,
upstream: &GpuTensor,
) -> Result<GpuTensor, GpuError> {
self.dispatch_backward(
entry,
Some(input),
None,
Some(upstream),
input.shape.clone(),
simple_backward_params(input.length)?,
input.length.div_ceil(256),
)
}
fn backward_softmax(
&self,
output: &GpuTensor,
upstream: &GpuTensor,
) -> Result<GpuTensor, GpuError> {
let columns = *output.shape.last().unwrap();
self.dispatch_backward(
"softmax_backward",
Some(output),
None,
Some(upstream),
output.shape.clone(),
BackwardParams {
columns: to_u32(columns, "softmax columns")?,
..simple_backward_params(output.length)?
},
output.length.div_ceil(256),
)
}
fn backward_rms_norm(
&self,
input: &GpuTensor,
weight: &GpuTensor,
upstream: &GpuTensor,
epsilon: f32,
) -> Result<(GpuTensor, GpuTensor), GpuError> {
let columns = weight.length;
let rows = input.length / columns;
let params = BackwardParams {
rows: to_u32(rows, "RMSNorm rows")?,
columns: to_u32(columns, "RMSNorm columns")?,
epsilon,
..simple_backward_params(input.length)?
};
let input_gradient = self.dispatch_backward(
"rms_input",
Some(input),
Some(weight),
Some(upstream),
input.shape.clone(),
params,
input.length.div_ceil(256),
)?;
let weight_gradient = self.dispatch_backward(
"rms_weight",
Some(input),
None,
Some(upstream),
weight.shape.clone(),
params,
columns.div_ceil(256),
)?;
Ok((input_gradient, weight_gradient))
}
fn backward_embedding(
&self,
indices: &GpuIndexTensor,
upstream: &GpuTensor,
weight_shape: Vec<usize>,
) -> Result<GpuTensor, GpuError> {
let length = weight_shape.iter().product::<usize>();
self.dispatch_index(
"embedding_weight",
None,
indices,
Some(upstream),
weight_shape.clone(),
BackwardParams {
length: to_u32(length, "embedding weight length")?,
rows: to_u32(indices.length, "embedding positions")?,
columns: to_u32(weight_shape[1], "embedding width")?,
inner: 0,
batch: 0,
padding: 0,
epsilon: 0.0,
scalar: 0.0,
},
length.div_ceil(256),
)
}
fn backward_cross_entropy(
&self,
logits: &GpuTensor,
targets: &GpuIndexTensor,
upstream: &GpuTensor,
valid_rows: usize,
ignore_index: Option<usize>,
) -> Result<GpuTensor, GpuError> {
let columns = *logits.shape.last().unwrap();
let rows = logits.length / columns;
self.dispatch_index(
"cross_entropy_backward",
Some(logits),
targets,
Some(upstream),
logits.shape.clone(),
BackwardParams {
length: to_u32(logits.length, "logit count")?,
rows: to_u32(rows, "cross entropy rows")?,
columns: to_u32(columns, "class count")?,
inner: u32::from(ignore_index.is_some()),
batch: to_u32(valid_rows, "cross entropy valid rows")?,
padding: to_u32(ignore_index.unwrap_or(0), "cross entropy ignore index")?,
epsilon: 0.0,
scalar: 0.0,
},
logits.length.div_ceil(256),
)
}
fn backward_rope(
&self,
upstream: &GpuTensor,
sequence_dim: usize,
theta: f32,
position_offset: usize,
scaling: Option<&crate::rope::RopeScaling>,
) -> Result<GpuTensor, GpuError> {
if sequence_dim != 2 || upstream.shape.len() != 4 {
return Err(GpuError::InvalidShape(
"GPU RoPE backward requires [B, H, S, D]".into(),
));
}
self.dispatch_attention(
"rope_backward",
upstream,
upstream.shape.clone(),
AttentionParams {
length: to_u32(upstream.length, "RoPE gradient length")?,
dim_0: to_u32(upstream.shape[0], "RoPE batch")?,
dim_1: to_u32(upstream.shape[1], "RoPE heads")?,
dim_2: to_u32(upstream.shape[2], "RoPE sequence")?,
dim_3: to_u32(upstream.shape[3], "RoPE width")?,
past_length: 0,
position_offset: to_u32(position_offset, "RoPE position")?,
theta,
rope_type: scaling.map_or(0, crate::rope::RopeScaling::kind_id),
rope_original_context: to_u32(
scaling.map_or(0, |scaling| scaling.original_max_position_embeddings),
"RoPE original context",
)?,
rope_factor: scaling.map_or(1.0, |scaling| scaling.factor),
rope_attention_factor: scaling
.map_or(1.0, crate::rope::RopeScaling::attention_factor),
rope_beta_fast: scaling.map_or(32.0, |scaling| scaling.beta_fast),
rope_beta_slow: scaling.map_or(1.0, |scaling| scaling.beta_slow),
padding_0: 0,
padding_1: 0,
},
upstream.length.div_ceil(2).div_ceil(256),
)
}
pub(crate) fn squared_norm(&self, input: &GpuTensor) -> Result<GpuTensor, GpuError> {
self.dispatch_backward(
"squared_norm",
Some(input),
None,
None,
vec![1],
simple_backward_params(input.length)?,
1,
)
}
pub(crate) fn adamw(
&self,
parameter: &GpuTensor,
gradient: &GpuTensor,
first: &GpuTensor,
second: &GpuTensor,
params: AdamWParams,
) -> Result<(), GpuError> {
for tensor in [parameter, gradient, first, second] {
self.check_context(tensor)?;
}
let pipeline = &self.inner.pipelines["adamw"];
let bind_group = self.cached_bind_group(
"adamw",
"adamw-bind-group",
"adamw-params",
&[
(0, parameter.buffer.as_ref()),
(1, gradient.buffer.as_ref()),
(2, first.buffer.as_ref()),
(3, second.buffer.as_ref()),
],
4,
bytemuck::bytes_of(¶ms),
);
self.submit(
pipeline,
&bind_group,
to_u32(parameter.length.div_ceil(256), "AdamW workgroups")?,
1,
1,
);
Ok(())
}
fn unary(&self, entry: &'static str, input: &GpuTensor) -> Result<GpuTensor, GpuError> {
self.check_context(input)?;
self.dispatch_elementwise(
entry,
input,
None,
input.shape.clone(),
1,
input.length,
0.0,
)
}
fn unary_tensor(
&self,
input: &Tensor,
entry: &'static str,
op: Op,
) -> Result<Tensor, GpuError> {
let gpu_input = input.gpu_storage(self)?;
let output = self.unary(entry, &gpu_input)?;
Ok(Tensor::from_gpu_operation(
output,
input.requires_grad(),
op,
))
}
fn dispatch_matmul(
&self,
entry: &'static str,
left: &GpuTensor,
right: &GpuTensor,
output: &GpuTensor,
params: &MatmulParams,
workgroups: (u32, u32, u32),
) {
self.dispatch_matmul_buffers(
entry,
left.buffer.as_ref(),
right.buffer.as_ref(),
output.buffer.as_ref(),
params,
workgroups,
)
}
fn dispatch_matmul_buffers(
&self,
entry: &'static str,
left: &TrackedStorageBuffer,
right: &TrackedStorageBuffer,
output: &TrackedStorageBuffer,
params: &MatmulParams,
workgroups: (u32, u32, u32),
) {
let pipeline = &self.inner.pipelines[entry];
let bind_group = self.cached_bind_group(
entry,
"matmul-bind-group",
"matmul-params",
&[(0, left), (1, right), (2, output)],
3,
bytemuck::bytes_of(params),
);
self.submit(
pipeline,
&bind_group,
workgroups.0,
workgroups.1,
workgroups.2,
);
}
fn dispatch_attention(
&self,
entry: &'static str,
input: &GpuTensor,
output_shape: Vec<usize>,
params: AttentionParams,
workgroups: usize,
) -> Result<GpuTensor, GpuError> {
let output = self.empty_tensor(output_shape, "attention-output")?;
let pipeline = &self.inner.pipelines[entry];
let bind_group = self.cached_bind_group(
entry,
"attention-bind-group",
"attention-params",
&[(0, input.buffer.as_ref()), (1, output.buffer.as_ref())],
2,
bytemuck::bytes_of(¶ms),
);
self.submit(
pipeline,
&bind_group,
to_u32(workgroups, "attention workgroups")?,
1,
1,
);
Ok(output)
}
fn dispatch_backward(
&self,
entry: &'static str,
input_a: Option<&GpuTensor>,
input_b: Option<&GpuTensor>,
upstream: Option<&GpuTensor>,
output_shape: Vec<usize>,
params: BackwardParams,
workgroups: usize,
) -> Result<GpuTensor, GpuError> {
for tensor in [input_a, input_b, upstream].into_iter().flatten() {
self.check_context(tensor)?;
}
let output = self.empty_tensor(output_shape, "backward-output")?;
let pipeline = &self.inner.pipelines[entry];
let mut buffers = Vec::with_capacity(4);
if let Some(input) = input_a {
buffers.push((0, input.buffer.as_ref()));
}
if let Some(input) = input_b {
buffers.push((1, input.buffer.as_ref()));
}
if let Some(gradient) = upstream {
buffers.push((2, gradient.buffer.as_ref()));
}
buffers.push((3, output.buffer.as_ref()));
let bind_group = self.cached_bind_group(
entry,
"backward-bind-group",
"backward-params",
&buffers,
4,
bytemuck::bytes_of(¶ms),
);
self.submit(
pipeline,
&bind_group,
to_u32(workgroups, "backward workgroups")?,
1,
1,
);
Ok(output)
}
fn dispatch_index(
&self,
entry: &'static str,
input: Option<&GpuTensor>,
indices: &GpuIndexTensor,
upstream: Option<&GpuTensor>,
output_shape: Vec<usize>,
params: BackwardParams,
workgroups: usize,
) -> Result<GpuTensor, GpuError> {
if let Some(input) = input {
self.check_context(input)?;
}
if let Some(upstream) = upstream {
self.check_context(upstream)?;
}
self.check_index_context(indices)?;
let output = self.empty_tensor(output_shape, "index-output")?;
let pipeline = &self.inner.pipelines[entry];
let mut buffers = Vec::with_capacity(4);
if let Some(input) = input {
buffers.push((0, input.buffer.as_ref()));
}
buffers.push((1, indices.buffer.as_ref()));
if let Some(upstream) = upstream {
buffers.push((2, upstream.buffer.as_ref()));
}
buffers.push((3, output.buffer.as_ref()));
let bind_group = self.cached_bind_group(
entry,
"index-bind-group",
"index-params",
&buffers,
4,
bytemuck::bytes_of(¶ms),
);
self.submit(
pipeline,
&bind_group,
to_u32(workgroups, "index workgroups")?,
1,
1,
);
Ok(output)
}
fn dispatch_fusion(
&self,
entry: &'static str,
inputs: [Option<&GpuTensor>; 6],
outputs: [Option<&GpuTensor>; 3],
params: FusionParams,
workgroups: (u32, u32, u32),
) -> Result<(), GpuError> {
for tensor in inputs.into_iter().chain(outputs).flatten() {
self.check_context(tensor)?;
}
let pipeline = &self.inner.pipelines[entry];
let mut buffers = Vec::with_capacity(9);
for (index, tensor) in inputs.into_iter().enumerate() {
if let Some(tensor) = tensor {
buffers.push((index as u32, tensor.buffer.as_ref()));
}
}
for (index, tensor) in outputs.into_iter().enumerate() {
if let Some(tensor) = tensor {
buffers.push(((index + 6) as u32, tensor.buffer.as_ref()));
}
}
let bind_group = self.cached_bind_group(
entry,
"fusion-bind-group",
"fusion-params",
&buffers,
9,
bytemuck::bytes_of(¶ms),
);
self.submit(
pipeline,
&bind_group,
workgroups.0,
workgroups.1,
workgroups.2,
);
Ok(())
}
fn dispatch_elementwise(
&self,
entry: &'static str,
input: &GpuTensor,
second: Option<&GpuTensor>,
output_shape: Vec<usize>,
rows: usize,
columns: usize,
epsilon: f32,
) -> Result<GpuTensor, GpuError> {
let output = self.empty_tensor(output_shape, "elementwise-output")?;
let params = ElementParams {
length: to_u32(input.length, "element count")?,
rows: to_u32(rows, "row count")?,
columns: to_u32(columns, "column count")?,
epsilon,
};
let pipeline = &self.inner.pipelines[entry];
let bind_group = if let Some(second) = second {
self.cached_bind_group(
entry,
"binary-bind-group",
"elementwise-params",
&[
(0, input.buffer.as_ref()),
(1, second.buffer.as_ref()),
(2, output.buffer.as_ref()),
],
3,
bytemuck::bytes_of(¶ms),
)
} else {
self.cached_bind_group(
entry,
"unary-bind-group",
"elementwise-params",
&[(0, input.buffer.as_ref()), (2, output.buffer.as_ref())],
3,
bytemuck::bytes_of(¶ms),
)
};
let workgroups = if entry == "softmax" || entry == "rms_norm" {
to_u32(rows, "reduction workgroups")?
} else {
to_u32(input.length.div_ceil(256), "element workgroups")?
};
self.submit(pipeline, &bind_group, workgroups, 1, 1);
Ok(output)
}
fn submit(
&self,
pipeline: &wgpu::ComputePipeline,
bind_group: &wgpu::BindGroup,
x: u32,
y: u32,
z: u32,
) {
let _span = profile::span("gpu.dispatch");
let mut encoder =
self.inner
.device
.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("llm-compute-encoder"),
});
let mut capture_guard = self.inner.timestamp_capture.lock().ok();
let timestamp_indices = capture_guard
.as_deref_mut()
.and_then(Option::as_mut)
.and_then(|capture| {
if capture.next_query + 2 <= TIMESTAMP_QUERY_COUNT {
let beginning = capture.next_query;
capture.next_query += 2;
Some((beginning, beginning + 1))
} else {
capture.overflowed = true;
None
}
});
let timestamp_writes = timestamp_indices.and_then(|(beginning, end)| {
capture_guard
.as_deref()
.and_then(Option::as_ref)
.map(|capture| wgpu::ComputePassTimestampWrites {
query_set: &capture.query_set,
beginning_of_pass_write_index: Some(beginning),
end_of_pass_write_index: Some(end),
})
});
{
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
label: Some("llm-compute-pass"),
timestamp_writes,
});
pass.set_pipeline(pipeline);
pass.set_bind_group(0, bind_group, &[]);
pass.dispatch_workgroups(x, y, z);
}
self.inner.queue.submit([encoder.finish()]);
}
fn read_buffer(&self, source: &wgpu::Buffer, length: usize) -> Result<Vec<f32>, GpuError> {
let byte_size = (length * std::mem::size_of::<f32>()) as u64;
let staging = self.acquire_staging(byte_size)?;
let mut encoder =
self.inner
.device
.create_command_encoder(&wgpu::CommandEncoderDescriptor {
label: Some("readback-encoder"),
});
encoder.copy_buffer_to_buffer(source, 0, &staging, 0, byte_size);
self.inner.queue.submit([encoder.finish()]);
self.inner.memory.readbacks.fetch_add(1, Ordering::Relaxed);
self.inner
.memory
.readback_bytes
.fetch_add(length * std::mem::size_of::<f32>(), Ordering::Relaxed);
let slice = staging.slice(..);
let (sender, receiver) = mpsc::channel();
slice.map_async(wgpu::MapMode::Read, move |result| {
let _ = sender.send(result);
});
self.inner
.device
.poll(wgpu::PollType::Wait {
submission_index: None,
timeout: None,
})
.map_err(|error| GpuError::Synchronization(error.to_string()))?;
receiver
.recv()
.map_err(|error| GpuError::Synchronization(error.to_string()))?
.map_err(|error| GpuError::Map(error.to_string()))?;
let mapped = slice
.get_mapped_range()
.map_err(|error| GpuError::Map(error.to_string()))?;
let values = bytemuck::cast_slice::<u8, f32>(&mapped).to_vec();
drop(mapped);
staging.unmap();
self.recycle_staging(staging)?;
Ok(values)
}
fn empty_tensor(&self, shape: Vec<usize>, label: &str) -> Result<GpuTensor, GpuError> {
if shape.is_empty() || shape.iter().any(|dimension| *dimension == 0) {
return Err(GpuError::InvalidShape(
"GPU tensors need at least one non-zero dimension".into(),
));
}
let length = shape.iter().product();
let buffer = self.acquire_output_buffer(&shape, label);
Ok(GpuTensor {
context: self.clone(),
buffer,
shape,
length,
})
}
fn create_storage_buffer(&self, length: usize, label: &str) -> Arc<TrackedStorageBuffer> {
let byte_size = length * std::mem::size_of::<f32>();
self.create_storage_buffer_bytes(byte_size, label)
}
fn create_storage_buffer_bytes(
&self,
byte_size: usize,
label: &str,
) -> Arc<TrackedStorageBuffer> {
let buffer = self.inner.device.create_buffer(&wgpu::BufferDescriptor {
label: Some(label),
size: byte_size as u64,
usage: wgpu::BufferUsages::STORAGE
| wgpu::BufferUsages::COPY_DST
| wgpu::BufferUsages::COPY_SRC,
mapped_at_creation: false,
});
let id = self.next_buffer_id();
self.track_storage_buffer(id, buffer, byte_size, None)
}
fn acquire_output_buffer(&self, shape: &[usize], label: &str) -> Arc<TrackedStorageBuffer> {
let byte_size = shape.iter().product::<usize>() * std::mem::size_of::<f32>();
let pooled = self
.inner
.output_pool
.lock()
.ok()
.and_then(|mut pool| pool.acquire(shape));
let (id, buffer) = match pooled {
Some(pooled) => {
self.inner
.memory
.output_buffer_reuses
.fetch_add(1, Ordering::Relaxed);
pooled
}
None => {
self.inner
.memory
.output_buffer_allocations
.fetch_add(1, Ordering::Relaxed);
(
self.next_buffer_id(),
self.inner.device.create_buffer(&wgpu::BufferDescriptor {
label: Some(label),
size: byte_size as u64,
usage: wgpu::BufferUsages::STORAGE
| wgpu::BufferUsages::COPY_DST
| wgpu::BufferUsages::COPY_SRC,
mapped_at_creation: false,
}),
)
}
};
self.track_storage_buffer(id, buffer, byte_size, Some(shape.to_vec()))
}
fn next_buffer_id(&self) -> usize {
self.inner
.memory
.next_buffer_id
.fetch_add(1, Ordering::Relaxed)
.wrapping_add(1)
}
fn track_storage_buffer(
&self,
id: usize,
buffer: wgpu::Buffer,
byte_size: usize,
recycle_shape: Option<Vec<usize>>,
) -> Arc<TrackedStorageBuffer> {
self.inner
.memory
.live_tensor_buffers
.fetch_add(1, Ordering::Relaxed);
let live_bytes = self
.inner
.memory
.live_tensor_bytes
.fetch_add(byte_size, Ordering::Relaxed)
+ byte_size;
self.inner
.memory
.peak_tensor_bytes
.fetch_max(live_bytes, Ordering::Relaxed);
Arc::new(TrackedStorageBuffer {
id,
buffer,
byte_size,
memory: self.inner.memory.clone(),
recycle_shape,
output_pool: self.inner.output_pool.clone(),
})
}
fn uniform_buffer(&self, bytes: &[u8], label: &str) -> wgpu::Buffer {
use wgpu::util::DeviceExt;
self.inner
.device
.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some(label),
contents: bytes,
usage: wgpu::BufferUsages::UNIFORM,
})
}
fn cached_bind_group(
&self,
entry: &'static str,
label: &str,
uniform_label: &str,
buffers: &[(u32, &TrackedStorageBuffer)],
uniform_binding: u32,
uniform_bytes: &[u8],
) -> wgpu::BindGroup {
let key = BindGroupCacheKey {
entry,
uniform_binding,
uniform_bytes: uniform_bytes.to_vec(),
buffers: buffers
.iter()
.map(|(binding, buffer)| BindGroupBufferKey {
binding: *binding,
buffer_id: buffer.id,
})
.collect(),
};
let mut cache = self.inner.bind_group_cache.lock().ok();
if let Some(bind_group) = cache.as_mut().and_then(|cache| cache.get(&key)) {
self.inner
.memory
.bind_group_cache_hits
.fetch_add(1, Ordering::Relaxed);
return bind_group;
}
let uniform = self.uniform_buffer(uniform_bytes, uniform_label);
let pipeline = &self.inner.pipelines[entry];
let mut entries = buffers
.iter()
.map(|(index, buffer)| binding(*index, &buffer.buffer))
.collect::<Vec<_>>();
entries.push(binding(uniform_binding, &uniform));
let bind_group = self
.inner
.device
.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some(label),
layout: &pipeline.get_bind_group_layout(0),
entries: &entries,
});
self.inner
.memory
.bind_group_creations
.fetch_add(1, Ordering::Relaxed);
if let Some(cache) = cache.as_mut() {
cache.insert(key, bind_group.clone(), uniform);
}
bind_group
}
fn acquire_staging(&self, size: u64) -> Result<wgpu::Buffer, GpuError> {
let mut pool = self
.inner
.staging_pool
.lock()
.map_err(|_| GpuError::Synchronization("staging pool lock poisoned".into()))?;
if let Some(index) = pool.iter().position(|buffer| buffer.size() == size) {
return Ok(pool.swap_remove(index));
}
Ok(self.inner.device.create_buffer(&wgpu::BufferDescriptor {
label: Some("readback-staging-buffer"),
size,
usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
}))
}
fn recycle_staging(&self, buffer: wgpu::Buffer) -> Result<(), GpuError> {
self.inner
.staging_pool
.lock()
.map_err(|_| GpuError::Synchronization("staging pool lock poisoned".into()))?
.push(buffer);
Ok(())
}
fn check_context(&self, tensor: &GpuTensor) -> Result<(), GpuError> {
if !Arc::ptr_eq(&self.inner, &tensor.context.inner) {
return Err(GpuError::InvalidShape(
"GPU tensors belong to different contexts".into(),
));
}
Ok(())
}
fn check_matmul_context(&self, tensor: &GpuMatmulTensor) -> Result<(), GpuError> {
if !Arc::ptr_eq(&self.inner, &tensor.context.inner) {
return Err(GpuError::InvalidShape(
"GPU matmul tensor belongs to another context".into(),
));
}
Ok(())
}
fn check_index_context(&self, tensor: &GpuIndexTensor) -> Result<(), GpuError> {
if !Arc::ptr_eq(&self.inner, &tensor.context.inner) {
return Err(GpuError::InvalidShape(
"GPU index tensor belongs to another context".into(),
));
}
Ok(())
}
}
impl GpuTensor {
pub fn shape(&self) -> &[usize] {
&self.shape
}
pub fn len(&self) -> usize {
self.length
}
pub fn is_empty(&self) -> bool {
self.length == 0
}
pub fn to_vec(&self) -> Result<Vec<f32>, GpuError> {
self.context.read_buffer(&self.buffer.buffer, self.length)
}
pub(crate) fn reshape(&self, shape: Vec<usize>) -> Self {
assert_eq!(shape.iter().product::<usize>(), self.length);
Self {
context: self.context.clone(),
buffer: self.buffer.clone(),
shape,
length: self.length,
}
}
pub(crate) fn belongs_to(&self, context: &GpuContext) -> bool {
Arc::ptr_eq(&self.context.inner, &context.inner)
}
pub fn to_tensor(&self, requires_grad: bool) -> Result<Tensor, GpuError> {
Ok(Tensor::from_gpu_operation(
self.clone(),
requires_grad,
Op::Leaf,
))
}
}
impl GpuMatmulTensor {
pub fn shape(&self) -> &[usize] {
&self.shape
}
pub fn len(&self) -> usize {
self.length
}
pub fn is_empty(&self) -> bool {
self.length == 0
}
pub fn format(&self) -> GpuFloatFormat {
self.format
}
}
impl GpuQuantizedLinearWeight {
pub(crate) fn belongs_to(&self, context: &GpuContext) -> bool {
Arc::ptr_eq(&self.context.inner, &context.inner)
}
}
impl GpuIndexTensor {
pub fn shape(&self) -> &[usize] {
&self.shape
}
pub fn len(&self) -> usize {
self.length
}
pub fn is_empty(&self) -> bool {
self.length == 0
}
pub(crate) fn belongs_to(&self, context: &GpuContext) -> bool {
Arc::ptr_eq(&self.context.inner, &context.inner)
}
}
impl IndexTensor {
pub fn to_gpu(&self, context: &GpuContext) -> Result<GpuIndexTensor, GpuError> {
if let Some(storage) = self.cached_gpu() {
if storage.belongs_to(context) {
return Ok(storage);
}
}
let storage = context.upload_indices(&self.data(), self.shape())?;
self.set_cached_gpu(storage.clone());
Ok(storage)
}
pub fn is_gpu_resident(&self, context: &GpuContext) -> bool {
self.cached_gpu()
.is_some_and(|storage| storage.belongs_to(context))
}
}
impl Tensor {
pub(crate) fn gpu_storage(&self, context: &GpuContext) -> Result<GpuTensor, GpuError> {
if let Some(storage) = self.cached_gpu() {
if storage.belongs_to(context) {
return Ok(storage);
}
}
if self.device() == Device::Gpu {
return Err(GpuError::InvalidShape(
"GPU tensor belongs to another context; call to_cpu() before moving it".into(),
));
}
Err(GpuError::InvalidShape(
"tensor is not GPU resident; call to_gpu() before GPU operations".into(),
))
}
pub fn to_gpu(&self, context: &GpuContext) -> Result<Tensor, GpuError> {
if self.device() == Device::Gpu {
self.gpu_storage(context)?;
return Ok(self.clone());
}
let storage = match self.gpu_storage(context) {
Ok(storage) => storage,
Err(_) => {
let storage = context.upload(&self.shared_data(), self.shape())?;
self.set_cached_gpu(storage.clone());
storage
}
};
Ok(self.copy_to_gpu_storage(storage))
}
pub fn is_gpu_resident(&self, context: &GpuContext) -> bool {
self.cached_gpu()
.is_some_and(|storage| storage.belongs_to(context))
}
pub fn to_cpu(&self) -> Result<Tensor, GpuError> {
if self.device() == Device::Cpu {
return Ok(self.clone());
}
let data = if let Some(data) = self.cached_cpu_data() {
data
} else if let Some(storage) = self.cached_gpu() {
storage.to_vec()?
} else {
return Err(GpuError::InvalidShape(
"tensor has no GPU storage to transfer".into(),
));
};
Ok(self.copy_to_cpu_storage(data))
}
pub fn gpu_grad_to_cpu(&self) -> Result<Option<Vec<f32>>, GpuError> {
self.gpu_grad()
.map(|gradient| gradient.to_vec())
.transpose()
}
pub fn backward_gpu(&self, context: &GpuContext) -> Result<(), GpuError> {
if self.numel() != 1 || !self.requires_grad() || self.device() != Device::Gpu {
return Err(GpuError::InvalidShape(
"backward_gpu() requires a gradient-tracked one-element GPU tensor".into(),
));
}
self.backward_gpu_with_gradient(context, context.upload(&[1.0], vec![1])?)
}
pub(crate) fn backward_gpu_with_gradient(
&self,
context: &GpuContext,
root_gradient: GpuTensor,
) -> Result<(), GpuError> {
let topology = self.topological_order();
for tensor in &topology {
if !tensor.is_leaf() {
tensor.set_gpu_grad(None);
}
}
self.set_gpu_grad(Some(root_gradient));
for tensor in topology.iter().rev() {
if let Some(output_gradient) = tensor.gpu_grad() {
let operation = tensor.inner.borrow().op.clone();
match operation {
Op::Leaf => {}
Op::Checkpoint { input, recompute } => {
Self::checkpoint_gpu_backward(
&input,
&recompute,
&output_gradient,
context,
)?;
}
Op::Transfer(input)
| Op::Reshape(input)
| Op::Contiguous(input)
| Op::FakeQuant(input) => {
accumulate_gpu_gradient(
&input,
output_gradient.reshape(input.shape()),
context,
)?;
}
Op::Add(left, right) => {
accumulate_gpu_gradient(&left, output_gradient.clone(), context)?;
accumulate_gpu_gradient(&right, output_gradient.clone(), context)?;
}
Op::Sub(left, right) => {
accumulate_gpu_gradient(&left, output_gradient.clone(), context)?;
let negative = context.scale(&output_gradient, -1.0)?;
accumulate_gpu_gradient(&right, negative, context)?;
}
Op::Mul(left, right) => {
if right.numel() == 1 && !right.requires_grad() {
let factor = right.item();
let gradient = context.scale(&output_gradient, factor)?;
accumulate_gpu_gradient(&left, gradient, context)?;
} else {
let (left_gradient, right_gradient) = context.backward_multiply(
&left.gpu_storage(context)?,
&right.gpu_storage(context)?,
&output_gradient,
)?;
accumulate_gpu_gradient(&left, left_gradient, context)?;
accumulate_gpu_gradient(&right, right_gradient, context)?;
}
}
Op::MatMul(left, right) => {
let (left_gradient, right_gradient) = context.backward_matmul(
&left.gpu_storage(context)?,
&right.gpu_storage(context)?,
&output_gradient,
)?;
accumulate_gpu_gradient(&left, left_gradient, context)?;
accumulate_gpu_gradient(&right, right_gradient, context)?;
}
Op::Permute { input, axes } => {
let mut inverse = vec![0; axes.len()];
for (output_axis, input_axis) in axes.iter().enumerate() {
inverse[*input_axis] = output_axis;
}
let gradient = context.permute(&output_gradient, &inverse)?;
accumulate_gpu_gradient(&input, gradient, context)?;
}
Op::Embedding { weight, indices } => {
let indices_gpu = indices.to_gpu(context)?;
let gradient = context.backward_embedding(
&indices_gpu,
&output_gradient,
weight.shape(),
)?;
accumulate_gpu_gradient(&weight, gradient, context)?;
}
Op::Rope {
input,
sequence_dim,
theta,
position_offset,
scaling,
} => {
let gradient = context.backward_rope(
&output_gradient,
sequence_dim,
theta,
position_offset,
scaling.as_ref(),
)?;
accumulate_gpu_gradient(&input, gradient, context)?;
}
Op::SoftmaxLast(input) => {
let gradient = context
.backward_softmax(&tensor.gpu_storage(context)?, &output_gradient)?;
accumulate_gpu_gradient(&input, gradient, context)?;
}
Op::CausalMask { input, past_length } => {
let gradient =
context.causal_mask_backward(&output_gradient, past_length)?;
accumulate_gpu_gradient(&input, gradient, context)?;
}
Op::Gelu(input) => {
let gradient = context.backward_unary(
"gelu_backward",
&input.gpu_storage(context)?,
&output_gradient,
)?;
accumulate_gpu_gradient(&input, gradient, context)?;
}
Op::Silu(input) => {
let gradient = context.backward_unary(
"silu_backward",
&input.gpu_storage(context)?,
&output_gradient,
)?;
accumulate_gpu_gradient(&input, gradient, context)?;
}
Op::RmsNormGpu {
input,
weight,
epsilon,
} => {
let (input_gradient, weight_gradient) = context.backward_rms_norm(
&input.gpu_storage(context)?,
&weight.gpu_storage(context)?,
&output_gradient,
epsilon,
)?;
accumulate_gpu_gradient(&input, input_gradient, context)?;
accumulate_gpu_gradient(&weight, weight_gradient, context)?;
}
Op::ResidualRmsNormGpu {
left,
right,
weight,
epsilon,
} => {
let residual = context
.add(&left.gpu_storage(context)?, &right.gpu_storage(context)?)?;
let (residual_gradient, weight_gradient) = context.backward_rms_norm(
&residual,
&weight.gpu_storage(context)?,
&output_gradient,
epsilon,
)?;
accumulate_gpu_gradient(&left, residual_gradient.clone(), context)?;
accumulate_gpu_gradient(&right, residual_gradient, context)?;
accumulate_gpu_gradient(&weight, weight_gradient, context)?;
}
Op::SwiGluGpu { gate, up } => {
let gate_upstream =
context.mul(&output_gradient, &up.gpu_storage(context)?)?;
let gate_gradient = context.backward_unary(
"silu_backward",
&gate.gpu_storage(context)?,
&gate_upstream,
)?;
let activated_gate = context.silu(&gate.gpu_storage(context)?)?;
let up_gradient = context.mul(&output_gradient, &activated_gate)?;
accumulate_gpu_gradient(&gate, gate_gradient, context)?;
accumulate_gpu_gradient(&up, up_gradient, context)?;
}
Op::MaskedSoftmaxGpu { input, past_length } => {
let gradient = context
.backward_softmax(&tensor.gpu_storage(context)?, &output_gradient)?;
let gradient = context.causal_mask_backward(&gradient, past_length)?;
accumulate_gpu_gradient(&input, gradient, context)?;
}
Op::TiledAttention {
query,
key,
value,
attention_mask,
logsumexp,
scale,
past_length,
dropout_probability,
dropout_seed,
} => {
let attention_mask_gpu = attention_mask
.as_ref()
.map(|mask| mask.to_gpu(context))
.transpose()?;
let (query_gradient, key_gradient, value_gradient) = context
.backward_tiled_attention(
&query.gpu_storage(context)?,
&key.gpu_storage(context)?,
&value.gpu_storage(context)?,
&tensor.gpu_storage(context)?,
&logsumexp.gpu_storage(context)?,
&output_gradient,
attention_mask_gpu.as_ref(),
scale,
past_length,
dropout_probability,
dropout_seed,
)?;
accumulate_gpu_gradient(&query, query_gradient, context)?;
accumulate_gpu_gradient(&key, key_gradient, context)?;
accumulate_gpu_gradient(&value, value_gradient, context)?;
}
Op::CrossEntropy {
logits,
targets,
ignore_index,
} => {
let valid_rows = targets
.data()
.iter()
.filter(|target| Some(**target) != ignore_index)
.count();
let targets_gpu = targets.to_gpu(context)?;
let gradient = context.backward_cross_entropy(
&logits.gpu_storage(context)?,
&targets_gpu,
&output_gradient,
valid_rows,
ignore_index,
)?;
accumulate_gpu_gradient(&logits, gradient, context)?;
}
Op::Pow(_, _)
| Op::Div(_, _)
| Op::Relu(_)
| Op::Sigmoid(_)
| Op::Sum(_)
| Op::SumDim { .. }
| Op::Mean(_)
| Op::MeanDim { .. }
| Op::Narrow { .. }
| Op::Cat { .. } => {
return Err(GpuError::InvalidShape(
"operation has no GPU backward kernel".into(),
));
}
}
}
tensor.release_finished_autograd_storage();
}
Ok(())
}
fn checkpoint_gpu_backward(
input: &Tensor,
recompute: &CheckpointRecompute,
output_gradient: &GpuTensor,
context: &GpuContext,
) -> Result<(), GpuError> {
let CheckpointRecompute::Gpu(recompute) = recompute else {
return Err(GpuError::InvalidShape(
"checkpoint GPU backward requires a GPU recompute closure".into(),
));
};
let detached = crate::detach_for_checkpoint(input);
let recomputed = {
let _guard = enable_grad();
recompute(&detached, context)?
};
recomputed.backward_gpu_with_gradient(context, output_gradient.clone())?;
if let Some(gradient) = detached.gpu_grad() {
accumulate_gpu_gradient(input, gradient, context)?;
}
Ok(())
}
pub fn gpu_matmul(&self, other: &Tensor, context: &GpuContext) -> Result<Tensor, GpuError> {
context.matmul_tensor(self, other)
}
pub fn gpu_batched_gemm(
&self,
other: &Tensor,
context: &GpuContext,
) -> Result<Tensor, GpuError> {
context.batched_gemm_tensor(self, other)
}
pub fn gpu_tiled_causal_attention(
&self,
key: &Tensor,
value: &Tensor,
scale: f32,
past_length: usize,
context: &GpuContext,
) -> Result<Tensor, GpuError> {
context.tiled_causal_attention_tensor(self, key, value, scale, past_length)
}
pub fn gpu_tiled_causal_attention_with_mask(
&self,
key: &Tensor,
value: &Tensor,
scale: f32,
past_length: usize,
attention_mask: &IndexTensor,
context: &GpuContext,
) -> Result<Tensor, GpuError> {
context.tiled_causal_attention_tensor_with_mask(
self,
key,
value,
scale,
past_length,
attention_mask,
)
}
pub fn gpu_add(&self, other: &Tensor, context: &GpuContext) -> Result<Tensor, GpuError> {
context.add_tensor(self, other)
}
pub fn gpu_mul(&self, other: &Tensor, context: &GpuContext) -> Result<Tensor, GpuError> {
context.mul_tensor(self, other)
}
pub fn gpu_scale(&self, factor: f32, context: &GpuContext) -> Result<Tensor, GpuError> {
context.scale_tensor(self, factor)
}
pub fn gpu_dropout(
&self,
probability: f32,
seed: u64,
context: &GpuContext,
) -> Result<Tensor, GpuError> {
context.dropout_tensor(self, probability, seed)
}
pub fn gpu_rope_with_offset(
&self,
sequence_dim: usize,
theta: f32,
position_offset: usize,
context: &GpuContext,
) -> Result<Tensor, GpuError> {
context.rope_tensor(self, sequence_dim, theta, position_offset)
}
pub fn gpu_rope_with_offset_and_scaling(
&self,
sequence_dim: usize,
theta: f32,
position_offset: usize,
scaling: Option<&crate::rope::RopeScaling>,
context: &GpuContext,
) -> Result<Tensor, GpuError> {
context.rope_tensor_with_scaling(self, sequence_dim, theta, position_offset, scaling)
}
pub fn gpu_permute(&self, axes: Vec<usize>, context: &GpuContext) -> Result<Tensor, GpuError> {
context.permute_tensor(self, axes)
}
pub fn gpu_causal_mask(
&self,
past_length: usize,
context: &GpuContext,
) -> Result<Tensor, GpuError> {
context.causal_mask_tensor(self, past_length)
}
pub fn gpu_relu(&self, context: &GpuContext) -> Result<Tensor, GpuError> {
context.relu_tensor(self)
}
pub fn gpu_gelu(&self, context: &GpuContext) -> Result<Tensor, GpuError> {
context.gelu_tensor(self)
}
pub fn gpu_silu(&self, context: &GpuContext) -> Result<Tensor, GpuError> {
context.silu_tensor(self)
}
pub fn gpu_softmax(&self, context: &GpuContext) -> Result<Tensor, GpuError> {
context.softmax_tensor(self)
}
pub fn gpu_rms_norm(
&self,
weight: &Tensor,
epsilon: f32,
context: &GpuContext,
) -> Result<Tensor, GpuError> {
context.rms_norm_tensor(self, weight, epsilon)
}
pub fn gpu_residual_rms_norm(
&self,
residual: &Tensor,
weight: &Tensor,
epsilon: f32,
context: &GpuContext,
) -> Result<(Tensor, Tensor), GpuError> {
context.residual_rms_norm_tensor(self, residual, weight, epsilon)
}
pub fn gpu_swiglu(&self, up: &Tensor, context: &GpuContext) -> Result<Tensor, GpuError> {
context.swiglu_tensor(self, up)
}
pub fn gpu_masked_softmax(
&self,
past_length: usize,
context: &GpuContext,
) -> Result<Tensor, GpuError> {
context.masked_softmax_tensor(self, past_length)
}
}
fn create_pipeline(
device: &wgpu::Device,
module: &wgpu::ShaderModule,
entry_point: &'static str,
) -> wgpu::ComputePipeline {
device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
label: Some(entry_point),
layout: None,
module,
entry_point: Some(entry_point),
compilation_options: wgpu::PipelineCompilationOptions::default(),
cache: None,
})
}
fn binding(binding: u32, buffer: &wgpu::Buffer) -> wgpu::BindGroupEntry<'_> {
wgpu::BindGroupEntry {
binding,
resource: buffer.as_entire_binding(),
}
}
fn validate_data_shape(data: &[f32], shape: &[usize]) -> Result<(), GpuError> {
if shape.is_empty() || shape.iter().any(|dimension| *dimension == 0) {
return Err(GpuError::InvalidShape(
"GPU tensors need at least one non-zero dimension".into(),
));
}
if shape.iter().product::<usize>() != data.len() {
return Err(GpuError::InvalidShape(
"data length does not match GPU tensor shape".into(),
));
}
Ok(())
}
fn pack_bf16(data: &[f32]) -> Vec<u8> {
let mut packed = Vec::with_capacity(data.len().div_ceil(2));
for values in data.chunks(2) {
let low = u32::from(bf16::from_f32(values[0]).to_bits());
let high = values
.get(1)
.map(|value| u32::from(bf16::from_f32(*value).to_bits()))
.unwrap_or(0);
packed.push(low | (high << 16));
}
bytemuck::cast_slice(&packed).to_vec()
}
fn to_u32(value: usize, label: &str) -> Result<u32, GpuError> {
u32::try_from(value)
.map_err(|_| GpuError::InvalidShape(format!("{label} exceeds WebGPU u32 limits")))
}
fn simple_backward_params(length: usize) -> Result<BackwardParams, GpuError> {
Ok(BackwardParams {
length: to_u32(length, "gradient length")?,
rows: 0,
columns: 0,
inner: 0,
batch: 0,
padding: 0,
epsilon: 0.0,
scalar: 0.0,
})
}
fn fusion_params(length: usize) -> Result<FusionParams, GpuError> {
Ok(FusionParams {
length: to_u32(length, "fusion length")?,
rows: 0,
input_width: 0,
output_width: 0,
query_length: 0,
key_length: 0,
past_length: 0,
padding: 0,
epsilon: 0.0,
scalar_0: 0.0,
scalar_1: 0.0,
scalar_2: 0.0,
})
}
struct GpuMatmulDimensions {
rows: usize,
shared: usize,
columns: usize,
batch: usize,
output_shape: Vec<usize>,
}
impl GpuMatmulConfig {
fn supports(&self, dimensions: &GpuMatmulDimensions) -> bool {
match self.kernel {
GpuMatmulKernel::Tile16 | GpuMatmulKernel::RegisterBlocked32 => true,
GpuMatmulKernel::ExactTile16 => {
dimensions.rows % 16 == 0
&& dimensions.shared % 16 == 0
&& dimensions.columns % 16 == 0
}
GpuMatmulKernel::Vectorized8x32 => {
dimensions.shared % 4 == 0 && dimensions.columns % 4 == 0
}
}
}
}
fn gpu_matmul_dimensions(
left: &GpuTensor,
right: &GpuTensor,
) -> Result<GpuMatmulDimensions, GpuError> {
gpu_matmul_dimensions_from_shapes(&left.shape, &right.shape)
}
fn gpu_matmul_dimensions_from_shapes(
left_shape: &[usize],
right_shape: &[usize],
) -> Result<GpuMatmulDimensions, GpuError> {
if left_shape.len() < 2 || left_shape.len() != right_shape.len() {
return Err(GpuError::InvalidShape(
"GPU matmul requires equal-rank tensors with at least two dimensions".into(),
));
}
let rank = left_shape.len();
if left_shape[..rank - 2] != right_shape[..rank - 2] {
return Err(GpuError::InvalidShape(
"matmul batch dimensions differ".into(),
));
}
let rows = left_shape[rank - 2];
let shared = left_shape[rank - 1];
let columns = right_shape[rank - 1];
if shared != right_shape[rank - 2] {
return Err(GpuError::InvalidShape(
"matmul inner dimensions differ".into(),
));
}
let batch = left_shape[..rank - 2].iter().product::<usize>();
let mut output_shape = left_shape[..rank - 2].to_vec();
output_shape.extend([rows, columns]);
Ok(GpuMatmulDimensions {
rows,
shared,
columns,
batch,
output_shape,
})
}
fn accumulate_gpu_gradient(
tensor: &Tensor,
contribution: GpuTensor,
context: &GpuContext,
) -> Result<(), GpuError> {
if !tensor.requires_grad() {
return Ok(());
}
let contribution = contribution.reshape(tensor.shape());
let gradient = if let Some(existing) = tensor.gpu_grad() {
context.add(&existing, &contribution)?
} else {
contribution
};
tensor.set_gpu_grad(Some(gradient));
Ok(())
}
#[cfg(test)]
mod tests {
use super::{
GpuCacheLimits, DEFAULT_BIND_GROUP_CACHE_ENTRIES, DEFAULT_OUTPUT_POOL_BYTES,
LOW_BIND_GROUP_CACHE_ENTRIES, LOW_MEMORY_BUDGET_THRESHOLD_BYTES, LOW_OUTPUT_POOL_BYTES,
};
#[test]
fn cache_limits_default_when_budget_unset_or_high() {
assert_eq!(
GpuCacheLimits::for_memory_budget(None),
GpuCacheLimits::default()
);
assert_eq!(
GpuCacheLimits::for_memory_budget(Some(LOW_MEMORY_BUDGET_THRESHOLD_BYTES + 1)),
GpuCacheLimits {
output_buffer_bytes: DEFAULT_OUTPUT_POOL_BYTES,
bind_group_entries: DEFAULT_BIND_GROUP_CACHE_ENTRIES,
}
);
assert!(!GpuCacheLimits::default().is_low_memory());
}
#[test]
fn cache_limits_shrink_when_budget_is_low() {
let low = GpuCacheLimits::for_memory_budget(Some(6 * 1024 * 1024 * 1024));
assert_eq!(
low,
GpuCacheLimits {
output_buffer_bytes: LOW_OUTPUT_POOL_BYTES,
bind_group_entries: LOW_BIND_GROUP_CACHE_ENTRIES,
}
);
assert!(low.is_low_memory());
assert_eq!(
GpuCacheLimits::for_memory_budget(Some(LOW_MEMORY_BUDGET_THRESHOLD_BYTES)),
low
);
}
}