use std::{collections::BTreeMap, fmt, mem::size_of_val, num::NonZeroU64};
use burn::tensor::{DType, Shape, Tensor};
use burn_wgpu::{
CubeTensor, RuntimeOptions, Wgpu, WgpuDevice, WgpuRuntime, WgpuSetup, init_device,
};
use bytemuck::Pod;
use cubecl::{Runtime, client::ComputeClient, server::Handle};
pub type ResidentTensor = Tensor<3>;
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct ReadEpoch(u64);
impl ReadEpoch {
pub const fn new(value: u64) -> Self {
Self(value)
}
pub const fn get(self) -> u64 {
self.0
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ChunkBounds {
pub origin: [i64; 3],
pub extent: [u32; 3],
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct DirtyRegion {
pub origin: [u32; 3],
pub extent: [u32; 3],
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum PlaneClass {
Exact,
FixedPoint,
Derived,
Temporary,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum PlaneElementType {
U8,
U16,
U32,
I8,
I16,
I32,
F32,
}
impl PlaneElementType {
pub const fn byte_width(self) -> usize {
match self {
Self::U8 | Self::I8 => 1,
Self::U16 | Self::I16 => 2,
Self::U32 | Self::I32 | Self::F32 => 4,
}
}
}
mod sealed {
pub trait Sealed {}
impl Sealed for u8 {}
impl Sealed for u16 {}
impl Sealed for u32 {}
impl Sealed for i8 {}
impl Sealed for i16 {}
impl Sealed for i32 {}
impl Sealed for f32 {}
}
pub trait ResidentElement: Pod + sealed::Sealed {
const ELEMENT_TYPE: PlaneElementType;
}
macro_rules! resident_elements {
($($ty:ty => $element:ident),+ $(,)?) => {
$(impl ResidentElement for $ty {
const ELEMENT_TYPE: PlaneElementType = PlaneElementType::$element;
})+
};
}
resident_elements! {
u8 => U8,
u16 => U16,
u32 => U32,
i8 => I8,
i16 => I16,
i32 => I32,
f32 => F32,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct PlaneId(String);
impl PlaneId {
pub fn new(name: impl Into<String>) -> Result<Self, ResidentChunkError> {
let name = name.into();
if name.trim().is_empty() {
return Err(ResidentChunkError::EmptyPlaneId);
}
Ok(Self(name))
}
pub fn as_str(&self) -> &str {
&self.0
}
}
impl fmt::Display for PlaneId {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(&self.0)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct PlaneLayout {
pub shape: [usize; 3],
pub strides: [usize; 3],
pub element_type: PlaneElementType,
}
impl PlaneLayout {
fn contiguous<T: ResidentElement>(shape: [usize; 3]) -> Result<Self, ResidentChunkError> {
if shape.contains(&0) {
return Err(ResidentChunkError::EmptyShape { shape });
}
let yz = shape[1]
.checked_mul(shape[2])
.ok_or(ResidentChunkError::ShapeOverflow { shape })?;
shape[0]
.checked_mul(yz)
.and_then(|elements| elements.checked_mul(T::ELEMENT_TYPE.byte_width()))
.ok_or(ResidentChunkError::ShapeOverflow { shape })?;
Ok(Self {
shape,
strides: [yz, shape[2], 1],
element_type: T::ELEMENT_TYPE,
})
}
pub fn element_count(self) -> usize {
self.shape[0] * self.shape[1] * self.shape[2]
}
pub fn byte_len(self) -> usize {
self.element_count() * self.element_type.byte_width()
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ChunkStamp {
pub revision: u64,
pub valid_read_epoch: ReadEpoch,
}
#[derive(Clone, Debug)]
pub struct BufferIdentity {
buffer: wgpu::Buffer,
offset: u64,
size: u64,
}
impl BufferIdentity {
pub fn buffer(&self) -> &wgpu::Buffer {
&self.buffer
}
pub const fn offset(&self) -> u64 {
self.offset
}
pub const fn size(&self) -> u64 {
self.size
}
pub fn same_buffer(&self, other: &Self) -> bool {
self.buffer == other.buffer
}
}
impl PartialEq for BufferIdentity {
fn eq(&self, other: &Self) -> bool {
self.buffer == other.buffer && self.offset == other.offset && self.size == other.size
}
}
impl Eq for BufferIdentity {}
#[derive(Debug)]
pub struct RawKernelView {
allocation: BufferIdentity,
layout: PlaneLayout,
class: PlaneClass,
stamp: ChunkStamp,
_lease: Handle,
}
impl RawKernelView {
pub fn allocation(&self) -> &BufferIdentity {
&self.allocation
}
pub const fn layout(&self) -> PlaneLayout {
self.layout
}
pub const fn class(&self) -> PlaneClass {
self.class
}
pub const fn stamp(&self) -> ChunkStamp {
self.stamp
}
pub fn lease(&self) -> SpatialLease<'_> {
SpatialLease {
buffer: self.allocation.buffer(),
offset: self.allocation.offset(),
size: self.allocation.size(),
shape: self.layout.shape,
element_type: self.layout.element_type,
stamp: self.stamp,
}
}
pub fn binding(&self) -> wgpu::BufferBinding<'_> {
wgpu::BufferBinding {
buffer: self.allocation.buffer(),
offset: self.allocation.offset(),
size: NonZeroU64::new(self.allocation.size().next_multiple_of(4)),
}
}
}
#[derive(Clone, Copy, Debug)]
pub struct SpatialLease<'a> {
pub buffer: &'a wgpu::Buffer,
pub offset: u64,
pub size: u64,
pub shape: [usize; 3],
pub element_type: PlaneElementType,
pub stamp: ChunkStamp,
}
impl SpatialLease<'_> {
pub fn byte_len(&self) -> u64 {
(self.shape[0] * self.shape[1] * self.shape[2] * self.element_type.byte_width()) as u64
}
pub fn fits(&self) -> bool {
self.byte_len() <= self.size
}
}
#[derive(Clone, Debug)]
pub struct BurnTensorView {
tensor: ResidentTensor,
allocation: BufferIdentity,
layout: PlaneLayout,
class: PlaneClass,
stamp: ChunkStamp,
}
impl BurnTensorView {
pub fn tensor(&self) -> &ResidentTensor {
&self.tensor
}
pub fn into_tensor(self) -> ResidentTensor {
self.tensor
}
pub fn allocation(&self) -> &BufferIdentity {
&self.allocation
}
pub const fn layout(&self) -> PlaneLayout {
self.layout
}
pub const fn class(&self) -> PlaneClass {
self.class
}
pub const fn stamp(&self) -> ChunkStamp {
self.stamp
}
}
#[derive(Clone)]
pub struct ResidentClient {
compute: ComputeClient<WgpuRuntime>,
device: WgpuDevice,
}
impl fmt::Debug for ResidentClient {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ResidentClient")
.field("device", &self.device)
.finish_non_exhaustive()
}
}
impl ResidentClient {
pub fn init(setup: WgpuSetup) -> Self {
let device = init_device(setup, RuntimeOptions::default());
Self::from_registered_device(device)
}
pub fn from_registered_device(device: WgpuDevice) -> Self {
let compute = WgpuRuntime::client(&device);
Self { compute, device }
}
pub fn compute_client(&self) -> &ComputeClient<WgpuRuntime> {
&self.compute
}
pub fn device(&self) -> &WgpuDevice {
&self.device
}
}
#[derive(Debug)]
struct ResidentPlane {
handle: Handle,
layout: PlaneLayout,
class: PlaneClass,
}
#[derive(Debug)]
pub struct ResidentChunk<I> {
client: ResidentClient,
identity: I,
bounds: ChunkBounds,
stamp: ChunkStamp,
dirty_regions: Vec<DirtyRegion>,
planes: BTreeMap<PlaneId, ResidentPlane>,
}
impl<I> ResidentChunk<I> {
pub fn new(
client: ResidentClient,
identity: I,
bounds: ChunkBounds,
revision: u64,
valid_read_epoch: ReadEpoch,
dirty_regions: Vec<DirtyRegion>,
) -> Self {
Self {
client,
identity,
bounds,
stamp: ChunkStamp {
revision,
valid_read_epoch,
},
dirty_regions,
planes: BTreeMap::new(),
}
}
pub fn identity(&self) -> &I {
&self.identity
}
pub const fn bounds(&self) -> ChunkBounds {
self.bounds
}
pub const fn stamp(&self) -> ChunkStamp {
self.stamp
}
pub fn dirty_regions(&self) -> &[DirtyRegion] {
&self.dirty_regions
}
pub fn plane_ids(&self) -> impl ExactSizeIterator<Item = &PlaneId> {
self.planes.keys()
}
pub fn insert_plane<T: ResidentElement>(
&mut self,
id: PlaneId,
class: PlaneClass,
shape: [usize; 3],
values: &[T],
) -> Result<(), ResidentChunkError> {
if self.planes.contains_key(&id) {
return Err(ResidentChunkError::DuplicatePlane(id));
}
let layout = PlaneLayout::contiguous::<T>(shape)?;
let expected = layout.element_count();
if values.len() != expected {
return Err(ResidentChunkError::ElementCount {
plane: id,
expected,
actual: values.len(),
});
}
let handle = self
.compute()
.create_from_slice(bytemuck::cast_slice(values));
self.planes.insert(
id,
ResidentPlane {
handle,
layout,
class,
},
);
Ok(())
}
pub fn commit_plane_patch<T: ResidentElement>(
&mut self,
queue: &wgpu::Queue,
id: &PlaneId,
expected_stamp: ChunkStamp,
element_offset: usize,
values: &[T],
committed_stamp: ChunkStamp,
dirty_regions: Vec<DirtyRegion>,
) -> Result<(), ResidentChunkError> {
if self.stamp != expected_stamp {
return Err(ResidentChunkError::StaleStamp {
expected: expected_stamp,
actual: self.stamp,
});
}
if committed_stamp.revision <= expected_stamp.revision
|| committed_stamp.valid_read_epoch <= expected_stamp.valid_read_epoch
{
return Err(ResidentChunkError::NonAdvancingStamp {
previous: expected_stamp,
committed: committed_stamp,
});
}
for region in &dirty_regions {
let fits = !region.extent.contains(&0)
&& (0..3).all(|axis| {
region.origin[axis]
.checked_add(region.extent[axis])
.is_some_and(|end| end <= self.bounds.extent[axis])
});
if !fits {
return Err(ResidentChunkError::DirtyRegionOutOfBounds {
region: *region,
chunk_extent: self.bounds.extent,
});
}
}
let plane = self.plane(id)?;
if self.planes.len() != 1 {
return Err(ResidentChunkError::PatchRequiresSinglePlane {
plane_count: self.planes.len(),
});
}
if plane.layout.element_type != T::ELEMENT_TYPE {
return Err(ResidentChunkError::PatchElementType {
plane: id.clone(),
expected: plane.layout.element_type,
actual: T::ELEMENT_TYPE,
});
}
let byte_offset = element_offset
.checked_mul(T::ELEMENT_TYPE.byte_width())
.ok_or_else(|| ResidentChunkError::PatchRangeOverflow { plane: id.clone() })?;
let byte_len = size_of_val(values);
let byte_end = byte_offset
.checked_add(byte_len)
.ok_or_else(|| ResidentChunkError::PatchRangeOverflow { plane: id.clone() })?;
if byte_len == 0 || byte_end > plane.layout.byte_len() {
return Err(ResidentChunkError::PatchRange {
plane: id.clone(),
byte_offset,
byte_len,
plane_byte_len: plane.layout.byte_len(),
});
}
let alignment = wgpu::COPY_BUFFER_ALIGNMENT as usize;
if !byte_offset.is_multiple_of(alignment) || !byte_len.is_multiple_of(alignment) {
return Err(ResidentChunkError::UnalignedPatch {
plane: id.clone(),
byte_offset,
byte_len,
required_alignment: alignment,
});
}
let allocation = self.allocation(&plane.handle)?;
queue.write_buffer(
allocation.buffer(),
allocation.offset() + byte_offset as u64,
bytemuck::cast_slice(values),
);
self.stamp = committed_stamp;
self.dirty_regions = dirty_regions;
Ok(())
}
pub fn raw_kernel_view(&self, id: &PlaneId) -> Result<RawKernelView, ResidentChunkError> {
let plane = self.plane(id)?;
let allocation = self.allocation(&plane.handle)?;
Ok(RawKernelView {
allocation,
layout: plane.layout,
class: plane.class,
stamp: self.stamp,
_lease: plane.handle.clone(),
})
}
pub fn burn_f32_view(&self, id: &PlaneId) -> Result<BurnTensorView, ResidentChunkError> {
let plane = self.plane(id)?;
if plane.layout.element_type != PlaneElementType::F32 {
return Err(ResidentChunkError::BurnElementType {
plane: id.clone(),
actual: plane.layout.element_type,
});
}
let allocation = self.allocation(&plane.handle)?;
let primitive = CubeTensor::<WgpuRuntime>::new_contiguous(
self.compute().clone(),
self.client.device.clone(),
Shape::new(plane.layout.shape),
plane.handle.clone(),
DType::F32,
);
let tensor = Tensor::from_primitive::<Wgpu>(primitive);
Ok(BurnTensorView {
tensor,
allocation,
layout: plane.layout,
class: plane.class,
stamp: self.stamp,
})
}
fn compute(&self) -> &ComputeClient<WgpuRuntime> {
&self.client.compute
}
fn plane(&self, id: &PlaneId) -> Result<&ResidentPlane, ResidentChunkError> {
self.planes
.get(id)
.ok_or_else(|| ResidentChunkError::UnknownPlane(id.clone()))
}
fn allocation(&self, handle: &Handle) -> Result<BufferIdentity, ResidentChunkError> {
let managed = self
.compute()
.get_resource(handle.clone())
.map_err(|error| ResidentChunkError::Resource(error.to_string()))?;
let resource = managed.resource();
Ok(BufferIdentity {
buffer: resource.buffer.clone(),
offset: resource.offset,
size: resource.size,
})
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum ResidentChunkError {
EmptyPlaneId,
EmptyShape {
shape: [usize; 3],
},
ShapeOverflow {
shape: [usize; 3],
},
DuplicatePlane(PlaneId),
UnknownPlane(PlaneId),
ElementCount {
plane: PlaneId,
expected: usize,
actual: usize,
},
BurnElementType {
plane: PlaneId,
actual: PlaneElementType,
},
StaleStamp {
expected: ChunkStamp,
actual: ChunkStamp,
},
NonAdvancingStamp {
previous: ChunkStamp,
committed: ChunkStamp,
},
DirtyRegionOutOfBounds {
region: DirtyRegion,
chunk_extent: [u32; 3],
},
PatchRequiresSinglePlane {
plane_count: usize,
},
PatchElementType {
plane: PlaneId,
expected: PlaneElementType,
actual: PlaneElementType,
},
PatchRangeOverflow {
plane: PlaneId,
},
PatchRange {
plane: PlaneId,
byte_offset: usize,
byte_len: usize,
plane_byte_len: usize,
},
UnalignedPatch {
plane: PlaneId,
byte_offset: usize,
byte_len: usize,
required_alignment: usize,
},
Resource(String),
}
impl fmt::Display for ResidentChunkError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::EmptyPlaneId => formatter.write_str("resident plane id cannot be empty"),
Self::EmptyShape { shape } => {
write!(formatter, "resident plane shape {shape:?} is empty")
}
Self::ShapeOverflow { shape } => {
write!(
formatter,
"resident plane shape {shape:?} overflows its allocation"
)
}
Self::DuplicatePlane(plane) => {
write!(formatter, "resident plane {plane} already exists")
}
Self::UnknownPlane(plane) => write!(formatter, "resident plane {plane} does not exist"),
Self::ElementCount {
plane,
expected,
actual,
} => write!(
formatter,
"resident plane {plane} needs {expected} elements, got {actual}"
),
Self::BurnElementType { plane, actual } => write!(
formatter,
"resident plane {plane} is {actual:?}, not F32, so it has no Burn float view"
),
Self::StaleStamp { expected, actual } => write!(
formatter,
"resident patch expected stamp {expected:?}, but the chunk is at {actual:?}"
),
Self::NonAdvancingStamp {
previous,
committed,
} => write!(
formatter,
"resident patch must advance revision and read epoch beyond {previous:?}, got {committed:?}"
),
Self::DirtyRegionOutOfBounds {
region,
chunk_extent,
} => write!(
formatter,
"resident dirty region {region:?} does not fit chunk extent {chunk_extent:?}"
),
Self::PatchRequiresSinglePlane { plane_count } => write!(
formatter,
"single-plane resident patch cannot restamp a bundle containing {plane_count} planes"
),
Self::PatchElementType {
plane,
expected,
actual,
} => write!(
formatter,
"resident plane {plane} stores {expected:?}, not patch element type {actual:?}"
),
Self::PatchRangeOverflow { plane } => {
write!(
formatter,
"resident patch range for plane {plane} overflowed"
)
}
Self::PatchRange {
plane,
byte_offset,
byte_len,
plane_byte_len,
} => write!(
formatter,
"resident patch [{byte_offset}..{}) does not fit plane {plane}'s {plane_byte_len} bytes",
byte_offset + byte_len
),
Self::UnalignedPatch {
plane,
byte_offset,
byte_len,
required_alignment,
} => write!(
formatter,
"resident patch for plane {plane} has byte offset {byte_offset} and length {byte_len}; both must be multiples of {required_alignment}"
),
Self::Resource(error) => write!(formatter, "CubeCL resource export failed: {error}"),
}
}
}
impl std::error::Error for ResidentChunkError {}