use core::{ffi::c_void, mem::ManuallyDrop};
#[cfg(feature = "cpu")]
use crate::cpu::{CPUPtr, CPU};
#[cfg(not(feature = "cpu"))]
use crate::CPU;
use crate::{
flag::AllocFlag, shape::Shape, Alloc, ClearBuf, CloneBuf, CommonPtrs, Device, DevicelessAble,
Ident, IsShapeIndep, MainMemory, PtrType, Read, ShallowCopy, WriteBuf,
};
pub use self::num::Num;
pub use impl_from_const::*;
mod impl_from;
mod impl_from_const;
mod num;
#[cfg_attr(feature = "cpu", doc = "```")]
#[cfg_attr(not(feature = "cpu"), doc = "```ignore")]
pub struct Buffer<'a, T = f32, D: Device = CPU, S: Shape = ()> {
pub ptr: D::Ptr<T, S>,
pub device: Option<&'a D>,
#[cfg(not(feature = "no-std"))]
pub ident: Option<Ident>,
}
unsafe impl<'a, T, D: Device, S: Shape> Send for Buffer<'a, T, D, S> {}
unsafe impl<'a, T, D: Device, S: Shape> Sync for Buffer<'a, T, D, S> {}
impl<'a, T, D: Device, S: Shape> Buffer<'a, T, D, S> {
#[cfg_attr(feature = "cpu", doc = "```")]
#[cfg_attr(not(feature = "cpu"), doc = "```ignore")]
#[inline]
pub fn new(device: &'a D, len: usize) -> Buffer<'a, T, D, S>
where
D: Alloc<'a, T, S>,
{
let ptr = device.alloc(len, AllocFlag::None);
#[cfg(not(feature = "no-std"))]
let ident = device.add_to_cache(&ptr);
Buffer {
ptr,
device: Some(device),
#[cfg(not(feature = "no-std"))]
ident,
}
}
#[cfg_attr(feature = "cpu", doc = "```")]
#[cfg_attr(not(feature = "cpu"), doc = "```ignore")]
#[inline]
pub fn deviceless<'b>(device: &'b D, len: usize) -> Buffer<'a, T, D, S>
where
D: DevicelessAble<'b, T, S>,
{
Buffer {
ptr: device.alloc(len, AllocFlag::None),
#[cfg(not(feature = "no-std"))]
ident: None,
device: None,
}
}
pub fn device(&self) -> &'a D {
self.device
.expect("Called device() on a deviceless buffer.")
}
#[inline]
pub fn read(&'a self) -> D::Read<'a>
where
T: Clone + Default,
D: Read<T, S>,
{
self.device().read(self)
}
#[cfg_attr(feature = "cpu", doc = "```")]
#[cfg_attr(not(feature = "cpu"), doc = "```ignore")]
#[inline]
#[cfg(not(feature = "no-std"))]
pub fn read_to_vec(&self) -> Vec<T>
where
D: Read<T, S>,
T: Default + Clone,
{
self.device().read_to_vec(self)
}
#[cfg_attr(feature = "cpu", doc = "```")]
#[cfg_attr(not(feature = "cpu"), doc = "```ignore")]
#[inline]
pub fn write(&mut self, data: &[T])
where
D: WriteBuf<T, S, D>,
{
self.device().write(self, data)
}
#[inline]
pub fn write_buf(&mut self, src: &Buffer<T, D, S>)
where
T: Clone,
D: WriteBuf<T, S, D>,
{
self.device().write_buf(self, src)
}
#[cfg_attr(feature = "cpu", doc = "```")]
#[cfg_attr(not(feature = "cpu"), doc = "```ignore")]
#[inline]
pub fn len(&self) -> usize {
self.ptr.size()
}
#[inline]
pub unsafe fn shallow(&self) -> Buffer<'a, T, D, S>
where
<D as Device>::Ptr<T, S>: ShallowCopy,
{
Buffer {
ptr: self.ptr.shallow(),
device: self.device,
#[cfg(not(feature = "no-std"))]
ident: self.ident,
}
}
pub unsafe fn shallow_or_clone(&self) -> Buffer<'a, T, D, S>
where
<D as Device>::Ptr<T, S>: ShallowCopy,
T: Clone,
D: CloneBuf<'a, T, S>,
{
{
#[cfg(not(feature = "realloc"))]
self.shallow()
}
#[cfg(feature = "realloc")]
self.clone()
}
#[inline]
pub fn id(&self) -> Ident {
#[cfg(feature = "no-std")]
{
unimplemented!("This buffer has no trackable id. Who?: e.g. 'Stack' Buffer, Buffers created via Buffer::from_raw_host..(..), `Num` (scalar) Buffer")
}
#[cfg(not(feature = "no-std"))]
self.ident.expect("This buffer has no trackable id. Who?: e.g. 'Stack' Buffer, Buffers created via Buffer::from_raw_host..(..), `Num` (scalar) Buffer")
}
pub fn clear(&mut self)
where
D: ClearBuf<T, S, D>,
{
self.device().clear(self)
}
}
impl<'a, T, D: Device, S: Shape> Drop for Buffer<'a, T, D, S> {
#[inline]
fn drop(&mut self) {
if self.ptr.flag() != AllocFlag::None {
return;
}
#[cfg(not(feature = "no-std"))]
if let Some(device) = self.device {
if let Some(ident) = self.ident {
device.remove(ident)
}
}
}
}
impl<'a, T, D: Device, S: Shape> Buffer<'a, T, D, S> {
#[cfg_attr(feature = "cpu", doc = "```")]
#[cfg_attr(not(feature = "cpu"), doc = "```ignore")]
#[inline]
pub fn to_dims<O: Shape>(self) -> Buffer<'a, T, D, O>
where
D: crate::ToDim<T, S, O>,
D::Ptr<T, S>: ShallowCopy,
{
let buf = ManuallyDrop::new(self);
let ptr = buf.device().to_dim(unsafe { buf.ptr.shallow() });
Buffer {
ptr,
device: buf.device,
#[cfg(not(feature = "no-std"))]
ident: buf.ident,
}
}
}
impl<'a, T, D: IsShapeIndep, S: Shape> Buffer<'a, T, D, S> {
#[inline]
pub fn as_dims<'b, O: Shape>(&self) -> &Buffer<'b, T, D, O> {
unsafe { &*(self as *const Self).cast() }
}
#[inline]
pub fn as_dims_mut<'b, O: Shape>(&mut self) -> &mut Buffer<'b, T, D, O> {
unsafe { &mut *(self as *mut Self).cast() }
}
}
impl<'a, T, D: Device, S: Shape> Buffer<'a, T, D, S>
where
D::Ptr<T, S>: CommonPtrs<T>,
{
#[inline]
pub fn ptrs(&self) -> (*const T, *mut c_void, u64) {
self.ptr.ptrs()
}
#[inline]
pub fn ptrs_mut(&mut self) -> (*mut T, *mut c_void, u64) {
self.ptr.ptrs_mut()
}
}
impl<'a, T, D: Device> Buffer<'a, T, D> {
#[inline]
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
impl<'a, T, D: Device, S: Shape> Buffer<'a, T, D, S> {
#[inline]
pub fn from_slice(device: &'a D, slice: &[T]) -> Self
where
T: Clone,
D: Alloc<'a, T, S>,
{
let ptr = device.with_slice(slice);
#[cfg(not(feature = "no-std"))]
let ident = device.add_to_cache(&ptr);
Buffer {
ptr,
#[cfg(not(feature = "no-std"))]
ident,
device: Some(device),
}
}
#[cfg(not(feature = "no-std"))]
#[inline]
pub fn from_vec(device: &'a D, data: Vec<T>) -> Self
where
T: Clone,
D: Alloc<'a, T, S>,
{
let ptr = device.alloc_with_vec(data);
let ident = device.add_to_cache(&ptr);
Buffer {
ptr,
ident,
device: Some(device),
}
}
#[inline]
pub fn from_array(device: &'a D, array: S::ARR<T>) -> Buffer<T, D, S>
where
T: Clone,
D: Alloc<'a, T, S>,
{
let ptr = device.with_array(array);
#[cfg(not(feature = "no-std"))]
let ident = device.add_to_cache(&ptr);
Buffer {
ptr,
#[cfg(not(feature = "no-std"))]
ident,
device: Some(device),
}
}
}
#[cfg(feature = "cpu")]
impl<'a, T, S: Shape> Buffer<'a, T, CPU, S> {
#[inline]
pub unsafe fn from_raw_host(ptr: *mut T, len: usize) -> Buffer<'a, T, CPU, S> {
Buffer {
ptr: CPUPtr::from_ptr(ptr, len, AllocFlag::Wrapper),
device: None,
ident: None,
}
}
#[inline]
pub unsafe fn from_raw_host_device(
device: &'a CPU,
ptr: *mut T,
len: usize,
) -> Buffer<'a, T, CPU, S> {
Buffer {
ptr: CPUPtr::from_ptr(ptr, len, AllocFlag::Wrapper),
device: Some(device),
ident: None,
}
}
}
#[cfg(feature = "opencl")]
impl<'a, T, S: Shape> Buffer<'a, T, crate::OpenCL, S> {
#[inline]
pub fn cl_ptr(&self) -> *mut c_void {
assert!(
!self.ptr.ptr.is_null(),
"called cl_ptr() on an invalid OpenCL buffer"
);
self.ptrs().1
}
}
#[cfg(feature = "cuda")]
impl<'a, T> Buffer<'a, T, crate::CUDA> {
#[inline]
pub fn cu_ptr(&self) -> u64 {
assert!(
self.ptrs().2 != 0,
"called cu_ptr() on an invalid CUDA buffer"
);
self.ptr.ptr
}
}
impl<'a, T, D: MainMemory, S: Shape> Buffer<'a, T, D, S> {
#[inline(always)]
pub fn as_slice(&self) -> &[T] {
self
}
#[inline(always)]
pub fn as_mut_slice(&mut self) -> &mut [T] {
self
}
}
impl<'a, T, D: MainMemory, S: Shape> Buffer<'a, T, D, S>
where
D::Ptr<T, S>: CommonPtrs<T>,
{
#[inline]
pub fn host_ptr(&self) -> *const T {
assert!(
!self.ptrs().0.is_null(),
"called host_ptr() on an invalid CPU buffer (this would dereference a null pointer)"
);
self.ptrs().0
}
#[inline]
pub fn host_ptr_mut(&mut self) -> *mut T {
assert!(
!self.ptrs().0.is_null(),
"called host_ptr_mut() on an invalid CPU buffer (this would dereference a null pointer)"
);
self.ptrs_mut().0
}
}
impl<'a, T, D, S> Clone for Buffer<'a, T, D, S>
where
T: Clone,
D: CloneBuf<'a, T, S> + Device,
S: Shape,
{
fn clone(&self) -> Self {
self.device().clone_buf(self)
}
}
impl<'a, T, D: Device, S: Shape> Default for Buffer<'a, T, D, S>
where
D::Ptr<T, S>: Default,
{
fn default() -> Self {
Self {
ptr: D::Ptr::<T, S>::default(),
device: None,
#[cfg(not(feature = "no-std"))]
ident: None,
}
}
}
impl<T, D: MainMemory> AsRef<[T]> for Buffer<'_, T, D> {
#[inline]
fn as_ref(&self) -> &[T] {
self
}
}
impl<T, D: MainMemory> AsMut<[T]> for Buffer<'_, T, D> {
#[inline]
fn as_mut(&mut self) -> &mut [T] {
self
}
}
#[cfg_attr(feature = "cpu", doc = "```")]
#[cfg_attr(not(feature = "cpu"), doc = "```ignore")]
impl<T, D: MainMemory, S: Shape> core::ops::Deref for Buffer<'_, T, D, S> {
type Target = [T];
#[inline]
fn deref(&self) -> &Self::Target {
unsafe { core::slice::from_raw_parts(D::as_ptr(&self.ptr), self.len()) }
}
}
#[cfg_attr(feature = "cpu", doc = "```")]
#[cfg_attr(not(feature = "cpu"), doc = "```ignore")]
impl<T, D: MainMemory, S: Shape> core::ops::DerefMut for Buffer<'_, T, D, S> {
#[inline]
fn deref_mut(&mut self) -> &mut Self::Target {
unsafe { core::slice::from_raw_parts_mut(D::as_ptr_mut(&mut self.ptr), self.len()) }
}
}
#[cfg(not(feature = "no-std"))]
use core::fmt::Debug;
#[cfg(not(feature = "no-std"))]
impl<'a, T, D> Debug for Buffer<'a, T, D>
where
T: Debug + Default + Clone + 'a,
D: Read<T> + Device + 'a,
for<'b> <D as Read<T>>::Read<'b>: Debug,
D::Ptr<T, ()>: CommonPtrs<T>,
{
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("Buffer")
.field("ptr (CPU, CL, CU)", &self.ptrs())
.field("len", &self.len());
writeln!(f, ",")?;
if !self.ptrs().0.is_null() {
let slice = unsafe { std::slice::from_raw_parts(self.ptrs().0, self.len()) };
writeln!(f, "CPU: {slice:?}")?;
}
#[cfg(feature = "opencl")]
if !self.ptrs().1.is_null() {
write!(f, "OpenCL: {:?}, ", self.read())?;
}
#[cfg(feature = "cuda")]
if self.ptrs().2 != 0 {
write!(f, "CUDA: {:?}, ", self.read())?;
}
write!(
f,
"datatype={}, device={device} }}",
core::any::type_name::<T>(),
device = core::any::type_name::<D>()
)
}
}
impl<'a, T, D: MainMemory, S: Shape> core::iter::IntoIterator for &'a Buffer<'_, T, D, S> {
type Item = &'a T;
type IntoIter = core::slice::Iter<'a, T>;
#[inline]
fn into_iter(self) -> Self::IntoIter {
self.iter()
}
}
impl<'a, T, D: MainMemory, S: Shape> core::iter::IntoIterator for &'a mut Buffer<'_, T, D, S> {
type Item = &'a mut T;
type IntoIter = core::slice::IterMut<'a, T>;
#[inline]
fn into_iter(self) -> Self::IntoIter {
self.iter_mut()
}
}
#[cfg(test)]
mod tests {
use crate::Buffer;
#[cfg(feature = "cpu")]
#[test]
fn test_deref() {
let device = crate::CPU::new();
let buf: Buffer<i32> = Buffer::from((&device, [1, 2, 3, 4]));
let slice = &*buf;
assert_eq!(slice, &[1, 2, 3, 4]);
}
#[cfg(feature = "opencl")]
#[cfg(unified_cl)]
#[test]
fn test_deref_cl() -> crate::Result<()> {
use crate::OpenCL;
let device = OpenCL::new(0)?;
let buf = Buffer::from((&device, [1, 2, 3, 4]));
let slice = &*buf;
assert_eq!(slice, &[1, 2, 3, 4]);
Ok(())
}
#[cfg(feature = "stack")]
#[test]
fn test_deref_stack() -> crate::Result<()> {
use crate::{shape::Dim1, stack::Stack};
let buf = Buffer::<i32, _, Dim1<4>>::from((Stack, [1i32, 2, 3, 4]));
let slice = &*buf;
assert_eq!(slice, &[1, 2, 3, 4]);
Ok(())
}
#[cfg(feature = "cpu")]
#[test]
fn test_debug_print() {
let device = crate::CPU::new();
let buf = Buffer::from((&device, [1, 2, 3, 4, 5, 6]));
println!("{buf:?}",);
}
#[cfg(feature = "cpu")]
#[test]
fn test_to_dims() {
use crate::Dim2;
let device = crate::CPU::new();
let buf = Buffer::from((&device, [1, 2, 3, 4, 5, 6]));
let buf_dim2 = buf.to_dims::<Dim2<3, 2>>();
buf_dim2.to_dims::<()>();
}
#[cfg(feature = "cpu")]
#[test]
fn test_id_cpu() {
use crate::{Ident, CPU};
let device = CPU::new();
let buf = Buffer::from((&device, [1, 2, 3, 4]));
assert_eq!(buf.id(), Ident { idx: 0, len: 4 })
}
#[cfg(feature = "stack")]
#[cfg(not(feature = "no-std"))]
#[should_panic]
#[test]
fn test_id_stack() {
use crate::{Stack, WithShape};
let device = Stack;
let buf = Buffer::with(&device, [1, 2, 3, 4]);
buf.id();
}
}