use super::Result;
use super::context::Context;
use cl3::device::{
CL_DEVICE_SVM_ATOMICS, CL_DEVICE_SVM_COARSE_GRAIN_BUFFER, CL_DEVICE_SVM_FINE_GRAIN_BUFFER,
CL_DEVICE_SVM_FINE_GRAIN_SYSTEM,
};
use cl3::memory::{
CL_MEM_READ_WRITE, CL_MEM_SVM_ATOMICS, CL_MEM_SVM_FINE_GRAIN_BUFFER, svm_alloc, svm_free,
};
use cl3::types::{cl_device_svm_capabilities, cl_svm_mem_flags, cl_uint};
use libc::c_void;
#[cfg(feature = "serde")]
use serde::de::{Deserialize, DeserializeSeed, Deserializer, Error, SeqAccess, Visitor};
#[cfg(feature = "serde")]
use serde::ser::{Serialize, SerializeSeq, Serializer};
use std::alloc::{self, Layout};
use std::fmt;
use std::fmt::Debug;
use std::iter::IntoIterator;
use std::marker::PhantomData;
use std::mem;
use std::ops::{Deref, DerefMut};
use std::ptr;
#[allow(unused_imports)]
use std::result;
struct SvmRawVec<'a, T> {
ptr: *mut T,
cap: usize,
context: &'a Context,
fine_grain_buffer: bool,
fine_grain_system: bool,
atomics: bool,
}
unsafe impl<T: Send> Send for SvmRawVec<'_, T> {}
unsafe impl<T: Sync> Sync for SvmRawVec<'_, T> {}
impl<'a, T> SvmRawVec<'a, T> {
fn new(context: &'a Context, svm_capabilities: cl_device_svm_capabilities) -> Self {
assert!(0 < mem::size_of::<T>(), "No Zero Sized Types!");
assert!(
0 != svm_capabilities
& (CL_DEVICE_SVM_COARSE_GRAIN_BUFFER | CL_DEVICE_SVM_FINE_GRAIN_BUFFER),
"No OpenCL SVM, use OpenCL buffers"
);
let fine_grain_buffer: bool = svm_capabilities & CL_DEVICE_SVM_FINE_GRAIN_BUFFER != 0;
let fine_grain_system: bool = svm_capabilities & CL_DEVICE_SVM_FINE_GRAIN_SYSTEM != 0;
let atomics: bool = (fine_grain_buffer || fine_grain_system)
&& (svm_capabilities & CL_DEVICE_SVM_ATOMICS != 0);
SvmRawVec {
ptr: ptr::null_mut(),
cap: 0,
context,
fine_grain_buffer,
fine_grain_system,
atomics,
}
}
fn with_capacity(
context: &'a Context,
svm_capabilities: cl_device_svm_capabilities,
capacity: usize,
) -> Result<Self> {
let mut v = Self::new(context, svm_capabilities);
v.grow(capacity)?;
Ok(v)
}
fn with_capacity_zeroed(
context: &'a Context,
svm_capabilities: cl_device_svm_capabilities,
capacity: usize,
) -> Result<Self> {
let mut v = Self::with_capacity(context, svm_capabilities, capacity)?;
v.zero(capacity);
Ok(v)
}
#[allow(clippy::cast_possible_truncation)]
fn grow(&mut self, count: usize) -> Result<()> {
let elem_size = mem::size_of::<T>();
let new_cap = if (0 < self.cap) && (count - self.cap == 1) {
2 * self.cap
} else {
count
};
let size = elem_size * new_cap;
assert!(size <= (isize::MAX as usize) / 2, "capacity overflow");
let ptr = if self.fine_grain_system {
let new_layout = Layout::array::<T>(new_cap).expect("Layout::array failure.");
let new_ptr = unsafe { alloc::alloc(new_layout).cast::<c_void>() };
if new_ptr.is_null() {
alloc::handle_alloc_error(new_layout);
}
new_ptr
} else {
let svm_mem_flags: cl_svm_mem_flags = if self.fine_grain_buffer {
if self.atomics {
CL_MEM_SVM_FINE_GRAIN_BUFFER | CL_MEM_READ_WRITE | CL_MEM_SVM_ATOMICS
} else {
CL_MEM_SVM_FINE_GRAIN_BUFFER | CL_MEM_READ_WRITE
}
} else {
CL_MEM_READ_WRITE
};
let alignment = mem::align_of::<T>();
unsafe {
svm_alloc(
self.context.get(),
svm_mem_flags,
size,
alignment as cl_uint,
)?
}
};
if 0 < self.cap {
unsafe { ptr::copy(self.ptr, ptr.cast::<T>(), self.cap) };
if self.fine_grain_system {
let layout = Layout::array::<T>(self.cap).expect("Layout::array failure.");
unsafe {
alloc::dealloc(self.ptr.cast::<u8>(), layout);
}
} else {
unsafe {
let _ = svm_free(self.context.get(), self.ptr.cast::<c_void>());
};
}
}
self.ptr = ptr.cast::<T>();
self.cap = new_cap;
Ok(())
}
const fn zero(&mut self, count: usize) {
unsafe { ptr::write_bytes(self.ptr, 0u8, count) };
}
}
impl<T> Drop for SvmRawVec<'_, T> {
fn drop(&mut self) {
if !self.ptr.is_null() {
if self.fine_grain_system {
let layout = Layout::array::<T>(self.cap).expect("Layout::array failure.");
unsafe {
alloc::dealloc(self.ptr.cast::<u8>(), layout);
}
} else {
unsafe {
let _ = svm_free(self.context.get(), self.ptr.cast::<c_void>());
};
}
self.ptr = ptr::null_mut();
}
}
}
pub struct SvmVec<'a, T> {
buf: SvmRawVec<'a, T>,
len: usize,
}
impl<'a, T> SvmVec<'a, T> {
#[must_use]
const fn ptr(&self) -> *mut T {
self.buf.ptr
}
#[must_use]
pub const fn cap(&self) -> usize {
self.buf.cap
}
#[must_use]
pub const fn len(&self) -> usize {
self.len
}
#[must_use]
pub const fn is_empty(&self) -> bool {
self.len == 0
}
#[must_use]
pub const fn is_fine_grain_buffer(&self) -> bool {
self.buf.fine_grain_buffer
}
#[must_use]
pub const fn is_fine_grain_system(&self) -> bool {
self.buf.fine_grain_system
}
#[must_use]
pub const fn is_fine_grained(&self) -> bool {
self.buf.fine_grain_buffer || self.buf.fine_grain_system
}
#[must_use]
pub const fn has_atomics(&self) -> bool {
self.buf.atomics
}
pub const fn clear(&mut self) {
self.len = 0;
}
pub unsafe fn set_len(&mut self, new_len: usize) -> Result<()> {
if self.cap() < new_len {
self.buf.grow(new_len)?;
}
self.len = new_len;
Ok(())
}
#[must_use]
pub fn new(context: &'a Context) -> Self {
let svm_capabilities = context.get_svm_mem_capability();
SvmVec {
buf: SvmRawVec::new(context, svm_capabilities),
len: 0,
}
}
pub fn allocate(context: &'a Context, len: usize) -> Result<Self> {
let svm_capabilities = context.get_svm_mem_capability();
Ok(SvmVec {
buf: SvmRawVec::with_capacity(context, svm_capabilities, len)?,
len,
})
}
pub fn with_capacity(context: &'a Context, capacity: usize) -> Result<Self> {
let svm_capabilities = context.get_svm_mem_capability();
Ok(SvmVec {
buf: SvmRawVec::with_capacity(context, svm_capabilities, capacity)?,
len: 0,
})
}
pub fn allocate_zeroed(context: &'a Context, len: usize) -> Result<Self> {
let svm_capabilities = context.get_svm_mem_capability();
let fine_grain_buffer: bool = svm_capabilities & CL_DEVICE_SVM_FINE_GRAIN_BUFFER != 0;
assert!(
fine_grain_buffer,
"SVM is not fine grained, use `allocate` instead."
);
Ok(SvmVec {
buf: SvmRawVec::with_capacity_zeroed(context, svm_capabilities, len)?,
len,
})
}
pub fn reserve(&mut self, capacity: usize) -> Result<()> {
self.buf.grow(capacity)
}
pub fn push(&mut self, elem: T) {
if self.len == self.cap() {
assert!(
self.is_fine_grained(),
"SVM is not fine grained, cannot grow the vector."
);
self.buf
.grow(self.len + 1)
.expect("Cannot grow the vector.");
}
unsafe {
ptr::write(self.ptr().add(self.len), elem);
}
self.len += 1;
}
pub const fn pop(&mut self) -> Option<T> {
if self.len == 0 {
None
} else {
self.len -= 1;
unsafe { Some(ptr::read(self.ptr().add(self.len))) }
}
}
pub fn insert(&mut self, index: usize, elem: T) {
assert!(index <= self.len, "index out of bounds");
if self.cap() == self.len {
assert!(
self.is_fine_grained(),
"SVM is not fine grained, cannot grow the vector."
);
self.buf.grow(self.len + 1).expect("Layout::array failure.");
}
unsafe {
if index < self.len {
ptr::copy(
self.ptr().add(index),
self.ptr().add(index + 1),
self.len - index,
);
}
ptr::write(self.ptr().add(index), elem);
self.len += 1;
}
}
pub fn remove(&mut self, index: usize) -> T {
assert!(index < self.len, "index out of bounds");
unsafe {
self.len -= 1;
let result = ptr::read(self.ptr().add(index));
ptr::copy(
self.ptr().add(index + 1),
self.ptr().add(index),
self.len - index,
);
result
}
}
pub fn drain(&mut self) -> Drain<'_, T> {
unsafe {
let iter = RawValIter::new(self);
self.len = 0;
Drain {
iter,
vec: PhantomData,
}
}
}
}
impl<'a, T> IntoIterator for SvmVec<'a, T> {
type Item = T;
type IntoIter = IntoIter<'a, Self::Item>;
fn into_iter(self) -> Self::IntoIter {
unsafe {
let iter = RawValIter::new(&self);
let buf = ptr::read(&self.buf);
mem::forget(self);
Self::IntoIter { iter, _buf: buf }
}
}
}
impl<T> Drop for SvmVec<'_, T> {
fn drop(&mut self) {
while self.pop().is_some() {}
}
}
impl<T> Deref for SvmVec<'_, T> {
type Target = [T];
fn deref(&self) -> &[T] {
unsafe { std::slice::from_raw_parts(self.ptr(), self.len) }
}
}
impl<T> DerefMut for SvmVec<'_, T> {
fn deref_mut(&mut self) -> &mut [T] {
unsafe { std::slice::from_raw_parts_mut(self.ptr(), self.len) }
}
}
impl<T: Debug> fmt::Debug for SvmVec<'_, T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
fmt::Debug::fmt(&**self, f)
}
}
#[cfg(feature = "serde")]
pub struct ExtendSvmVec<'a, 'b, T: 'a>(pub &'a mut SvmVec<'b, T>);
#[cfg(feature = "serde")]
impl<'de, T> DeserializeSeed<'de> for ExtendSvmVec<'_, '_, T>
where
T: Deserialize<'de>,
{
type Value = ();
fn deserialize<D>(self, deserializer: D) -> result::Result<Self::Value, D::Error>
where
D: Deserializer<'de>,
{
struct ExtendSvmVecVisitor<'a, 'b, T: 'a>(&'a mut SvmVec<'b, T>);
impl<'de, T> Visitor<'de> for ExtendSvmVecVisitor<'_, '_, T>
where
T: Deserialize<'de>,
{
type Value = ();
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("an array")
}
fn visit_seq<A>(self, mut seq: A) -> result::Result<(), A::Error>
where
A: SeqAccess<'de>,
{
if let Some(size) = seq.size_hint() {
let len = self.0.len + size;
self.0.reserve(len).map_err(A::Error::custom)?;
}
while let Some(elem) = seq.next_element()? {
self.0.push(elem);
}
Ok(())
}
}
deserializer.deserialize_seq(ExtendSvmVecVisitor(self.0))
}
}
#[cfg(feature = "serde")]
impl<T> Serialize for SvmVec<'_, T>
where
T: Serialize,
{
fn serialize<S>(&self, serializer: S) -> result::Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut seq = serializer.serialize_seq(Some(self.len()))?;
for element in self.iter() {
seq.serialize_element(element)?;
}
seq.end()
}
}
struct RawValIter<T> {
start: *const T,
end: *const T,
}
unsafe impl<T: Send> Send for RawValIter<T> {}
impl<T> RawValIter<T> {
unsafe fn new(slice: &[T]) -> Self {
unsafe {
Self {
start: slice.as_ptr(),
end: if mem::size_of::<T>() == 0 {
((slice.as_ptr() as usize) + slice.len()) as *const _
} else if slice.is_empty() {
slice.as_ptr()
} else {
slice.as_ptr().add(slice.len())
},
}
}
}
}
impl<T> Iterator for RawValIter<T> {
type Item = T;
fn next(&mut self) -> Option<T> {
if self.start == self.end {
None
} else {
unsafe {
let result = ptr::read(self.start);
self.start = if mem::size_of::<T>() == 0 {
(self.start as usize + 1) as *const _
} else {
self.start.offset(1)
};
Some(result)
}
}
}
fn size_hint(&self) -> (usize, Option<usize>) {
let elem_size = mem::size_of::<T>();
let len =
(self.end as usize - self.start as usize) / if elem_size == 0 { 1 } else { elem_size };
(len, Some(len))
}
}
impl<T> DoubleEndedIterator for RawValIter<T> {
fn next_back(&mut self) -> Option<T> {
if self.start == self.end {
None
} else {
unsafe {
self.end = if mem::size_of::<T>() == 0 {
(self.end as usize - 1) as *const _
} else {
self.end.offset(-1)
};
Some(ptr::read(self.end))
}
}
}
}
pub struct IntoIter<'a, T> {
_buf: SvmRawVec<'a, T>, iter: RawValIter<T>,
}
impl<T> Iterator for IntoIter<'_, T> {
type Item = T;
fn next(&mut self) -> Option<T> {
self.iter.next()
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.iter.size_hint()
}
}
impl<T> DoubleEndedIterator for IntoIter<'_, T> {
fn next_back(&mut self) -> Option<T> {
self.iter.next_back()
}
}
impl<T> Drop for IntoIter<'_, T> {
fn drop(&mut self) {
for _ in &mut *self {}
}
}
pub struct Drain<'a, T: 'a> {
vec: PhantomData<&'a mut SvmVec<'a, T>>,
iter: RawValIter<T>,
}
impl<T> Iterator for Drain<'_, T> {
type Item = T;
fn next(&mut self) -> Option<T> {
self.iter.next()
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.iter.size_hint()
}
}
impl<T> DoubleEndedIterator for Drain<'_, T> {
fn next_back(&mut self) -> Option<T> {
self.iter.next_back()
}
}
impl<T> Drop for Drain<'_, T> {
fn drop(&mut self) {
for _ in &mut self.iter {}
}
}