use crate::any::AnyView;
use crate::collections::shape::Shape;
use crate::derive::{Object, ObjectRef};
use crate::dtype::AsDLDataType;
use crate::dtype::DLDataTypeExt;
use crate::error::Result;
use crate::object::{Object, ObjectArc, ObjectCore, ObjectCoreWithExtraItems};
use tvm_ffi_sys::TVMFFIAny;
use tvm_ffi_sys::TVMFFITypeIndex as TypeIndex;
use tvm_ffi_sys::dlpack::{DLDataType, DLDevice, DLDeviceType, DLTensor};
use tvm_ffi_sys::{TVMFFIEnvTensorAlloc, TVMFFIObjectHandle};
pub unsafe trait NDAllocator: Send + Sync + 'static {
const MIN_ALIGN: usize;
unsafe fn alloc_data(&mut self, prototype: &DLTensor) -> *mut core::ffi::c_void;
unsafe fn free_data(&mut self, tensor: &DLTensor);
}
pub trait DLTensorExt {
unsafe fn numel(&self) -> usize;
}
impl DLTensorExt for DLTensor {
unsafe fn numel(&self) -> usize {
let shape =
unsafe { crate::util::slice_from_nullable_raw_parts(self.shape, self.ndim as usize) };
checked_numel(shape).expect("tensor shape has a negative extent or overflows usize")
}
}
fn checked_numel(shape: &[i64]) -> Option<usize> {
shape.iter().try_fold(1usize, |numel, &extent| {
numel.checked_mul(usize::try_from(extent).ok()?)
})
}
unsafe fn data_size(tensor: &DLTensor) -> usize {
let bits = tensor.dtype.bits as usize * tensor.dtype.lanes as usize;
unsafe { tensor.numel() }
.checked_mul(bits)
.map(|total| total.div_ceil(8))
.filter(|&size| size <= isize::MAX as usize)
.expect("tensor byte size overflows isize")
}
#[repr(C)]
#[derive(Object)]
#[object(type_key = "ffi.Tensor", unsafe type_index = TypeIndex::kTVMFFITensor)]
pub struct TensorObj {
object: Object,
dltensor: DLTensor,
}
#[repr(C)]
#[derive(ObjectRef, Clone)]
#[object_ref(skip_deref)]
pub struct Tensor {
data: ObjectArc<TensorObj>,
}
impl Tensor {
pub fn data_ptr(&self) -> *mut core::ffi::c_void {
self.data.dltensor.data
}
pub fn is_contiguous(&self) -> bool {
TensorView::from(self).is_contiguous()
}
pub fn data_as_slice<T: AsDLDataType>(&self) -> Result<&[T]> {
let (data, len) = self.cpu_elements::<T>()?;
unsafe { Ok(crate::util::slice_from_nullable_raw_parts(data, len)) }
}
#[allow(clippy::wrong_self_convention, clippy::mut_from_ref)]
pub unsafe fn data_as_slice_mut<T: AsDLDataType>(&self) -> Result<&mut [T]> {
let (data, len) = self.cpu_elements::<T>()?;
unsafe { Ok(crate::util::slice_from_nullable_raw_parts_mut(data, len)) }
}
fn cpu_elements<T: AsDLDataType>(&self) -> Result<(*mut T, usize)> {
let dtype = T::DL_DATA_TYPE;
if self.dtype() != dtype {
crate::bail!(
crate::error::TYPE_ERROR,
"Data type mismatch: expected {}, got {}",
dtype.to_string(),
self.dtype().to_string()
);
}
if self.device().device_type != DLDeviceType::kDLCPU as i32 {
crate::bail!(crate::error::RUNTIME_ERROR, "Tensor is not on CPU");
}
crate::ensure!(
self.is_contiguous(),
crate::error::RUNTIME_ERROR,
"Tensor is not contiguous"
);
let len = self.numel();
let t = &self.data.dltensor;
let data = t
.data
.cast::<u8>()
.wrapping_add(t.byte_offset as usize)
.cast::<T>();
crate::ensure!(
len == 0 || data.is_aligned(),
crate::error::RUNTIME_ERROR,
"Tensor data is not aligned for {}",
dtype.to_string()
);
Ok((data, len))
}
pub fn shape(&self) -> &[i64] {
let t = &self.data.dltensor;
unsafe { crate::util::slice_from_nullable_raw_parts(t.shape, t.ndim as usize) }
}
pub fn ndim(&self) -> usize {
self.data.dltensor.ndim as usize
}
pub fn numel(&self) -> usize {
unsafe { self.data.dltensor.numel() }
}
pub fn strides(&self) -> &[i64] {
let t = &self.data.dltensor;
if t.ndim == 0 {
return &[];
}
assert!(
!t.strides.is_null(),
"Tensor::strides of a DLTensor with null strides"
);
unsafe { std::slice::from_raw_parts(t.strides, t.ndim as usize) }
}
pub fn dtype(&self) -> DLDataType {
self.data.dltensor.dtype
}
pub fn device(&self) -> DLDevice {
self.data.dltensor.device
}
}
#[repr(C)]
struct TensorObjFromNDAlloc<TNDAlloc>
where
TNDAlloc: NDAllocator,
{
base_: TensorObj,
alloc: TNDAlloc,
}
unsafe impl<TNDAlloc: NDAllocator> ObjectCore for TensorObjFromNDAlloc<TNDAlloc> {
type Parent = TensorObj;
const TYPE_KEY: &'static str = TensorObj::TYPE_KEY;
const TYPE_DEPTH: i32 = TensorObj::TYPE_DEPTH;
fn type_index() -> i32 {
TensorObj::type_index()
}
unsafe fn object_header_mut(this: &mut Self) -> &mut tvm_ffi_sys::TVMFFIObject {
unsafe { TensorObj::object_header_mut(&mut this.base_) }
}
}
unsafe impl<TNDAlloc: NDAllocator> ObjectCoreWithExtraItems for TensorObjFromNDAlloc<TNDAlloc> {
type ExtraItem = i64;
fn extra_items_count(this: &Self) -> usize {
(this.base_.dltensor.ndim * 2) as usize
}
}
impl<TNDAlloc: NDAllocator> Drop for TensorObjFromNDAlloc<TNDAlloc> {
fn drop(&mut self) {
unsafe {
self.alloc.free_data(&self.base_.dltensor);
}
}
}
impl Tensor {
pub fn from_nd_alloc<TNDAlloc>(
alloc: TNDAlloc,
shape: &[i64],
dtype: DLDataType,
device: DLDevice,
) -> Self
where
TNDAlloc: NDAllocator,
{
let ndim = i32::try_from(shape.len()).expect("tensor ndim exceeds i32");
let dltensor = DLTensor {
data: std::ptr::null_mut(),
device,
ndim,
dtype,
shape: shape.as_ptr() as *mut i64,
strides: std::ptr::null_mut(),
byte_offset: 0,
};
unsafe { data_size(&dltensor) };
let tensor_obj = TensorObjFromNDAlloc {
base_: TensorObj {
object: Object::new(),
dltensor: DLTensor {
shape: std::ptr::null_mut(),
..dltensor
},
},
alloc,
};
unsafe {
let mut obj_arc = ObjectArc::new_with_extra_items(tensor_obj);
let object = ObjectArc::as_mut_ptr(&mut obj_arc);
let items = TensorObjFromNDAlloc::extra_items_ptr(object);
std::ptr::copy_nonoverlapping(shape.as_ptr(), items, shape.len());
let strides = items.add(shape.len());
Shape::fill_strides_from_shape(
shape,
std::slice::from_raw_parts_mut(strides, shape.len()),
);
let object = &mut *object;
object.base_.dltensor.shape = items;
object.base_.dltensor.strides = strides;
let dltensor_ptr = &object.base_.dltensor as *const DLTensor;
object.base_.dltensor.data = object.alloc.alloc_data(&*dltensor_ptr);
Self {
data: ObjectArc::from_raw(ObjectArc::into_raw(obj_arc) as *mut TensorObj),
}
}
}
pub fn from_env_alloc(shape: &[i64], dtype: DLDataType, device: DLDevice) -> Result<Self> {
let ndim = i32::try_from(shape.len()).expect("tensor ndim exceeds i32");
let mut prototype = DLTensor {
data: std::ptr::null_mut(),
device,
ndim,
dtype,
shape: shape.as_ptr() as *mut i64,
strides: std::ptr::null_mut(),
byte_offset: 0,
};
let mut out: TVMFFIObjectHandle = std::ptr::null_mut();
unsafe {
if TVMFFIEnvTensorAlloc(&mut prototype, &mut out) != 0 {
return Err(crate::error::Error::from_raised());
}
Ok(Self {
data: ObjectArc::from_raw(out as *const TensorObj),
})
}
}
pub fn from_slice<T: AsDLDataType>(slice: &[T], shape: &[i64]) -> Result<Self> {
if checked_numel(shape) != Some(slice.len()) {
crate::bail!(crate::error::VALUE_ERROR, "Slice length mismatch");
}
let dtype = T::DL_DATA_TYPE;
let device = DLDevice::new(DLDeviceType::kDLCPU, 0);
let tensor = Tensor::from_nd_alloc(CPUNDAlloc, shape, dtype, device);
unsafe { tensor.data_as_slice_mut::<T>()? }.copy_from_slice(slice);
Ok(tensor)
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct CPUNDAlloc;
unsafe impl NDAllocator for CPUNDAlloc {
const MIN_ALIGN: usize = 64;
unsafe fn alloc_data(&mut self, prototype: &DLTensor) -> *mut core::ffi::c_void {
let size = unsafe { data_size(prototype) };
if size == 0 {
return Self::MIN_ALIGN as *mut core::ffi::c_void;
}
let layout = std::alloc::Layout::from_size_align(size, Self::MIN_ALIGN).unwrap();
let ptr = unsafe { std::alloc::alloc(layout) };
if ptr.is_null() {
std::alloc::handle_alloc_error(layout);
}
ptr as *mut core::ffi::c_void
}
unsafe fn free_data(&mut self, tensor: &DLTensor) {
let size = unsafe { data_size(tensor) };
if size == 0 {
return;
}
let layout = std::alloc::Layout::from_size_align(size, Self::MIN_ALIGN).unwrap();
unsafe { std::alloc::dealloc(tensor.data as *mut u8, layout) };
}
}
#[derive(Clone, Copy, Debug)]
pub struct TensorView<'a> {
tensor: *const DLTensor,
_borrow: std::marker::PhantomData<&'a DLTensor>,
}
impl<'a> TensorView<'a> {
pub unsafe fn from_raw(tensor: *const DLTensor) -> Self {
assert!(!tensor.is_null(), "TensorView of a null DLTensor");
Self {
tensor,
_borrow: std::marker::PhantomData,
}
}
pub fn as_raw(&self) -> *const DLTensor {
self.tensor
}
fn dltensor(&self) -> &'a DLTensor {
unsafe { &*self.tensor }
}
pub fn data_ptr(&self) -> *mut core::ffi::c_void {
self.dltensor().data
}
pub fn device(&self) -> DLDevice {
self.dltensor().device
}
pub fn dtype(&self) -> DLDataType {
self.dltensor().dtype
}
pub fn ndim(&self) -> usize {
self.dltensor().ndim as usize
}
pub fn shape(&self) -> &'a [i64] {
let t = self.dltensor();
unsafe { crate::util::slice_from_nullable_raw_parts(t.shape, t.ndim as usize) }
}
pub fn numel(&self) -> usize {
unsafe { self.dltensor().numel() }
}
pub fn strides(&self) -> &'a [i64] {
let t = self.dltensor();
if t.ndim == 0 {
return &[];
}
assert!(
!t.strides.is_null(),
"TensorView::strides of a DLTensor with null strides"
);
unsafe { std::slice::from_raw_parts(t.strides, t.ndim as usize) }
}
pub fn is_contiguous(&self) -> bool {
if self.dltensor().strides.is_null() {
return true;
}
let shape = self.shape();
if shape.contains(&0) {
return true;
}
let mut expected_stride = 1;
for (size, stride) in shape.iter().zip(self.strides()).rev() {
if *size == 1 {
continue;
}
if *stride != expected_stride {
return false;
}
expected_stride *= size;
}
true
}
}
impl<'a> From<&'a Tensor> for TensorView<'a> {
fn from(tensor: &'a Tensor) -> Self {
Self {
tensor: &tensor.data.dltensor as *const DLTensor,
_borrow: std::marker::PhantomData,
}
}
}
impl<'a> TensorView<'a> {
fn from_cell(value: &TVMFFIAny) -> Option<Self> {
let tensor = if value.type_index == TypeIndex::kTVMFFIDLTensorPtr as i32 {
unsafe { value.data_union.v_ptr as *const DLTensor }
} else if value.type_index == TypeIndex::kTVMFFITensor as i32 {
unsafe { &(*(value.data_union.v_obj as *const TensorObj)).dltensor as *const DLTensor }
} else {
return None;
};
Some(Self {
tensor,
_borrow: std::marker::PhantomData,
})
}
fn mismatch(value: &TVMFFIAny, context: &str) -> crate::error::Error {
let message = format!(
"{context}Cannot convert from type `{}` to `DLTensor*`",
crate::object::type_key_of_index(value.type_index)
);
crate::error::Error::new(crate::error::TYPE_ERROR, &message)
}
}
impl<'a> TryFrom<AnyView<'a>> for TensorView<'a> {
type Error = crate::error::Error;
#[inline]
fn try_from(value: AnyView<'a>) -> Result<Self> {
let cell = value.as_raw_ffi_any();
Self::from_cell(cell).ok_or_else(|| Self::mismatch(cell, ""))
}
}
impl crate::type_traits::TypeSchema for TensorView<'_> {
fn type_schema() -> String {
crate::type_traits::type_schema("DLTensor*", &[])
}
}
impl crate::any::ArgTryFromAnyView for TensorView<'_> {
fn try_from_any_view(value: &AnyView, arg_index: usize) -> Result<Self> {
let cell = value.as_raw_ffi_any();
Self::from_cell(cell)
.ok_or_else(|| Self::mismatch(cell, &format!("Argument #{arg_index}: ")))
}
}
impl<'a> From<&TensorView<'a>> for AnyView<'a> {
fn from(view: &TensorView<'a>) -> Self {
let mut cell = TVMFFIAny::new();
cell.type_index = TypeIndex::kTVMFFIDLTensorPtr as i32;
cell.data_union.v_ptr = view.tensor as *mut core::ffi::c_void;
unsafe { AnyView::from_raw_ffi_any(cell) }
}
}
impl<'a> crate::function_internal::PackedCallArg for TensorView<'a> {
type Holder = Self;
#[inline(always)]
fn into_packed_arg(self) -> Self {
self
}
#[inline(always)]
fn as_packed_arg(holder: &Self::Holder) -> AnyView<'_> {
AnyView::from(holder)
}
}
impl<'a, 'b> crate::function_internal::PackedCallArg for &'b TensorView<'a> {
type Holder = Self;
#[inline(always)]
fn into_packed_arg(self) -> Self {
self
}
#[inline(always)]
fn as_packed_arg(holder: &Self::Holder) -> AnyView<'_> {
AnyView::from(*holder)
}
}