use num_complex::{Complex, Complex32, Complex64};
use num_traits::{One, Zero};
use std::any::Any;
use std::fmt::Debug;
use std::sync::Arc;
use crate::config::SliceConfig;
use tenferro_tensor_core::SliceSpec as CoreSliceSpec;
pub use tenferro_tensor_core::{DynRank, Rank, TensorLayout, TensorRank};
mod accessors;
mod shape_packing;
mod strided_view;
pub use strided_view::StridedSliceSpec;
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub enum MemoryKind {
Device,
PinnedHost,
UnpinnedHost,
Managed,
Other(String),
}
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub enum DeviceKind {
Cpu,
Gpu(GpuBackendKind),
Other(String),
}
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub enum GpuBackendKind {
Cuda,
WebGpu,
Rocm,
Other(String),
}
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub struct DeviceId {
pub kind: DeviceKind,
pub ordinal: usize,
}
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub struct Placement {
pub memory_kind: MemoryKind,
pub device: Option<DeviceId>,
}
#[derive(Clone)]
pub struct BufferHandle<T> {
id: u64,
len: usize,
_phantom: std::marker::PhantomData<T>,
}
impl<T> Debug for BufferHandle<T> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("BufferHandle")
.field("id", &self.id)
.finish()
}
}
impl<T> BufferHandle<T> {
pub fn new(id: u64) -> Self {
Self::new_with_len(id, 0)
}
pub fn new_with_len(id: u64, len: usize) -> Self {
Self {
id,
len,
_phantom: std::marker::PhantomData,
}
}
}
pub trait BackendBuffer<T>: Debug + Send + Sync + 'static {
fn backend_family(&self) -> &'static str;
fn len(&self) -> usize;
fn is_empty(&self) -> bool {
self.len() == 0
}
fn as_any(&self) -> &dyn Any;
}
impl<T: Send + Sync + 'static> BackendBuffer<T> for BufferHandle<T> {
fn backend_family(&self) -> &'static str {
"opaque"
}
fn len(&self) -> usize {
self.len
}
fn as_any(&self) -> &dyn Any {
self
}
}
#[derive(Clone, Debug)]
pub enum Buffer<T> {
Host(Vec<T>),
Backend(Arc<dyn BackendBuffer<T>>),
}
impl<T: 'static> Buffer<T> {
pub fn len(&self) -> usize {
match self {
Self::Host(data) => data.len(),
Self::Backend(buffer) => buffer.len(),
}
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
pub fn is_backend(&self) -> bool {
matches!(self, Self::Backend(_))
}
}
#[derive(Clone, Debug)]
pub struct TypedTensor<T, R: TensorRank = DynRank> {
buffer: Buffer<T>,
layout: TensorLayout<R>,
placement: Placement,
}
#[derive(Debug)]
pub enum TensorBufferRef<'a, T> {
Host(&'a [T]),
Backend(Arc<dyn BackendBuffer<T>>),
}
impl<T> Clone for TensorBufferRef<'_, T> {
fn clone(&self) -> Self {
match self {
Self::Host(data) => Self::Host(data),
Self::Backend(buffer) => Self::Backend(Arc::clone(buffer)),
}
}
}
impl<T: 'static> TensorBufferRef<'_, T> {
pub fn len(&self) -> usize {
match self {
Self::Host(data) => data.len(),
Self::Backend(buffer) => buffer.len(),
}
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
#[derive(Debug)]
pub enum TensorBufferRefMut<'a, T> {
Host(&'a mut [T]),
Backend(Arc<dyn BackendBuffer<T>>),
}
impl<T: 'static> TensorBufferRefMut<'_, T> {
pub fn len(&self) -> usize {
match self {
Self::Host(data) => data.len(),
Self::Backend(buffer) => buffer.len(),
}
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
#[derive(Clone, Debug)]
pub struct TypedTensorView<'a, T, R: TensorRank = DynRank> {
buffer: TensorBufferRef<'a, T>,
layout: TensorLayout<R>,
placement: Placement,
}
impl<'a, T: 'static> TypedTensorView<'a, T, DynRank> {
pub fn from_col_major(shape: &[usize], data: &'a [T]) -> crate::Result<Self> {
let layout = TensorLayout::<DynRank>::compact(shape.to_vec().into())
.map_err(|err| tensor_layout_error("TypedTensorView::from_col_major", err))?;
Self::from_buffer_ref(
layout.shape().to_vec(),
layout.strides().to_vec(),
layout.offset(),
TensorBufferRef::Host(data),
default_placement(),
"TypedTensorView::from_col_major",
)
}
pub fn from_slice(
shape: impl AsRef<[usize]>,
strides: impl AsRef<[isize]>,
offset: isize,
data: &'a [T],
) -> crate::Result<Self> {
Self::from_buffer_ref(
shape.as_ref().to_vec(),
strides.as_ref().to_vec(),
offset,
TensorBufferRef::Host(data),
default_placement(),
"TypedTensorView::from_slice",
)
}
}
impl<'a, T: 'static, R: TensorRank> TypedTensorView<'a, T, R> {
pub fn from_slice_ranked(
shape: impl Into<R::Shape>,
strides: impl Into<R::Strides>,
offset: isize,
data: &'a [T],
) -> crate::Result<Self> {
Self::from_buffer_ref(
shape,
strides,
offset,
TensorBufferRef::Host(data),
default_placement(),
"TypedTensorView::from_slice_ranked",
)
}
fn from_buffer_ref(
shape: impl Into<R::Shape>,
strides: impl Into<R::Strides>,
offset: isize,
buffer: TensorBufferRef<'a, T>,
placement: Placement,
op: &'static str,
) -> crate::Result<Self> {
let layout = TensorLayout::from_parts(shape.into(), strides.into(), offset, buffer.len())
.map_err(|err| tensor_layout_error(op, err))?;
Ok(Self {
buffer,
layout,
placement,
})
}
pub fn shape(&self) -> &[usize] {
self.layout.shape()
}
pub fn strides(&self) -> &[isize] {
self.layout.strides()
}
pub fn offset(&self) -> isize {
self.layout.offset()
}
pub fn host_storage(&self) -> crate::Result<&'a [T]> {
match &self.buffer {
TensorBufferRef::Host(data) => Ok(data),
TensorBufferRef::Backend(_) => Err(crate::Error::backend_failure(
"TypedTensorView::host_storage",
"backend buffers cannot expose host storage; download explicitly first",
)),
}
}
pub fn n_elements(&self) -> usize {
match checked_view_element_count(self.shape(), "TypedTensorView::n_elements") {
Ok(n) => n,
Err(err) => {
unreachable!("TypedTensorView layout shape is validated at construction: {err}")
}
}
}
pub fn layout(&self) -> &TensorLayout<R> {
&self.layout
}
pub fn placement(&self) -> &Placement {
&self.placement
}
#[doc(hidden)]
pub fn backend_buffer(&self) -> Option<&Arc<dyn BackendBuffer<T>>> {
match &self.buffer {
TensorBufferRef::Host(_) => None,
TensorBufferRef::Backend(buffer) => Some(buffer),
}
}
pub fn linear_offset(&self, indices: &[usize]) -> Option<usize> {
checked_view_offset(self.shape(), self.strides(), self.offset(), indices)
}
pub fn get(&self, indices: &[usize]) -> Option<&T> {
let offset = self.linear_offset(indices)?;
match &self.buffer {
TensorBufferRef::Host(data) => data.get(offset),
TensorBufferRef::Backend(_) => None,
}
}
pub fn as_slice(&self) -> crate::Result<&'a [T]> {
let data =
match &self.buffer {
TensorBufferRef::Host(data) => data,
TensorBufferRef::Backend(_) => return Err(crate::Error::backend_failure(
"TypedTensorView::as_slice",
"backend buffers cannot be inspected as host slices; download explicitly first",
)),
};
contiguous_layout_slice(self.layout(), data, "TypedTensorView::as_slice")
}
pub fn to_contiguous(&self) -> crate::Result<TypedTensor<T, R>>
where
T: Clone,
{
let op = "TypedTensorView::to_contiguous";
let data = materialize_view_buffer_col_major(
self.shape(),
self.strides(),
self.offset(),
&self.buffer,
op,
)?;
let shape = R::shape_from_vec(self.shape().to_vec().into())
.map_err(|err| tensor_layout_error(op, err))?;
TypedTensor::from_buffer_col_major(shape, Buffer::Host(data), self.placement.clone())
}
pub fn transpose_view(&self, axes: impl AsRef<[usize]>) -> crate::Result<Self> {
let layout = self
.layout
.transpose_view(axes)
.map_err(|err| tensor_layout_error("TypedTensorView::transpose_view", err))?;
Ok(Self {
buffer: self.buffer.clone(),
layout,
placement: self.placement.clone(),
})
}
pub fn try_slice(&self, slices: &[StridedSliceSpec]) -> crate::Result<Self> {
let specs = core_slice_specs(slices, self.shape(), "TypedTensorView::try_slice")?;
let layout = self
.layout
.slice_view(specs, self.buffer.len())
.map_err(|err| tensor_layout_error("TypedTensorView::try_slice", err))?;
Ok(Self {
buffer: self.buffer.clone(),
layout,
placement: self.placement.clone(),
})
}
pub fn try_slice_axis(&self, axis: usize, slice: StridedSliceSpec) -> crate::Result<Self> {
let slices = slice_axis_specs(
self.shape().len(),
axis,
slice,
"TypedTensorView::try_slice_axis",
)?;
self.try_slice(&slices)
}
pub fn try_reshape(&self, shape: &[usize]) -> crate::Result<TypedTensorView<'a, T, DynRank>> {
let layout = reshape_layout_dyn(
&self.layout,
shape,
self.buffer.len(),
"TypedTensorView::try_reshape",
)?;
Ok(TypedTensorView {
buffer: self.buffer.clone(),
layout,
placement: self.placement.clone(),
})
}
}
#[derive(Debug)]
pub struct TypedTensorViewMut<'a, T, R: TensorRank = DynRank> {
buffer: TensorBufferRefMut<'a, T>,
layout: TensorLayout<R>,
placement: Placement,
}
impl<'a, T: 'static> TypedTensorViewMut<'a, T, DynRank> {
pub fn from_col_major(shape: &[usize], data: &'a mut [T]) -> crate::Result<Self> {
let layout = TensorLayout::<DynRank>::compact(shape.to_vec().into())
.map_err(|err| tensor_layout_error("TypedTensorViewMut::from_col_major", err))?;
Self::from_buffer_ref_mut(
layout.shape().to_vec(),
layout.strides().to_vec(),
layout.offset(),
TensorBufferRefMut::Host(data),
default_placement(),
"TypedTensorViewMut::from_col_major",
)
}
pub fn from_slice(
shape: impl AsRef<[usize]>,
strides: impl AsRef<[isize]>,
offset: isize,
data: &'a mut [T],
) -> crate::Result<Self> {
Self::from_buffer_ref_mut(
shape.as_ref().to_vec(),
strides.as_ref().to_vec(),
offset,
TensorBufferRefMut::Host(data),
default_placement(),
"TypedTensorViewMut::from_slice",
)
}
}
impl<'a, T: 'static, R: TensorRank> TypedTensorViewMut<'a, T, R> {
pub fn from_slice_ranked(
shape: impl Into<R::Shape>,
strides: impl Into<R::Strides>,
offset: isize,
data: &'a mut [T],
) -> crate::Result<Self> {
Self::from_buffer_ref_mut(
shape,
strides,
offset,
TensorBufferRefMut::Host(data),
default_placement(),
"TypedTensorViewMut::from_slice_ranked",
)
}
fn from_buffer_ref_mut(
shape: impl Into<R::Shape>,
strides: impl Into<R::Strides>,
offset: isize,
buffer: TensorBufferRefMut<'a, T>,
placement: Placement,
op: &'static str,
) -> crate::Result<Self> {
let layout = TensorLayout::from_parts(shape.into(), strides.into(), offset, buffer.len())
.map_err(|err| tensor_layout_error(op, err))?;
layout
.validate_mutable_no_overlap()
.map_err(|err| tensor_layout_error(op, err))?;
Ok(Self {
buffer,
layout,
placement,
})
}
pub fn shape(&self) -> &[usize] {
self.layout.shape()
}
pub fn strides(&self) -> &[isize] {
self.layout.strides()
}
pub fn offset(&self) -> isize {
self.layout.offset()
}
pub fn host_storage(&self) -> crate::Result<&[T]> {
match &self.buffer {
TensorBufferRefMut::Host(data) => Ok(data),
TensorBufferRefMut::Backend(_) => Err(crate::Error::backend_failure(
"TypedTensorViewMut::host_storage",
"backend buffers cannot expose host storage; download explicitly first",
)),
}
}
pub fn host_storage_mut(&mut self) -> crate::Result<&mut [T]> {
match &mut self.buffer {
TensorBufferRefMut::Host(data) => Ok(data),
TensorBufferRefMut::Backend(_) => Err(crate::Error::backend_failure(
"TypedTensorViewMut::host_storage_mut",
"backend buffers cannot expose mutable host storage; download explicitly first",
)),
}
}
pub fn n_elements(&self) -> usize {
match checked_view_element_count(self.shape(), "TypedTensorViewMut::n_elements") {
Ok(n) => n,
Err(err) => {
unreachable!("TypedTensorViewMut layout shape is validated at construction: {err}")
}
}
}
pub fn layout(&self) -> &TensorLayout<R> {
&self.layout
}
pub fn placement(&self) -> &Placement {
&self.placement
}
#[doc(hidden)]
pub fn backend_buffer(&self) -> Option<&Arc<dyn BackendBuffer<T>>> {
match &self.buffer {
TensorBufferRefMut::Host(_) => None,
TensorBufferRefMut::Backend(buffer) => Some(buffer),
}
}
pub fn linear_offset(&self, indices: &[usize]) -> Option<usize> {
checked_view_offset(self.shape(), self.strides(), self.offset(), indices)
}
pub fn get(&self, indices: &[usize]) -> Option<&T> {
let offset = self.linear_offset(indices)?;
match &self.buffer {
TensorBufferRefMut::Host(data) => data.get(offset),
TensorBufferRefMut::Backend(_) => None,
}
}
pub fn get_mut(&mut self, indices: &[usize]) -> Option<&mut T> {
let offset = self.linear_offset(indices)?;
match &mut self.buffer {
TensorBufferRefMut::Host(data) => data.get_mut(offset),
TensorBufferRefMut::Backend(_) => None,
}
}
pub fn copy_from_contiguous(&mut self, src: &TypedTensor<T, R>) -> crate::Result<()>
where
T: Clone,
{
let op = "TypedTensorViewMut::copy_from_contiguous";
if self.shape() != src.shape() {
return Err(crate::Error::InvalidConfig {
op,
message: format!(
"shape mismatch: destination {:?} does not match source {:?}",
self.shape(),
src.shape()
),
});
}
let src_data = match &src.buffer {
Buffer::Host(data) => contiguous_layout_slice(src.layout(), data, op)?,
Buffer::Backend(_) => {
return Err(crate::Error::backend_failure(
op,
"source backend buffer cannot be copied through host memory; download explicitly first",
))
}
};
let shape = self.shape().to_vec();
let strides = self.strides().to_vec();
let offset = self.offset();
let dst_data = match &mut self.buffer {
TensorBufferRefMut::Host(data) => data,
TensorBufferRefMut::Backend(_) => {
return Err(crate::Error::backend_failure(
op,
"destination backend buffer cannot be updated through host memory; download explicitly first",
))
}
};
let mut src_iter = src_data.iter();
for_each_layout_offset_col_major(&shape, &strides, offset, op, |offset| {
let value = src_iter.next().ok_or_else(|| crate::Error::InvalidConfig {
op,
message: "source tensor ended before destination view".to_string(),
})?;
let dst = dst_data
.get_mut(offset)
.ok_or_else(|| crate::Error::InvalidConfig {
op,
message: "destination view offset is outside host buffer".to_string(),
})?;
*dst = value.clone();
Ok(())
})?;
if src_iter.next().is_some() {
return Err(crate::Error::InvalidConfig {
op,
message: "source tensor has elements remaining after destination copy".to_string(),
});
}
Ok(())
}
pub fn as_read_only(&self) -> TypedTensorView<'_, T, R> {
let buffer = match &self.buffer {
TensorBufferRefMut::Host(data) => TensorBufferRef::Host(data),
TensorBufferRefMut::Backend(buffer) => TensorBufferRef::Backend(Arc::clone(buffer)),
};
TypedTensorView {
buffer,
layout: self.layout.clone(),
placement: self.placement.clone(),
}
}
pub fn into_read_only(self) -> TypedTensorView<'a, T, R> {
let buffer = match self.buffer {
TensorBufferRefMut::Host(data) => TensorBufferRef::Host(data),
TensorBufferRefMut::Backend(buffer) => TensorBufferRef::Backend(buffer),
};
TypedTensorView {
buffer,
layout: self.layout,
placement: self.placement,
}
}
pub fn transpose_view(
self,
axes: impl AsRef<[usize]>,
) -> crate::Result<TypedTensorViewMut<'a, T, R>> {
let Self {
buffer,
layout,
placement,
} = self;
let layout = layout
.transpose_view(axes)
.map_err(|err| tensor_layout_error("TypedTensorViewMut::transpose_view", err))?;
layout
.validate_mutable_no_overlap()
.map_err(|err| tensor_layout_error("TypedTensorViewMut::transpose_view", err))?;
match buffer {
TensorBufferRefMut::Host(data) => Ok(TypedTensorViewMut {
buffer: TensorBufferRefMut::Host(data),
layout,
placement,
}),
TensorBufferRefMut::Backend(buffer) => Ok(TypedTensorViewMut {
buffer: TensorBufferRefMut::Backend(buffer),
layout,
placement,
}),
}
}
pub fn try_slice(
&mut self,
slices: &[StridedSliceSpec],
) -> crate::Result<TypedTensorViewMut<'_, T, R>> {
let specs = core_slice_specs(slices, self.shape(), "TypedTensorViewMut::try_slice")?;
let layout = self
.layout
.slice_view(specs, self.buffer.len())
.map_err(|err| tensor_layout_error("TypedTensorViewMut::try_slice", err))?;
layout
.validate_mutable_no_overlap()
.map_err(|err| tensor_layout_error("TypedTensorViewMut::try_slice", err))?;
let placement = self.placement.clone();
match &mut self.buffer {
TensorBufferRefMut::Host(data) => Ok(TypedTensorViewMut {
buffer: TensorBufferRefMut::Host(data),
layout,
placement,
}),
TensorBufferRefMut::Backend(buffer) => Ok(TypedTensorViewMut {
buffer: TensorBufferRefMut::Backend(Arc::clone(buffer)),
layout,
placement,
}),
}
}
pub fn try_slice_axis(
&mut self,
axis: usize,
slice: StridedSliceSpec,
) -> crate::Result<TypedTensorViewMut<'_, T, R>> {
let slices = slice_axis_specs(
self.shape().len(),
axis,
slice,
"TypedTensorViewMut::try_slice_axis",
)?;
self.try_slice(&slices)
}
pub fn try_multi_slice_mut(
&mut self,
first: &[StridedSliceSpec],
second: &[StridedSliceSpec],
) -> Option<(TypedTensorViewMut<'_, T, R>, TypedTensorViewMut<'_, T, R>)> {
let first_specs = core_slice_specs(
first,
self.shape(),
"TypedTensorViewMut::try_multi_slice_mut",
)
.ok()?;
let second_specs = core_slice_specs(
second,
self.shape(),
"TypedTensorViewMut::try_multi_slice_mut",
)
.ok()?;
let buffer_len = self.buffer.len();
let first_layout = self.layout.slice_view(first_specs, buffer_len).ok()?;
let second_layout = self.layout.slice_view(second_specs, buffer_len).ok()?;
first_layout.validate_mutable_no_overlap().ok()?;
second_layout.validate_mutable_no_overlap().ok()?;
match (
reachable_layout_span(
first_layout.shape(),
first_layout.strides(),
first_layout.offset(),
)
.ok()?,
reachable_layout_span(
second_layout.shape(),
second_layout.strides(),
second_layout.offset(),
)
.ok()?,
) {
(Some(first_span), Some(second_span)) => {
let first_offset = adjusted_view_offset(first_layout.offset(), first_span.0)?;
let second_offset = adjusted_view_offset(second_layout.offset(), second_span.0)?;
let (first_data, second_data) = match &mut self.buffer {
TensorBufferRefMut::Host(data) => {
split_two_mut_ranges(data, first_span, second_span)?
}
TensorBufferRefMut::Backend(_) => return None,
};
let first_view = view_mut_from_layout_and_slice(
&first_layout,
first_offset,
first_data,
self.placement.clone(),
)
.ok()?;
let second_view = view_mut_from_layout_and_slice(
&second_layout,
second_offset,
second_data,
self.placement.clone(),
)
.ok()?;
Some((first_view, second_view))
}
(None, Some(second_span)) => {
let second_offset = adjusted_view_offset(second_layout.offset(), second_span.0)?;
let (_, after_start) = match &mut self.buffer {
TensorBufferRefMut::Host(data) => data.split_at_mut(second_span.0),
TensorBufferRefMut::Backend(_) => return None,
};
let (second_data, _) = after_start.split_at_mut(second_span.1 - second_span.0 + 1);
let first_view = view_mut_from_layout_and_slice(
&first_layout,
0,
&mut [],
self.placement.clone(),
)
.ok()?;
let second_view = view_mut_from_layout_and_slice(
&second_layout,
second_offset,
second_data,
self.placement.clone(),
)
.ok()?;
Some((first_view, second_view))
}
(Some(first_span), None) => {
let first_offset = adjusted_view_offset(first_layout.offset(), first_span.0)?;
let (_, after_start) = match &mut self.buffer {
TensorBufferRefMut::Host(data) => data.split_at_mut(first_span.0),
TensorBufferRefMut::Backend(_) => return None,
};
let (first_data, _) = after_start.split_at_mut(first_span.1 - first_span.0 + 1);
let first_view = view_mut_from_layout_and_slice(
&first_layout,
first_offset,
first_data,
self.placement.clone(),
)
.ok()?;
let second_view = view_mut_from_layout_and_slice(
&second_layout,
0,
&mut [],
self.placement.clone(),
)
.ok()?;
Some((first_view, second_view))
}
(None, None) => {
let first_view = view_mut_from_layout_and_slice(
&first_layout,
0,
&mut [],
self.placement.clone(),
)
.ok()?;
let second_view = view_mut_from_layout_and_slice(
&second_layout,
0,
&mut [],
self.placement.clone(),
)
.ok()?;
Some((first_view, second_view))
}
}
}
pub fn try_reshape(
&mut self,
shape: &[usize],
) -> crate::Result<TypedTensorViewMut<'_, T, DynRank>> {
let layout = reshape_layout_dyn(
&self.layout,
shape,
self.buffer.len(),
"TypedTensorViewMut::try_reshape",
)?;
layout
.validate_mutable_no_overlap()
.map_err(|err| tensor_layout_error("TypedTensorViewMut::try_reshape", err))?;
let placement = self.placement.clone();
match &mut self.buffer {
TensorBufferRefMut::Host(data) => Ok(TypedTensorViewMut {
buffer: TensorBufferRefMut::Host(data),
layout,
placement,
}),
TensorBufferRefMut::Backend(buffer) => Ok(TypedTensorViewMut {
buffer: TensorBufferRefMut::Backend(Arc::clone(buffer)),
layout,
placement,
}),
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum DType {
F32,
F64,
I32,
I64,
Bool,
C32,
C64,
}
pub trait TensorScalar: Copy + Clone + Send + Sync + 'static + private::Sealed {
type Real: TensorScalar;
fn dtype() -> DType;
fn into_tensor(shape: Vec<usize>, data: Vec<Self>) -> crate::Result<Tensor>;
fn tensor_read(tensor: &TypedTensor<Self>) -> TensorRead<'_>;
fn as_slice(tensor: &Tensor) -> crate::Result<&[Self]>;
fn as_slice_mut(tensor: &mut Tensor) -> crate::Result<&mut [Self]>;
fn into_typed(tensor: Tensor) -> crate::Result<TypedTensor<Self>>;
}
mod private {
pub trait Sealed {}
impl Sealed for f64 {}
impl Sealed for f32 {}
impl Sealed for i32 {}
impl Sealed for i64 {}
impl Sealed for bool {}
impl Sealed for num_complex::Complex64 {}
impl Sealed for num_complex::Complex32 {}
}
macro_rules! impl_tensor_scalar {
($ty:ty, $real:ty, $dtype:ident, $variant:ident) => {
impl TensorScalar for $ty {
type Real = $real;
fn dtype() -> DType {
DType::$dtype
}
fn into_tensor(shape: Vec<usize>, data: Vec<Self>) -> crate::Result<Tensor> {
TypedTensor::from_vec_col_major(shape, data).map(Tensor::$variant)
}
fn tensor_read(tensor: &TypedTensor<Self>) -> TensorRead<'_> {
TensorRead::from_view(TensorView::$variant(tensor.as_view()))
}
fn as_slice(tensor: &Tensor) -> crate::Result<&[Self]> {
let actual = tensor.dtype();
match tensor {
Tensor::$variant(t) => t.host_data(),
_ => Err(crate::Error::DTypeMismatch {
op: "Tensor::as_slice",
lhs: Self::dtype(),
rhs: actual,
}),
}
}
fn as_slice_mut(tensor: &mut Tensor) -> crate::Result<&mut [Self]> {
let actual = tensor.dtype();
match tensor {
Tensor::$variant(t) => t.host_data_mut(),
_ => Err(crate::Error::DTypeMismatch {
op: "Tensor::as_slice_mut",
lhs: Self::dtype(),
rhs: actual,
}),
}
}
fn into_typed(tensor: Tensor) -> crate::Result<TypedTensor<Self>> {
let actual = tensor.dtype();
match tensor {
Tensor::$variant(inner) => Ok(inner),
_ => Err(crate::Error::DTypeMismatch {
op: "TensorScalar::into_typed",
lhs: Self::dtype(),
rhs: actual,
}),
}
}
}
};
}
impl_tensor_scalar!(f64, f64, F64, F64);
impl_tensor_scalar!(f32, f32, F32, F32);
impl_tensor_scalar!(i64, i64, I64, I64);
impl_tensor_scalar!(i32, i32, I32, I32);
impl_tensor_scalar!(bool, bool, Bool, Bool);
impl_tensor_scalar!(Complex64, f64, C64, C64);
impl_tensor_scalar!(Complex32, f32, C32, C32);
#[derive(Clone, Debug)]
pub enum Tensor {
F32(TypedTensor<f32>),
F64(TypedTensor<f64>),
I32(TypedTensor<i32>),
I64(TypedTensor<i64>),
Bool(TypedTensor<bool>),
C32(TypedTensor<Complex<f32>>),
C64(TypedTensor<Complex<f64>>),
}
#[derive(Clone, Debug)]
pub enum TensorView<'a> {
F32(TypedTensorView<'a, f32>),
F64(TypedTensorView<'a, f64>),
I32(TypedTensorView<'a, i32>),
I64(TypedTensorView<'a, i64>),
Bool(TypedTensorView<'a, bool>),
C32(TypedTensorView<'a, Complex<f32>>),
C64(TypedTensorView<'a, Complex<f64>>),
}
#[allow(clippy::large_enum_variant)]
#[derive(Clone, Debug)]
pub enum TensorRead<'a> {
Tensor(&'a Tensor),
View(TensorView<'a>),
}
#[derive(Clone, Debug)]
pub struct TensorOwnedView {
base: Arc<Tensor>,
layout: TensorLayout<DynRank>,
}
#[derive(Clone, Debug)]
pub enum TensorValue {
Tensor(Arc<Tensor>),
View(TensorOwnedView),
}
impl TensorOwnedView {
pub fn from_tensor(base: Arc<Tensor>) -> Self {
let layout = tensor_layout(base.as_ref());
Self { base, layout }
}
pub fn from_parts(
base: Arc<Tensor>,
shape: Vec<usize>,
strides: Vec<isize>,
offset: isize,
) -> crate::Result<Self> {
let layout = TensorLayout::from_parts(
shape.into(),
strides.into(),
offset,
tensor_buffer_len(&base),
)
.map_err(|err| tensor_layout_error("TensorOwnedView::from_parts", err))?;
Ok(Self { base, layout })
}
pub fn dtype(&self) -> DType {
self.base.dtype()
}
pub fn shape(&self) -> &[usize] {
self.layout.shape()
}
pub fn strides(&self) -> &[isize] {
self.layout.strides()
}
pub fn offset(&self) -> isize {
self.layout.offset()
}
pub fn tensor_view(&self) -> TensorView<'_> {
tensor_view_with_layout(self.base.as_ref(), self.layout.clone())
}
pub fn tensor_read(&self) -> TensorRead<'_> {
TensorRead::from_view(self.tensor_view())
}
pub fn to_tensor(&self) -> crate::Result<Tensor> {
self.tensor_view().to_tensor()
}
pub fn transpose_view(&self, axes: impl AsRef<[usize]>) -> crate::Result<Self> {
let layout = self
.layout
.transpose_view(axes)
.map_err(|err| tensor_layout_error("TensorOwnedView::transpose_view", err))?;
Ok(Self {
base: Arc::clone(&self.base),
layout,
})
}
pub fn reshape_view(&self, shape: &[usize]) -> crate::Result<Self> {
let layout = reshape_layout_dyn(
&self.layout,
shape,
tensor_buffer_len(&self.base),
"TensorOwnedView::reshape_view",
)?;
Ok(Self {
base: Arc::clone(&self.base),
layout,
})
}
pub fn slice_view(&self, config: &SliceConfig) -> crate::Result<Self> {
let op = "TensorOwnedView::slice_view";
if config.starts.len() != self.shape().len() {
return Err(crate::Error::RankMismatch {
op,
expected: self.shape().len(),
actual: config.starts.len(),
});
}
if config.limits.len() != self.shape().len() {
return Err(crate::Error::RankMismatch {
op,
expected: self.shape().len(),
actual: config.limits.len(),
});
}
if config.strides.len() != self.shape().len() {
return Err(crate::Error::RankMismatch {
op,
expected: self.shape().len(),
actual: config.strides.len(),
});
}
let mut slices = Vec::with_capacity(self.shape().len());
for ((&start, &limit), &stride) in config
.starts
.iter()
.zip(config.limits.iter())
.zip(config.strides.iter())
{
let start = isize::try_from(start).map_err(|_| crate::Error::InvalidConfig {
op,
message: format!("slice start {start} does not fit in isize"),
})?;
let limit = isize::try_from(limit).map_err(|_| crate::Error::InvalidConfig {
op,
message: format!("slice limit {limit} does not fit in isize"),
})?;
let stride = isize::try_from(stride).map_err(|_| crate::Error::InvalidConfig {
op,
message: format!("slice stride {stride} does not fit in isize"),
})?;
slices.push(StridedSliceSpec::new(start, Some(limit), stride));
}
let specs = core_slice_specs(&slices, self.shape(), op)?;
let layout = self
.layout
.slice_view(&specs, tensor_buffer_len(&self.base))
.map_err(|err| tensor_layout_error(op, err))?;
Ok(Self {
base: Arc::clone(&self.base),
layout,
})
}
pub fn broadcast_in_dim_view(&self, shape: &[usize], dims: &[usize]) -> crate::Result<Self> {
let layout = self
.layout
.broadcast_in_dim_view::<DynRank>(
shape.to_vec().into(),
dims,
tensor_buffer_len(&self.base),
)
.map_err(|err| tensor_layout_error("TensorOwnedView::broadcast_in_dim_view", err))?;
Ok(Self {
base: Arc::clone(&self.base),
layout,
})
}
}
impl TensorValue {
pub fn from_tensor(tensor: Tensor) -> Self {
Self::Tensor(Arc::new(tensor))
}
pub fn from_tensor_arc(tensor: Arc<Tensor>) -> Self {
Self::Tensor(tensor)
}
pub fn as_tensor_arc(&self) -> Option<&Arc<Tensor>> {
match self {
Self::Tensor(tensor) => Some(tensor),
Self::View(_) => None,
}
}
pub fn dtype(&self) -> DType {
match self {
Self::Tensor(tensor) => tensor.dtype(),
Self::View(view) => view.dtype(),
}
}
pub fn shape(&self) -> &[usize] {
match self {
Self::Tensor(tensor) => tensor.shape(),
Self::View(view) => view.shape(),
}
}
pub fn tensor_read(&self) -> TensorRead<'_> {
match self {
Self::Tensor(tensor) => TensorRead::from_tensor(tensor.as_ref()),
Self::View(view) => view.tensor_read(),
}
}
pub fn to_tensor(&self) -> crate::Result<Tensor> {
match self {
Self::Tensor(tensor) => Ok(tensor.as_ref().clone()),
Self::View(view) => view.to_tensor(),
}
}
pub fn transpose_view(&self, axes: impl AsRef<[usize]>) -> crate::Result<Self> {
match self {
Self::Tensor(tensor) => TensorOwnedView::from_tensor(Arc::clone(tensor))
.transpose_view(axes)
.map(Self::View),
Self::View(view) => view.transpose_view(axes).map(Self::View),
}
}
pub fn reshape_view(&self, shape: &[usize]) -> crate::Result<Self> {
match self {
Self::Tensor(tensor) => TensorOwnedView::from_tensor(Arc::clone(tensor))
.reshape_view(shape)
.map(Self::View),
Self::View(view) => view.reshape_view(shape).map(Self::View),
}
}
pub fn slice_view(&self, config: &SliceConfig) -> crate::Result<Self> {
match self {
Self::Tensor(tensor) => TensorOwnedView::from_tensor(Arc::clone(tensor))
.slice_view(config)
.map(Self::View),
Self::View(view) => view.slice_view(config).map(Self::View),
}
}
pub fn broadcast_in_dim_view(&self, shape: &[usize], dims: &[usize]) -> crate::Result<Self> {
match self {
Self::Tensor(tensor) => TensorOwnedView::from_tensor(Arc::clone(tensor))
.broadcast_in_dim_view(shape, dims)
.map(Self::View),
Self::View(view) => view.broadcast_in_dim_view(shape, dims).map(Self::View),
}
}
}
fn tensor_layout(tensor: &Tensor) -> TensorLayout<DynRank> {
match tensor {
Tensor::F32(tensor) => tensor.layout.clone(),
Tensor::F64(tensor) => tensor.layout.clone(),
Tensor::I32(tensor) => tensor.layout.clone(),
Tensor::I64(tensor) => tensor.layout.clone(),
Tensor::Bool(tensor) => tensor.layout.clone(),
Tensor::C32(tensor) => tensor.layout.clone(),
Tensor::C64(tensor) => tensor.layout.clone(),
}
}
fn tensor_buffer_len(tensor: &Tensor) -> usize {
match tensor {
Tensor::F32(tensor) => buffer_len(&tensor.buffer),
Tensor::F64(tensor) => buffer_len(&tensor.buffer),
Tensor::I32(tensor) => buffer_len(&tensor.buffer),
Tensor::I64(tensor) => buffer_len(&tensor.buffer),
Tensor::Bool(tensor) => buffer_len(&tensor.buffer),
Tensor::C32(tensor) => buffer_len(&tensor.buffer),
Tensor::C64(tensor) => buffer_len(&tensor.buffer),
}
}
fn buffer_len<T: 'static>(buffer: &Buffer<T>) -> usize {
match buffer {
Buffer::Host(data) => data.len(),
Buffer::Backend(buffer) => buffer.len(),
}
}
fn tensor_view_with_layout(tensor: &Tensor, layout: TensorLayout<DynRank>) -> TensorView<'_> {
match tensor {
Tensor::F32(tensor) => TensorView::F32(typed_view_with_layout(tensor, layout)),
Tensor::F64(tensor) => TensorView::F64(typed_view_with_layout(tensor, layout)),
Tensor::I32(tensor) => TensorView::I32(typed_view_with_layout(tensor, layout)),
Tensor::I64(tensor) => TensorView::I64(typed_view_with_layout(tensor, layout)),
Tensor::Bool(tensor) => TensorView::Bool(typed_view_with_layout(tensor, layout)),
Tensor::C32(tensor) => TensorView::C32(typed_view_with_layout(tensor, layout)),
Tensor::C64(tensor) => TensorView::C64(typed_view_with_layout(tensor, layout)),
}
}
fn typed_view_with_layout<T: 'static>(
tensor: &TypedTensor<T>,
layout: TensorLayout<DynRank>,
) -> TypedTensorView<'_, T> {
let buffer = match &tensor.buffer {
Buffer::Host(data) => TensorBufferRef::Host(data),
Buffer::Backend(buffer) => TensorBufferRef::Backend(Arc::clone(buffer)),
};
TypedTensorView {
buffer,
layout,
placement: tensor.placement.clone(),
}
}
impl From<TypedTensor<f64>> for Tensor {
fn from(t: TypedTensor<f64>) -> Self {
Tensor::F64(t)
}
}
impl From<TypedTensor<f32>> for Tensor {
fn from(t: TypedTensor<f32>) -> Self {
Tensor::F32(t)
}
}
impl From<TypedTensor<i64>> for Tensor {
fn from(t: TypedTensor<i64>) -> Self {
Tensor::I64(t)
}
}
impl From<TypedTensor<i32>> for Tensor {
fn from(t: TypedTensor<i32>) -> Self {
Tensor::I32(t)
}
}
impl From<TypedTensor<bool>> for Tensor {
fn from(t: TypedTensor<bool>) -> Self {
Tensor::Bool(t)
}
}
impl From<TypedTensor<Complex<f64>>> for Tensor {
fn from(t: TypedTensor<Complex<f64>>) -> Self {
Tensor::C64(t)
}
}
impl From<TypedTensor<Complex<f32>>> for Tensor {
fn from(t: TypedTensor<Complex<f32>>) -> Self {
Tensor::C32(t)
}
}
impl<'a> TensorView<'a> {
pub fn f32(shape: &'a [usize], data: &'a [f32]) -> crate::Result<Self> {
Ok(Self::F32(TypedTensorView::from_col_major(shape, data)?))
}
pub fn f64(shape: &'a [usize], data: &'a [f64]) -> crate::Result<Self> {
Ok(Self::F64(TypedTensorView::from_col_major(shape, data)?))
}
pub fn i64(shape: &'a [usize], data: &'a [i64]) -> crate::Result<Self> {
Ok(Self::I64(TypedTensorView::from_col_major(shape, data)?))
}
pub fn i32(shape: &'a [usize], data: &'a [i32]) -> crate::Result<Self> {
Ok(Self::I32(TypedTensorView::from_col_major(shape, data)?))
}
pub fn bool(shape: &'a [usize], data: &'a [bool]) -> crate::Result<Self> {
Ok(Self::Bool(TypedTensorView::from_col_major(shape, data)?))
}
pub fn c32(shape: &'a [usize], data: &'a [Complex32]) -> crate::Result<Self> {
Ok(Self::C32(TypedTensorView::from_col_major(shape, data)?))
}
pub fn c64(shape: &'a [usize], data: &'a [Complex64]) -> crate::Result<Self> {
Ok(Self::C64(TypedTensorView::from_col_major(shape, data)?))
}
pub fn dtype(&self) -> DType {
match self {
Self::F32(_) => DType::F32,
Self::F64(_) => DType::F64,
Self::I32(_) => DType::I32,
Self::I64(_) => DType::I64,
Self::Bool(_) => DType::Bool,
Self::C32(_) => DType::C32,
Self::C64(_) => DType::C64,
}
}
pub fn shape(&self) -> &[usize] {
match self {
Self::F32(t) => t.shape(),
Self::F64(t) => t.shape(),
Self::I32(t) => t.shape(),
Self::I64(t) => t.shape(),
Self::Bool(t) => t.shape(),
Self::C32(t) => t.shape(),
Self::C64(t) => t.shape(),
}
}
pub fn to_tensor(&self) -> crate::Result<Tensor> {
match self {
Self::F32(t) => {
materialize_typed_view_col_major(t, "TensorView::to_tensor").map(Tensor::F32)
}
Self::F64(t) => {
materialize_typed_view_col_major(t, "TensorView::to_tensor").map(Tensor::F64)
}
Self::I32(t) => {
materialize_typed_view_col_major(t, "TensorView::to_tensor").map(Tensor::I32)
}
Self::I64(t) => {
materialize_typed_view_col_major(t, "TensorView::to_tensor").map(Tensor::I64)
}
Self::Bool(t) => {
materialize_typed_view_col_major(t, "TensorView::to_tensor").map(Tensor::Bool)
}
Self::C32(t) => {
materialize_typed_view_col_major(t, "TensorView::to_tensor").map(Tensor::C32)
}
Self::C64(t) => {
materialize_typed_view_col_major(t, "TensorView::to_tensor").map(Tensor::C64)
}
}
}
}
impl<'a> TensorRead<'a> {
pub fn from_tensor(tensor: &'a Tensor) -> Self {
Self::Tensor(tensor)
}
pub fn from_view(view: TensorView<'a>) -> Self {
Self::View(view)
}
pub fn dtype(&self) -> DType {
match self {
Self::Tensor(tensor) => tensor.dtype(),
Self::View(view) => view.dtype(),
}
}
pub fn shape(&self) -> &[usize] {
match self {
Self::Tensor(tensor) => tensor.shape(),
Self::View(view) => view.shape(),
}
}
pub fn as_tensor(&self) -> Option<&'a Tensor> {
match self {
Self::Tensor(tensor) => Some(*tensor),
Self::View(_) => None,
}
}
pub fn to_tensor(&self) -> crate::Result<Tensor> {
match self {
Self::Tensor(tensor) => Ok((*tensor).clone()),
Self::View(view) => view.to_tensor(),
}
}
}
pub fn col_major_strides(shape: &[usize]) -> crate::Result<Vec<isize>> {
let mut strides = Vec::with_capacity(shape.len());
let mut stride = 1isize;
for &extent in shape {
strides.push(stride);
let extent = isize::try_from(extent).map_err(|_| crate::Error::InvalidConfig {
op: "col_major_strides",
message: format!("shape extent {extent} does not fit in isize"),
})?;
stride = stride
.checked_mul(extent)
.ok_or_else(|| crate::Error::InvalidConfig {
op: "col_major_strides",
message: format!("column-major stride overflows for shape {shape:?}"),
})?;
}
Ok(strides)
}
fn try_linear_offset_for_shape(
shape: &[usize],
indices: &[usize],
op: &'static str,
) -> crate::Result<usize> {
if indices.len() != shape.len() {
return Err(crate::Error::RankMismatch {
op,
expected: shape.len(),
actual: indices.len(),
});
}
let mut offset = 0usize;
let mut stride = 1usize;
for (axis, (&idx, &extent)) in indices.iter().zip(shape).enumerate() {
if idx >= extent {
return Err(crate::Error::InvalidConfig {
op,
message: format!("index {idx} out of bounds for axis {axis} extent {extent}"),
});
}
offset = offset
.checked_add(
idx.checked_mul(stride)
.ok_or_else(|| crate::Error::InvalidConfig {
op,
message: "linear offset multiply overflows".to_string(),
})?,
)
.ok_or_else(|| crate::Error::InvalidConfig {
op,
message: "linear offset add overflows".to_string(),
})?;
stride = stride
.checked_mul(extent)
.ok_or_else(|| crate::Error::InvalidConfig {
op,
message: "linear offset stride overflows".to_string(),
})?;
}
Ok(offset)
}
fn try_shape_product(shape: &[usize], op: &'static str) -> crate::Result<usize> {
shape.iter().try_fold(1usize, |acc, &dim| {
acc.checked_mul(dim)
.ok_or_else(|| crate::Error::InvalidConfig {
op,
message: format!("shape product overflows for shape {shape:?}"),
})
})
}
fn try_checked_shape_len(shape: &[usize], data_len: usize, op: &'static str) -> crate::Result<()> {
let n = try_shape_product(shape, op)?;
if data_len != n {
return Err(crate::Error::InvalidConfig {
op,
message: format!("data length {data_len} does not match shape product {n}"),
});
}
Ok(())
}
fn try_compact_layout<R: TensorRank>(
shape: impl Into<R::Shape>,
op: &'static str,
) -> crate::Result<TensorLayout<R>> {
TensorLayout::compact(shape.into()).map_err(|err| tensor_layout_error(op, err))
}
fn tensor_layout_error(op: &'static str, err: tenferro_tensor_core::Error) -> crate::Error {
match err {
tenferro_tensor_core::Error::RankMismatch { expected, actual } => {
crate::Error::RankMismatch {
op,
expected,
actual,
}
}
tenferro_tensor_core::Error::AxisOutOfBounds { axis, rank } => {
crate::Error::AxisOutOfBounds { op, axis, rank }
}
tenferro_tensor_core::Error::DuplicateAxis { axis } => crate::Error::DuplicateAxis {
op,
axis,
role: "permutation",
},
tenferro_tensor_core::Error::InvalidPermutationLength { expected, actual } => {
crate::Error::RankMismatch {
op,
expected,
actual,
}
}
other => crate::Error::InvalidConfig {
op,
message: other.to_string(),
},
}
}
fn checked_view_element_count(shape: &[usize], op: &'static str) -> crate::Result<usize> {
shape.iter().try_fold(1usize, |product, &dim| {
if dim == 0 {
Ok(0)
} else {
product
.checked_mul(dim)
.ok_or_else(|| crate::Error::InvalidConfig {
op,
message: format!("shape product overflows for shape {shape:?}"),
})
}
})
}
fn checked_view_offset(
shape: &[usize],
strides: &[isize],
base_offset: isize,
indices: &[usize],
) -> Option<usize> {
if indices.len() != shape.len() {
return None;
}
let mut offset = base_offset;
for ((&index, &extent), &stride) in indices.iter().zip(shape).zip(strides) {
if index >= extent {
return None;
}
let index = isize::try_from(index).ok()?;
let delta = index.checked_mul(stride)?;
offset = offset.checked_add(delta)?;
}
usize::try_from(offset).ok()
}
fn for_each_layout_offset_col_major(
shape: &[usize],
strides: &[isize],
base_offset: isize,
op: &'static str,
mut f: impl FnMut(usize) -> crate::Result<()>,
) -> crate::Result<()> {
if shape.len() != strides.len() {
return Err(crate::Error::InvalidConfig {
op,
message: format!(
"shape rank {} does not match stride rank {}",
shape.len(),
strides.len()
),
});
}
if shape.contains(&0) {
return Ok(());
}
let mut offset = base_offset;
if shape.is_empty() {
let offset = usize::try_from(offset).map_err(|_| crate::Error::InvalidConfig {
op,
message: "view offset is negative".to_string(),
})?;
return f(offset);
}
let mut index = vec![0usize; shape.len()];
loop {
let physical = usize::try_from(offset).map_err(|_| crate::Error::InvalidConfig {
op,
message: "view offset is negative".to_string(),
})?;
f(physical)?;
let mut advance_axis = None;
for axis in 0..shape.len() {
let next_index =
index[axis]
.checked_add(1)
.ok_or_else(|| crate::Error::InvalidConfig {
op,
message: "logical index overflows".to_string(),
})?;
if next_index < shape[axis] {
advance_axis = Some((axis, next_index));
break;
}
}
let Some((advance_axis, next_index)) = advance_axis else {
return Ok(());
};
for axis in 0..advance_axis {
let steps = isize::try_from(index[axis]).map_err(|_| crate::Error::InvalidConfig {
op,
message: "logical index does not fit in isize".to_string(),
})?;
let rewind =
strides[axis]
.checked_mul(steps)
.ok_or_else(|| crate::Error::InvalidConfig {
op,
message: "stride rewind overflows".to_string(),
})?;
offset = offset
.checked_sub(rewind)
.ok_or_else(|| crate::Error::InvalidConfig {
op,
message: "view offset rewind overflows".to_string(),
})?;
index[axis] = 0;
}
offset = offset.checked_add(strides[advance_axis]).ok_or_else(|| {
crate::Error::InvalidConfig {
op,
message: "view offset overflows".to_string(),
}
})?;
index[advance_axis] = next_index;
}
}
fn reachable_layout_span(
shape: &[usize],
strides: &[isize],
offset: isize,
) -> crate::Result<Option<(usize, usize)>> {
if shape.contains(&0) {
return Ok(None);
}
let mut min_offset = offset;
let mut max_offset = offset;
for (&extent, &stride) in shape.iter().zip(strides) {
let steps =
isize::try_from(extent.saturating_sub(1)).map_err(|_| crate::Error::InvalidConfig {
op: "TypedTensorViewMut::try_multi_slice_mut",
message: "shape extent does not fit in isize".to_string(),
})?;
let end = stride
.checked_mul(steps)
.ok_or_else(|| crate::Error::InvalidConfig {
op: "TypedTensorViewMut::try_multi_slice_mut",
message: "stride span overflows".to_string(),
})?;
let (axis_min, axis_max) = if end < 0 { (end, 0) } else { (0, end) };
min_offset =
min_offset
.checked_add(axis_min)
.ok_or_else(|| crate::Error::InvalidConfig {
op: "TypedTensorViewMut::try_multi_slice_mut",
message: "minimum reachable offset overflows".to_string(),
})?;
max_offset =
max_offset
.checked_add(axis_max)
.ok_or_else(|| crate::Error::InvalidConfig {
op: "TypedTensorViewMut::try_multi_slice_mut",
message: "maximum reachable offset overflows".to_string(),
})?;
}
let min_offset = usize::try_from(min_offset).map_err(|_| crate::Error::InvalidConfig {
op: "TypedTensorViewMut::try_multi_slice_mut",
message: "minimum reachable offset is negative".to_string(),
})?;
let max_offset = usize::try_from(max_offset).map_err(|_| crate::Error::InvalidConfig {
op: "TypedTensorViewMut::try_multi_slice_mut",
message: "maximum reachable offset is negative".to_string(),
})?;
Ok(Some((min_offset, max_offset)))
}
fn split_two_mut_ranges<T>(
data: &mut [T],
first: (usize, usize),
second: (usize, usize),
) -> Option<(&mut [T], &mut [T])> {
if first.1 < second.0 {
let (_, after_first_start) = data.split_at_mut(first.0);
let (first_slice, after_first) = after_first_start.split_at_mut(first.1 - first.0 + 1);
let (_, after_gap) = after_first.split_at_mut(second.0 - first.1 - 1);
let (second_slice, _) = after_gap.split_at_mut(second.1 - second.0 + 1);
Some((first_slice, second_slice))
} else if second.1 < first.0 {
let (_, after_second_start) = data.split_at_mut(second.0);
let (second_slice, after_second) = after_second_start.split_at_mut(second.1 - second.0 + 1);
let (_, after_gap) = after_second.split_at_mut(first.0 - second.1 - 1);
let (first_slice, _) = after_gap.split_at_mut(first.1 - first.0 + 1);
Some((first_slice, second_slice))
} else {
None
}
}
fn adjusted_view_offset(offset: isize, span_start: usize) -> Option<isize> {
let span_start = isize::try_from(span_start).ok()?;
offset.checked_sub(span_start)
}
fn view_mut_from_layout_and_slice<'a, T: 'static, R: TensorRank>(
layout: &TensorLayout<R>,
offset: isize,
data: &'a mut [T],
placement: Placement,
) -> crate::Result<TypedTensorViewMut<'a, T, R>> {
let shape = R::shape_from_vec(layout.shape().to_vec().into())
.map_err(|err| tensor_layout_error("TypedTensorViewMut::try_multi_slice_mut", err))?;
let strides = R::strides_from_vec(layout.strides().to_vec().into())
.map_err(|err| tensor_layout_error("TypedTensorViewMut::try_multi_slice_mut", err))?;
TypedTensorViewMut::from_buffer_ref_mut(
shape,
strides,
offset,
TensorBufferRefMut::Host(data),
placement,
"TypedTensorViewMut::try_multi_slice_mut",
)
}
fn contiguous_layout_slice<'a, T, R: TensorRank>(
layout: &TensorLayout<R>,
data: &'a [T],
op: &'static str,
) -> crate::Result<&'a [T]> {
if !layout.is_compact_col_major() {
return Err(crate::Error::InvalidConfig {
op,
message: "view is not contiguous column-major".to_string(),
});
}
let len = checked_view_element_count(layout.shape(), op)?;
let start = usize::try_from(layout.offset()).map_err(|_| crate::Error::InvalidConfig {
op,
message: "view offset is negative".to_string(),
})?;
let end = start
.checked_add(len)
.ok_or_else(|| crate::Error::InvalidConfig {
op,
message: "contiguous view range overflows".to_string(),
})?;
data.get(start..end)
.ok_or_else(|| crate::Error::InvalidConfig {
op,
message: "contiguous view range is outside host buffer".to_string(),
})
}
fn materialize_view_buffer_col_major<T: Clone>(
shape: &[usize],
strides: &[isize],
offset: isize,
buffer: &TensorBufferRef<'_, T>,
op: &'static str,
) -> crate::Result<Vec<T>> {
let source = match buffer {
TensorBufferRef::Host(data) => *data,
TensorBufferRef::Backend(_) => return Err(crate::Error::backend_failure(
op,
"backend buffers cannot be materialized through host memory; download explicitly first",
)),
};
let n_elements = checked_view_element_count(shape, op)?;
let mut out = Vec::with_capacity(n_elements);
for_each_layout_offset_col_major(shape, strides, offset, op, |physical| {
let value = source
.get(physical)
.ok_or_else(|| crate::Error::InvalidConfig {
op,
message: "view offset is outside host buffer".to_string(),
})?;
out.push(value.clone());
Ok(())
})?;
Ok(out)
}
fn relaxed_col_major_contiguous(
shape: &[usize],
strides: &[isize],
op: &'static str,
) -> crate::Result<bool> {
let mut expected = 1isize;
for (&extent, &stride) in shape.iter().zip(strides) {
if extent <= 1 {
continue;
}
if stride != expected {
return Ok(false);
}
let extent = isize::try_from(extent).map_err(|_| crate::Error::InvalidConfig {
op,
message: "shape extent does not fit in isize".to_string(),
})?;
expected = expected
.checked_mul(extent)
.ok_or_else(|| crate::Error::InvalidConfig {
op,
message: "contiguous stride overflows".to_string(),
})?;
}
Ok(true)
}
fn reshape_layout_dyn<R: TensorRank>(
layout: &TensorLayout<R>,
shape: &[usize],
buffer_len: usize,
op: &'static str,
) -> crate::Result<TensorLayout<DynRank>> {
match layout.reshape_view_as::<DynRank>(shape.to_vec().into(), buffer_len) {
Ok(layout) => Ok(layout),
Err(err) => {
if !relaxed_col_major_contiguous(layout.shape(), layout.strides(), op)? {
return Err(tensor_layout_error(op, err));
}
let from = checked_view_element_count(layout.shape(), op)?;
let to = checked_view_element_count(shape, op)?;
if from != to {
return Err(tensor_layout_error(
op,
tenferro_tensor_core::Error::ReshapeElementCountMismatch { from, to },
));
}
TensorLayout::<DynRank>::compact(shape.to_vec().into())
.and_then(|compact| {
TensorLayout::from_parts(
compact.shape().to_vec().into(),
compact.strides().to_vec().into(),
layout.offset(),
buffer_len,
)
})
.map_err(|err| tensor_layout_error(op, err))
}
}
}
fn core_slice_specs(
slices: &[StridedSliceSpec],
shape: &[usize],
op: &'static str,
) -> crate::Result<Vec<CoreSliceSpec>> {
if slices.len() != shape.len() {
return Err(crate::Error::RankMismatch {
op,
expected: shape.len(),
actual: slices.len(),
});
}
let mut specs = Vec::with_capacity(slices.len());
for (slice, &axis_len) in slices.iter().zip(shape) {
specs.push(core_slice_spec(*slice, axis_len, op)?);
}
Ok(specs)
}
fn core_slice_spec(
slice: StridedSliceSpec,
axis_len: usize,
op: &'static str,
) -> crate::Result<CoreSliceSpec> {
if slice.step() == 0 {
return Err(crate::Error::InvalidConfig {
op,
message: "slice step must not be zero".to_string(),
});
}
let start = normalize_strided_bound(slice.start(), axis_len, op, "slice start")?;
let end = match slice.end() {
Some(end) => normalize_strided_bound(end, axis_len, op, "slice end")?,
None => isize::try_from(axis_len).map_err(|_| crate::Error::InvalidConfig {
op,
message: format!("axis length {axis_len} does not fit in isize"),
})?,
};
if slice.step() > 0 {
return Ok(CoreSliceSpec {
start,
end,
step: slice.step(),
});
}
if start >= end {
return Ok(CoreSliceSpec {
start,
end: start,
step: slice.step(),
});
}
Ok(CoreSliceSpec {
start: end
.checked_sub(1)
.ok_or_else(|| crate::Error::InvalidConfig {
op,
message: "negative-step slice start overflows".to_string(),
})?,
end: start
.checked_sub(1)
.ok_or_else(|| crate::Error::InvalidConfig {
op,
message: "negative-step slice end overflows".to_string(),
})?,
step: slice.step(),
})
}
fn normalize_strided_bound(
bound: isize,
axis_len: usize,
op: &'static str,
role: &'static str,
) -> crate::Result<isize> {
let axis_len = isize::try_from(axis_len).map_err(|_| crate::Error::InvalidConfig {
op,
message: format!("axis length {axis_len} does not fit in isize"),
})?;
let bound = if bound < 0 {
axis_len
.checked_add(bound)
.ok_or_else(|| crate::Error::InvalidConfig {
op,
message: format!("{role} {bound} overflows"),
})?
} else {
bound
};
if !(0..=axis_len).contains(&bound) {
return Err(crate::Error::InvalidConfig {
op,
message: format!("{role} {bound} is outside 0..={axis_len}"),
});
}
Ok(bound)
}
fn slice_axis_specs(
rank: usize,
axis: usize,
slice: StridedSliceSpec,
op: &'static str,
) -> crate::Result<Vec<StridedSliceSpec>> {
if axis >= rank {
return Err(crate::Error::AxisOutOfBounds { op, axis, rank });
}
let mut slices = vec![StridedSliceSpec::all(); rank];
slices[axis] = slice;
Ok(slices)
}
pub(crate) fn materialize_typed_view_col_major<T: Clone + 'static, R: TensorRank>(
view: &TypedTensorView<'_, T, R>,
op: &'static str,
) -> crate::Result<TypedTensor<T>> {
let data = materialize_view_buffer_col_major(
view.shape(),
view.strides(),
view.offset(),
&view.buffer,
op,
)?;
TypedTensor::from_vec_col_major(view.shape().to_vec(), data)
}
pub(crate) fn default_placement() -> Placement {
Placement {
memory_kind: MemoryKind::UnpinnedHost,
device: None,
}
}
fn typed_tensor_from_vec_col_major<T, R: TensorRank>(
shape: impl Into<R::Shape>,
data: Vec<T>,
op: &'static str,
) -> crate::Result<TypedTensor<T, R>> {
try_typed_tensor_from_vec_col_major(shape, data, op)
}
fn try_typed_tensor_from_vec_col_major<T, R: TensorRank>(
shape: impl Into<R::Shape>,
data: Vec<T>,
op: &'static str,
) -> crate::Result<TypedTensor<T, R>> {
let layout = try_compact_layout(shape, op)?;
try_checked_shape_len(layout.shape(), data.len(), op)?;
Ok(TypedTensor {
buffer: Buffer::Host(data),
layout,
placement: default_placement(),
})
}
fn typed_tensor_zeros<T: Clone + Zero, R: TensorRank>(
shape: impl Into<R::Shape>,
) -> crate::Result<TypedTensor<T, R>> {
try_typed_tensor_zeros(shape)
}
fn try_typed_tensor_zeros<T: Clone + Zero, R: TensorRank>(
shape: impl Into<R::Shape>,
) -> crate::Result<TypedTensor<T, R>> {
let layout = try_compact_layout(shape, "zeros")?;
let n = try_shape_product(layout.shape(), "zeros")?;
Ok(TypedTensor {
buffer: Buffer::Host(vec![T::zero(); n]),
layout,
placement: default_placement(),
})
}
fn typed_tensor_ones<T: Clone + One + Zero, R: TensorRank>(
shape: impl Into<R::Shape>,
) -> crate::Result<TypedTensor<T, R>> {
try_typed_tensor_ones(shape)
}
fn try_typed_tensor_ones<T: Clone + One + Zero, R: TensorRank>(
shape: impl Into<R::Shape>,
) -> crate::Result<TypedTensor<T, R>> {
let layout = try_compact_layout(shape, "ones")?;
let n = try_shape_product(layout.shape(), "ones")?;
Ok(TypedTensor {
buffer: Buffer::Host(vec![T::one(); n]),
layout,
placement: default_placement(),
})
}
fn typed_tensor_from_buffer_col_major<T: 'static, R: TensorRank>(
shape: impl Into<R::Shape>,
buffer: Buffer<T>,
placement: Placement,
) -> crate::Result<TypedTensor<T, R>> {
try_typed_tensor_from_buffer_col_major(shape, buffer, placement)
}
fn try_typed_tensor_from_buffer_col_major<T: 'static, R: TensorRank>(
shape: impl Into<R::Shape>,
buffer: Buffer<T>,
placement: Placement,
) -> crate::Result<TypedTensor<T, R>> {
let layout = try_compact_layout(shape, "from_buffer_col_major")?;
let len = buffer.len();
try_checked_shape_len(layout.shape(), len, "from_buffer_col_major")?;
Ok(TypedTensor {
buffer,
layout,
placement,
})
}
impl<T: Clone + Zero, R: TensorRank> TypedTensor<T, R> {
pub fn zeros(shape: impl Into<R::Shape>) -> crate::Result<Self> {
typed_tensor_zeros(shape)
}
}
impl<T: Clone + One + Zero, R: TensorRank> TypedTensor<T, R> {
pub fn ones(shape: impl Into<R::Shape>) -> crate::Result<Self> {
typed_tensor_ones(shape)
}
}
impl<T, R: TensorRank> TypedTensor<T, R> {
pub fn from_buffer_col_major(
shape: impl Into<R::Shape>,
buffer: Buffer<T>,
placement: Placement,
) -> crate::Result<Self>
where
T: 'static,
{
typed_tensor_from_buffer_col_major(shape, buffer, placement)
}
pub fn try_into_rank<const N: usize>(self) -> crate::Result<TypedTensor<T, Rank<N>>> {
let op = "TypedTensor::try_into_rank";
let shape = <Rank<N> as TensorRank>::shape_from_vec(self.shape().to_vec().into())
.map_err(|err| tensor_layout_error(op, err))?;
let layout =
TensorLayout::<Rank<N>>::compact(shape).map_err(|err| tensor_layout_error(op, err))?;
Ok(TypedTensor {
buffer: self.buffer,
layout,
placement: self.placement,
})
}
pub fn n_elements(&self) -> usize {
match try_shape_product(self.shape(), "TypedTensor::n_elements") {
Ok(n) => n,
Err(err) => {
unreachable!("TypedTensor compact shape is validated at construction: {err}")
}
}
}
pub fn shape(&self) -> &[usize] {
self.layout.shape()
}
pub fn rank(&self) -> usize {
self.shape().len()
}
pub fn layout(&self) -> &TensorLayout<R> {
&self.layout
}
pub fn buffer(&self) -> &Buffer<T> {
&self.buffer
}
pub fn placement(&self) -> &Placement {
&self.placement
}
pub fn set_placement(&mut self, placement: Placement) {
self.placement = placement;
}
pub fn as_view(&self) -> TypedTensorView<'_, T, R>
where
T: 'static,
{
let buffer = match &self.buffer {
Buffer::Host(data) => TensorBufferRef::Host(data),
Buffer::Backend(buffer) => TensorBufferRef::Backend(Arc::clone(buffer)),
};
TypedTensorView {
buffer,
layout: self.layout.clone(),
placement: self.placement.clone(),
}
}
pub fn as_view_mut(&mut self) -> TypedTensorViewMut<'_, T, R>
where
T: 'static,
{
let layout = self.layout.clone();
let placement = self.placement.clone();
let buffer = match &mut self.buffer {
Buffer::Host(data) => TensorBufferRefMut::Host(data),
Buffer::Backend(buffer) => TensorBufferRefMut::Backend(Arc::clone(buffer)),
};
TypedTensorViewMut {
buffer,
layout,
placement,
}
}
pub fn into_layout(self) -> TensorLayout<R> {
self.layout
}
pub fn into_parts(self) -> (Buffer<T>, TensorLayout<R>, Placement) {
(self.buffer, self.layout, self.placement)
}
}
impl<T: Clone, R: TensorRank> TypedTensor<T, R> {
pub fn from_vec_col_major(shape: impl Into<R::Shape>, data: Vec<T>) -> crate::Result<Self> {
typed_tensor_from_vec_col_major(shape, data, "from_vec_col_major")
}
pub fn into_vec_col_major(self) -> crate::Result<(Vec<usize>, Vec<T>)> {
let shape = self.shape().to_vec();
match self.buffer {
Buffer::Host(data) => Ok((shape, data)),
Buffer::Backend(_) => Err(crate::Error::backend_failure(
"into_vec_col_major",
"backend buffers cannot be exported as host Vec",
)),
}
}
pub fn host_data(&self) -> crate::Result<&[T]> {
match &self.buffer {
Buffer::Host(v) => Ok(v),
Buffer::Backend(_) => Err(crate::Error::backend_failure(
"TypedTensor::host_data",
"backend buffers cannot be inspected as host slices; download explicitly first",
)),
}
}
pub fn as_slice(&self) -> crate::Result<&[T]> {
self.host_data()
}
pub fn host_data_mut(&mut self) -> crate::Result<&mut [T]> {
match &mut self.buffer {
Buffer::Host(v) => Ok(v),
Buffer::Backend(_) => Err(crate::Error::backend_failure(
"TypedTensor::host_data_mut",
"backend buffers cannot be mutated as host slices; download explicitly first",
)),
}
}
pub fn linear_offset(&self, indices: &[usize]) -> crate::Result<usize> {
try_linear_offset_for_shape(self.shape(), indices, "TypedTensor::linear_offset")
}
pub fn get(&self, indices: &[usize]) -> crate::Result<&T> {
let off = self.linear_offset(indices)?;
self.host_data()?
.get(off)
.ok_or_else(|| crate::Error::InvalidConfig {
op: "TypedTensor::get",
message: format!("linear offset {off} is outside host buffer"),
})
}
pub fn get_mut(&mut self, indices: &[usize]) -> crate::Result<&mut T> {
let off = self.linear_offset(indices)?;
self.host_data_mut()?
.get_mut(off)
.ok_or_else(|| crate::Error::InvalidConfig {
op: "TypedTensor::get_mut",
message: format!("linear offset {off} is outside host buffer"),
})
}
}
impl Tensor {
pub fn from_vec_col_major<T: TensorScalar>(
shape: Vec<usize>,
data: Vec<T>,
) -> crate::Result<Self> {
T::into_tensor(shape, data)
}
pub fn shape(&self) -> &[usize] {
match self {
Tensor::F32(t) => t.shape(),
Tensor::F64(t) => t.shape(),
Tensor::I32(t) => t.shape(),
Tensor::I64(t) => t.shape(),
Tensor::Bool(t) => t.shape(),
Tensor::C32(t) => t.shape(),
Tensor::C64(t) => t.shape(),
}
}
pub fn dtype(&self) -> DType {
match self {
Tensor::F32(_) => DType::F32,
Tensor::F64(_) => DType::F64,
Tensor::I32(_) => DType::I32,
Tensor::I64(_) => DType::I64,
Tensor::Bool(_) => DType::Bool,
Tensor::C32(_) => DType::C32,
Tensor::C64(_) => DType::C64,
}
}
pub fn placement(&self) -> &Placement {
match self {
Tensor::F32(t) => t.placement(),
Tensor::F64(t) => t.placement(),
Tensor::I32(t) => t.placement(),
Tensor::I64(t) => t.placement(),
Tensor::Bool(t) => t.placement(),
Tensor::C32(t) => t.placement(),
Tensor::C64(t) => t.placement(),
}
}
pub fn is_backend_buffer(&self) -> bool {
match self {
Tensor::F32(t) => t.buffer().is_backend(),
Tensor::F64(t) => t.buffer().is_backend(),
Tensor::I32(t) => t.buffer().is_backend(),
Tensor::I64(t) => t.buffer().is_backend(),
Tensor::Bool(t) => t.buffer().is_backend(),
Tensor::C32(t) => t.buffer().is_backend(),
Tensor::C64(t) => t.buffer().is_backend(),
}
}
pub fn as_slice<T: TensorScalar>(&self) -> crate::Result<&[T]> {
T::as_slice(self)
}
pub fn into_vec_col_major<T: TensorScalar>(self) -> crate::Result<(Vec<usize>, Vec<T>)> {
let typed = T::into_typed(self)?;
typed.into_vec_col_major()
}
}
#[allow(dead_code)]
pub(crate) fn flat_to_multi(mut flat: usize, shape: &[usize], out: &mut [usize]) {
for i in 0..shape.len() {
out[i] = flat % shape[i];
flat /= shape[i];
}
}