1#![doc(hidden)]
2
3pub type Result<T> = tenferro_tensor::Result<T>;
15use tenferro_tensor::CompareDir;
16pub use tenferro_tensor::{
17 CacheStats, DType, Error, ErrorKind, TensorRank, TensorScalar, TypedTensor, TypedTensorView,
18};
19
20pub mod buffer_pool;
21pub use buffer_pool::{BufferPool, PoolScalar};
22mod pooled_uninit_output;
23pub use pooled_uninit_output::PooledUninitOutput;
24
25use num_complex::{Complex32, Complex64};
26use std::mem::MaybeUninit;
27use std::ptr::NonNull;
28use strided_basic::{
29 ErasedRawStridedMut, ErasedRawStridedPtr, ErasedRawStridedRef, ErasedRawStridedUninitMut,
30 KernelDType, StridedView,
31};
32
33const _: () = assert!(tenferro_tensor::INLINE_RANK == strided_basic::INLINE_RANK);
37
38#[doc(hidden)]
40pub fn cpu_backend_buffer_error(op: &'static str) -> Error {
41 Error::runtime_state(
42 op,
43 "CPU backend received backend buffer; download to host before CPU execution",
44 )
45}
46
47#[doc(hidden)]
53pub fn cpu_non_host_operand_error(op: &'static str, role: &'static str) -> Error {
54 Error::runtime_state(
55 op,
56 format!(
57 "CPU backend requires {role} host placement for {op}: the payload is a backend buffer; download to host before CPU execution"
58 ),
59 )
60}
61
62#[derive(Debug, thiserror::Error)]
63#[doc(hidden)]
64pub enum CpuNumericalError {
65 #[error("{op} detected division by zero for dtype {dtype:?}")]
66 DivisionByZero { op: &'static str, dtype: DType },
67}
68
69#[doc(hidden)]
70pub fn cpu_division_by_zero(op: &'static str, dtype: DType) -> Error {
71 Error::extension(
72 op,
73 "cpu",
74 ErrorKind::NumericalFailure,
75 CpuNumericalError::DivisionByZero { op, dtype },
76 )
77}
78
79#[doc(hidden)]
80pub trait ConjElem {
81 fn conj_elem(self) -> Self;
90}
91
92impl ConjElem for f32 {
93 fn conj_elem(self) -> Self {
94 self
95 }
96}
97impl ConjElem for f64 {
98 fn conj_elem(self) -> Self {
99 self
100 }
101}
102impl ConjElem for Complex32 {
103 fn conj_elem(self) -> Self {
104 self.conj()
105 }
106}
107impl ConjElem for Complex64 {
108 fn conj_elem(self) -> Self {
109 self.conj()
110 }
111}
112
113#[doc(hidden)]
114pub fn is_complex_dtype(dtype: DType) -> bool {
115 matches!(dtype, DType::C32 | DType::C64)
116}
117
118#[doc(hidden)]
119pub fn ordered_complex_error(op: &'static str) -> Error {
120 Error::unsupported(
121 op,
122 "complex tensors do not have a total order; compute abs/norm explicitly before ordered operations",
123 )
124}
125
126#[doc(hidden)]
127pub fn reject_complex_ordered_dtypes(op: &'static str, dtypes: &[DType]) -> Result<()> {
128 if dtypes.iter().copied().any(is_complex_dtype) {
129 return Err(ordered_complex_error(op));
130 }
131 Ok(())
132}
133
134#[doc(hidden)]
135pub fn reject_complex_unsupported_compare_dtypes(dir: &CompareDir, dtypes: &[DType]) -> Result<()> {
136 if *dir != CompareDir::Eq {
137 reject_complex_ordered_dtypes("compare", dtypes)?;
138 }
139 Ok(())
140}
141
142#[doc(hidden)]
143pub fn typed_host_data<'a, T: TensorScalar>(
144 op: &'static str,
145 tensor: &'a TypedTensor<T>,
146) -> Result<&'a [T]> {
147 if tensor.backend_buffer().is_some() {
148 return Err(cpu_backend_buffer_error(op));
149 }
150 tensor.host_data()
151}
152
153#[doc(hidden)]
154pub fn typed_view<'a, T: Copy + TensorScalar>(
155 op: &'static str,
156 tensor: &'a TypedTensor<T>,
157) -> Result<StridedView<'a, T>> {
158 if tensor.backend_buffer().is_some() {
159 return Err(cpu_backend_buffer_error(op));
160 }
161 let data = tensor.host_data()?;
162 StridedView::new(data, tensor.shape(), tensor.strides(), 0)
165 .map_err(|err| Error::backend_source(op, err))
166}
167
168#[doc(hidden)]
169pub fn typed_view_from_view<'a, T: Copy + 'static, R: TensorRank>(
170 op: &'static str,
171 view: &TypedTensorView<'a, T, R>,
172) -> Result<StridedView<'a, T>> {
173 if view.backend_buffer().is_some() {
174 return Err(cpu_backend_buffer_error(op));
175 }
176 StridedView::new(
177 view.host_storage()?,
178 view.shape(),
179 view.strides(),
180 view.offset(),
181 )
182 .map_err(|err| Error::backend_source(op, err))
183}
184
185#[doc(hidden)]
191pub unsafe fn erased_raw_strided_ref<'a>(
192 dtype: KernelDType,
193 data: &'a [u8],
194 dims: &'a [usize],
195 strides: &'a [isize],
196 offset: isize,
197) -> strided_basic::Result<ErasedRawStridedRef<'a>> {
198 let data_ptr = NonNull::new(data.as_ptr().cast_mut()).unwrap_or_else(NonNull::dangling);
199 unsafe {
201 ErasedRawStridedRef::from_raw_parts(dtype, data_ptr, data.len(), dims, strides, offset)
202 }
203}
204
205#[doc(hidden)]
211pub unsafe fn erased_raw_strided_ptr<'a>(
212 dtype: KernelDType,
213 data: &'a [u8],
214 dims: &'a [usize],
215 strides: &'a [isize],
216 offset: isize,
217) -> strided_basic::Result<ErasedRawStridedPtr<'a>> {
218 let data_ptr = NonNull::new(data.as_ptr().cast_mut()).unwrap_or_else(NonNull::dangling);
219 unsafe {
221 ErasedRawStridedPtr::from_raw_parts(dtype, data_ptr, data.len(), dims, strides, offset)
222 }
223}
224
225#[doc(hidden)]
231pub unsafe fn erased_raw_strided_mut<'a>(
232 dtype: KernelDType,
233 data: &'a mut [u8],
234 dims: &'a [usize],
235 strides: &'a [isize],
236 offset: isize,
237) -> strided_basic::Result<ErasedRawStridedMut<'a>> {
238 let data_ptr = NonNull::new(data.as_mut_ptr()).unwrap_or_else(NonNull::dangling);
239 unsafe {
241 ErasedRawStridedMut::from_raw_parts(dtype, data_ptr, data.len(), dims, strides, offset)
242 }
243}
244
245#[doc(hidden)]
252pub unsafe fn erased_raw_strided_uninit_mut<'a>(
253 dtype: KernelDType,
254 data: &'a mut [MaybeUninit<u8>],
255 dims: &'a [usize],
256 strides: &'a [isize],
257 offset: isize,
258) -> strided_basic::Result<ErasedRawStridedUninitMut<'a>> {
259 let data_ptr = NonNull::new(data.as_mut_ptr().cast::<u8>()).unwrap_or_else(NonNull::dangling);
260 unsafe {
262 ErasedRawStridedUninitMut::from_raw_parts(
263 dtype,
264 data_ptr,
265 data.len(),
266 dims,
267 strides,
268 offset,
269 )
270 }
271}