use crate::fuse::expr_node::CpuExprNode;
use crate::ptr::MutPtr;
use crate::CpuBackend;
use coeus_core::{BackendError, CpuAddressableStorageMut, Layout, Scalar};
use coeus_tensor::Tensor;
#[repr(transparent)]
#[derive(Clone, Copy)]
struct ScopedExpression(*const ());
unsafe impl Send for ScopedExpression {}
unsafe impl Sync for ScopedExpression {}
impl ScopedExpression {
fn new<E>(expression: &E) -> Self {
Self(std::ptr::from_ref(expression).cast())
}
unsafe fn read<E: Copy>(self) -> E {
unsafe { self.0.cast::<E>().read() }
}
}
#[inline(always)]
fn logical_to_coords(mut temp: usize, ndim: usize, out_strides: &[usize], coords: &mut [usize]) {
match ndim {
0 => {}
1 => {
coords[0] = temp;
}
2 => {
let s0 = out_strides[0];
coords[0] = temp / s0;
coords[1] = temp % s0;
}
3 => {
let s0 = out_strides[0];
let s1 = out_strides[1];
coords[0] = temp / s0;
let rem0 = temp % s0;
coords[1] = rem0 / s1;
coords[2] = rem0 % s1;
}
4 => {
let s0 = out_strides[0];
let s1 = out_strides[1];
let s2 = out_strides[2];
coords[0] = temp / s0;
let rem0 = temp % s0;
coords[1] = rem0 / s1;
let rem1 = rem0 % s1;
coords[2] = rem1 / s2;
coords[3] = rem1 % s2;
}
_ => {
for d in 0..ndim {
coords[d] = temp / out_strides[d];
temp %= out_strides[d];
}
}
}
}
fn write_fused_values<E, T, B>(
expr: E,
out_ptr: MutPtr<T>,
out_numel: usize,
out_layout: &Layout,
contiguous_fast_path: bool,
backend: &B,
) where
E: CpuExprNode<T, B> + Copy + Send,
T: Scalar,
B: CpuBackend,
{
let ndim = out_layout.ndim();
let out_strides = out_layout.strides_cloned();
let expression = ScopedExpression::new(&expr);
if contiguous_fast_path {
backend.parallel_for(0, out_numel, move |idx| unsafe {
let expr = expression.read::<E>();
let val = expr.eval_cpu_flat(idx);
out_ptr.write(idx, val);
});
} else {
backend.parallel_for(0, out_numel, move |idx| {
let expr = unsafe { expression.read::<E>() };
if ndim <= 8 {
let mut coords = [0usize; 8];
logical_to_coords(idx, ndim, &out_strides, &mut coords[..ndim]);
unsafe {
let val = expr.eval_cpu(&coords[..ndim]);
out_ptr.write(idx, val);
}
} else {
let mut coords = smallvec::SmallVec::<[usize; 16]>::from_elem(0, ndim);
logical_to_coords(idx, ndim, &out_strides, &mut coords);
unsafe {
let val = expr.eval_cpu(&coords);
out_ptr.write(idx, val);
}
}
});
}
}
#[inline(always)]
fn reduce_pair<T: Scalar>(op: crate::ReductionOp, acc: T, val: T) -> T {
match op {
crate::ReductionOp::Sum | crate::ReductionOp::Mean => acc + val,
crate::ReductionOp::Prod => acc * val,
crate::ReductionOp::Max => {
if val > acc {
val
} else {
acc
}
}
crate::ReductionOp::Min => {
if val < acc {
val
} else {
acc
}
}
}
}
#[derive(Clone, Copy)]
struct FusedReductionPlan<'a> {
out_layout: &'a Layout,
axis: usize,
axis_len: usize,
op: crate::ReductionOp,
}
unsafe fn eval_reduction_at<E, T, B>(
expr: E,
coords: &mut [usize],
axis: usize,
axis_len: usize,
op: crate::ReductionOp,
) -> T
where
E: CpuExprNode<T, B> + Copy,
T: Scalar,
B: CpuBackend,
{
coords[axis] = 0;
let mut acc = expr.eval_cpu(coords);
for k in 1..axis_len {
coords[axis] = k;
let val = expr.eval_cpu(coords);
acc = reduce_pair(op, acc, val);
}
if matches!(op, crate::ReductionOp::Mean) {
acc / T::from_f64(axis_len as f64)
} else {
acc
}
}
fn write_fused_reductions<E, T, B>(
expr: E,
out_ptr: MutPtr<T>,
out_numel: usize,
plan: FusedReductionPlan<'_>,
backend: &B,
) where
E: CpuExprNode<T, B> + Copy + Send,
T: Scalar,
B: CpuBackend,
{
let ndim = plan.out_layout.ndim();
let out_strides = plan.out_layout.strides_cloned();
let expression = ScopedExpression::new(&expr);
backend.parallel_for(0, out_numel, move |idx| {
let expr = unsafe { expression.read::<E>() };
if ndim <= 8 {
let mut coords = [0usize; 8];
logical_to_coords(idx, ndim, &out_strides, &mut coords[..ndim]);
unsafe {
let acc =
eval_reduction_at(expr, &mut coords[..ndim], plan.axis, plan.axis_len, plan.op);
out_ptr.write(idx, acc);
}
} else {
let mut coords = smallvec::SmallVec::<[usize; 16]>::from_elem(0, ndim);
logical_to_coords(idx, ndim, &out_strides, &mut coords);
unsafe {
let acc = eval_reduction_at(expr, &mut coords, plan.axis, plan.axis_len, plan.op);
out_ptr.write(idx, acc);
}
}
});
}
pub fn validate_fused_reduction_axis(
op: crate::ReductionOp,
axis_len: usize,
) -> Result<(), BackendError> {
if axis_len == 0
&& matches!(
op,
crate::ReductionOp::Mean | crate::ReductionOp::Max | crate::ReductionOp::Min
)
{
return Err(BackendError::EmptyReduction {
operation: "fused reduction",
reduction: op,
});
}
Ok(())
}
pub fn evaluate_fused_cpu<E, T, B>(expr: &E, backend: &B) -> Result<Tensor<T, B>, BackendError>
where
E: CpuExprNode<T, B> + Copy + Send,
T: Scalar,
B: CpuBackend,
B::DeviceBuffer<T>: CpuAddressableStorageMut<T>,
{
let out_shape = expr.shape()?.ok_or_else(|| BackendError::Storage {
operation: "fused expression",
reason: "expression has no tensor input from which to derive its shape".to_string(),
})?;
let out_layout = Layout::new(out_shape.clone());
let mut out = Tensor::alloc_on(out_shape, backend);
let out_numel = out.numel();
let contiguous_fast_path = expr.is_contiguous_and_same_shape(out_layout.shape());
let out_ptr = MutPtr(out.storage_mut().as_mut_slice().as_mut_ptr());
write_fused_values(
*expr,
out_ptr,
out_numel,
&out_layout,
contiguous_fast_path,
backend,
);
Ok(out)
}
pub fn evaluate_fused_reduce_cpu<E, T, B>(
expr: &E,
op: crate::ReductionOp,
axis: usize,
backend: &B,
) -> Result<Tensor<T, B>, BackendError>
where
E: CpuExprNode<T, B> + Copy + Send,
T: Scalar,
B: CpuBackend,
B::DeviceBuffer<T>: CpuAddressableStorageMut<T>,
{
let expr_shape = expr.shape()?.ok_or_else(|| BackendError::Storage {
operation: "fused reduction",
reason: "expression has no tensor input from which to derive its shape".to_string(),
})?;
if axis >= expr_shape.len() {
return Err(BackendError::AxisOutOfRange {
operation: "fused reduction",
axis,
rank: expr_shape.len(),
});
}
let mut out_shape = expr_shape.clone();
out_shape[axis] = 1;
let out_layout = Layout::new(out_shape.clone());
let mut out = Tensor::alloc_on(out_shape, backend);
let out_numel = out.numel();
let axis_len = expr_shape[axis];
validate_fused_reduction_axis(op, axis_len)?;
if axis_len == 0 {
let identity = match op {
crate::ReductionOp::Sum => T::zero(),
crate::ReductionOp::Prod => T::one(),
crate::ReductionOp::Mean | crate::ReductionOp::Max | crate::ReductionOp::Min => {
unreachable!("invariant: undefined empty reductions were rejected")
}
};
backend.fill(out.storage_mut(), identity);
return Ok(out);
}
let plan = FusedReductionPlan {
out_layout: &out_layout,
axis,
axis_len,
op,
};
let out_ptr = MutPtr(out.storage_mut().as_mut_slice().as_mut_ptr());
write_fused_reductions(*expr, out_ptr, out_numel, plan, backend);
Ok(out)
}