tenferro-internal-cpu-kernels 0.6.0

use std::mem::MaybeUninit;
use std::ops::{Add, Div, Mul, Neg, Rem as StdRem, Sub};

use num_complex::Complex;
use num_traits::{One, Zero};
use strided_kernel::{
    batched_outer_product_into_uninit, broadcast_mul_into_uninit, compare_into_uninit, map_into,
    mul_into_uninit, reduce, zip_map2_into, zip_map3_into, CompareOp, StridedView,
};

use tenferro_cpu_basic::{BufferPool, ConjElem, PoolScalar, PooledUninitOutput};
use tenferro_tensor::{
    col_major_strides, CompareDir, DType, Tensor, TensorRank, TensorRead, TensorScalar,
    TensorValue, TensorView, TypedTensor, TypedTensorView,
};

use super::{typed_host_data, typed_view, typed_view_from_view};
use tenferro_cpu_basic::{
    reject_complex_ordered_dtypes, reject_complex_unsupported_compare_dtypes,
};

/// The typed operands behind a same-dtype triple, or this module's refusal for one.
/// The Rust scalar type behind a preset variant name a macro received.
macro_rules! preset_scalar {
    (F32) => {
        f32
    };
    (F64) => {
        f64
    };
    (I32) => {
        i32
    };
    (I64) => {
        i64
    };
    (Bool) => {
        bool
    };
    (C32) => {
        num_complex::Complex32
    };
    (C64) => {
        num_complex::Complex64
    };
}
macro_rules! dispatch_ternary_result_with_pool {
    ($op:literal, $a:expr, $b:expr, $c:expr, |$x:ident, $y:ident, $z:ident| $body:expr) => {{
        match $a.dtype() {
            DType::F32 => {
                let ($x, $y, $z) = ternary_operands::<f32>($op, $a, $b, $c)?;
                Ok(Tensor::from_typed::<f32>($body?))
            }
            DType::F64 => {
                let ($x, $y, $z) = ternary_operands::<f64>($op, $a, $b, $c)?;
                Ok(Tensor::from_typed::<f64>($body?))
            }
            _ => Err(ternary_dtype_error(
                $op,
                [$a.dtype(), $b.dtype(), $c.dtype()],
            )),
        }
    }};
}
fn ternary_operands<'a, T: TensorScalar>(
    op: &'static str,
    a: &'a Tensor,
    b: &'a Tensor,
    c: &'a Tensor,
) -> crate::Result<(&'a TypedTensor<T>, &'a TypedTensor<T>, &'a TypedTensor<T>)> {
    let mismatch = || ternary_dtype_error(op, [a.dtype(), b.dtype(), c.dtype()]);
    let a_t = a.as_typed::<T>().ok_or_else(mismatch)?;
    let b_t = b.as_typed::<T>().ok_or_else(mismatch)?;
    let c_t = c.as_typed::<T>().ok_or_else(mismatch)?;
    Ok((a_t, b_t, c_t))
}

fn ternary_dtype_error(op: &'static str, dtypes: [DType; 3]) -> crate::Error {
    if dtypes[0] != dtypes[1] {
        dtype_pair_error(op, dtypes[0], dtypes[1])
    } else if dtypes[0] != dtypes[2] {
        dtype_pair_error(op, dtypes[0], dtypes[2])
    } else {
        dtype_pair_error(op, dtypes[0], dtypes[0])
    }
}

fn dtype_pair_error(op: &'static str, lhs: DType, rhs: DType) -> crate::Error {
    if lhs == rhs {
        let supported = match op {
            "clamp" => "F32/F64",
            "maximum" | "minimum" | "rem" => "F32/F64/I32/I64",
            "add" | "mul" | "sub" => "F32/F64/I32/I64/C32/C64",
            _ => unreachable!("dtype_pair_error has no supported-dtype contract for {op}"),
        };
        crate::Error::unsupported(
            op,
            format!("unsupported dtype {lhs:?}; supported dtypes: {supported}"),
        )
    } else {
        crate::Error::dtype_mismatch(op, lhs, rhs)
    }
}

fn unary_dtype_error(
    op: &'static str,
    dtype: DType,
    supported: &'static str,
    recommend_f64: bool,
) -> crate::Error {
    let remedy = (recommend_f64 && matches!(dtype, DType::I32 | DType::I64))
        .then_some("; convert to F64 before this operation");
    crate::Error::unsupported(
        op,
        format!(
            "unsupported dtype {dtype:?}; supported dtypes: {supported}{}",
            remedy.unwrap_or("")
        ),
    )
}

/// The typed operand behind `operand`, or the refusal this table's wildcard arm
/// produces for the pair.
///
/// `lhs` and `rhs` are the pair being dispatched on, in that order, so the refusal
/// reads the same way whichever operand could not be typed. Callers reach this from
/// a match on the pair's dtypes, so `None` means the table and the runtime dtype
/// disagree rather than a caller mistake.
fn pair_operand<'a, T: TensorScalar>(
    op: &'static str,
    lhs: &Tensor,
    rhs: &Tensor,
    operand: &'a Tensor,
) -> crate::Result<&'a TypedTensor<T>> {
    operand
        .as_typed::<T>()
        .ok_or_else(|| dtype_pair_error(op, lhs.dtype(), rhs.dtype()))
}

fn tensor_pair_error(op: &'static str, lhs: &Tensor, rhs: &Tensor) -> crate::Error {
    dtype_pair_error(op, lhs.dtype(), rhs.dtype())
}

fn read_pair_error(op: &'static str, lhs: TensorRead<'_>, rhs: TensorRead<'_>) -> crate::Error {
    dtype_pair_error(op, lhs.dtype(), rhs.dtype())
}

#[doc(hidden)]
pub trait Tier2Elem: Copy + Clone + One + Zero + Send + Sync {
    fn abs_elem(self) -> Self;
    fn sign_elem(self) -> Self;
}

// Keep ordering separate from abs/sign so complex tensors cannot silently pick
// a magnitude ordering. Callers should compute abs/norm explicitly first.
#[doc(hidden)]
pub trait OrderedElem: Copy + Clone + Send + Sync {
    fn max_elem(self, other: Self) -> Self;
    fn min_elem(self, other: Self) -> Self;
}

#[doc(hidden)]
pub trait CompareElem: Copy + Send + Sync {
    fn compare_elem(self, other: Self, dir: &CompareDir) -> bool;
}

trait WrappingIntegerElem:
    Copy
    + PoolScalar
    + TensorScalar
    + Zero
    + PartialEq
    + Eq
    + Send
    + Sync
    + Mul<Output = Self>
    + 'static
{
    fn wrapping_add_elem(self, other: Self) -> Self;
    fn wrapping_sub_elem(self, other: Self) -> Self;
    fn wrapping_mul_elem(self, other: Self) -> Self;
    fn wrapping_div_elem(self, other: Self) -> Self;
    fn wrapping_rem_elem(self, other: Self) -> Self;
    fn wrapping_neg_elem(self) -> Self;
    fn wrapping_abs_elem(self) -> Self;
    fn signum_elem(self) -> Self;
}

macro_rules! impl_tier2_elem_real {
    ($ty:ty) => {
        impl Tier2Elem for $ty {
            fn abs_elem(self) -> Self {
                self.abs()
            }

            fn sign_elem(self) -> Self {
                if self == Self::zero() {
                    Self::zero()
                } else {
                    self.signum()
                }
            }
        }

        impl OrderedElem for $ty {
            fn max_elem(self, other: Self) -> Self {
                if self.is_nan() || other.is_nan() {
                    <$ty>::NAN
                } else if self >= other {
                    self
                } else {
                    other
                }
            }

            fn min_elem(self, other: Self) -> Self {
                if self.is_nan() || other.is_nan() {
                    <$ty>::NAN
                } else if self <= other {
                    self
                } else {
                    other
                }
            }
        }

        impl CompareElem for $ty {
            fn compare_elem(self, other: Self, dir: &CompareDir) -> bool {
                match dir {
                    CompareDir::Eq => self == other,
                    CompareDir::Lt => self < other,
                    CompareDir::Le => self <= other,
                    CompareDir::Gt => self > other,
                    CompareDir::Ge => self >= other,
                }
            }
        }
    };
}

macro_rules! impl_tier2_elem_complex {
    ($real:ty) => {
        impl Tier2Elem for Complex<$real> {
            fn abs_elem(self) -> Self {
                Self::new(self.norm(), <$real>::zero())
            }

            fn sign_elem(self) -> Self {
                if self.is_zero() {
                    Self::zero()
                } else {
                    // Divide the components by the real modulus rather than
                    // using complex division: `Complex::div` squares the
                    // divisor's components, which underflows for a tiny
                    // magnitude such as `1e-200` in f64 and produces NaN
                    // instead of the unit phase.
                    let modulus = self.norm();
                    Self::new(self.re / modulus, self.im / modulus)
                }
            }
        }

        impl CompareElem for Complex<$real> {
            fn compare_elem(self, other: Self, dir: &CompareDir) -> bool {
                match dir {
                    CompareDir::Eq => self == other,
                    CompareDir::Lt | CompareDir::Le | CompareDir::Gt | CompareDir::Ge => false,
                }
            }
        }
    };
}

impl_tier2_elem_real!(f32);
impl_tier2_elem_real!(f64);
impl_tier2_elem_complex!(f32);
impl_tier2_elem_complex!(f64);

macro_rules! impl_compare_elem_ord {
    ($ty:ty) => {
        impl CompareElem for $ty {
            fn compare_elem(self, other: Self, dir: &CompareDir) -> bool {
                match dir {
                    CompareDir::Eq => self == other,
                    CompareDir::Lt => self < other,
                    CompareDir::Le => self <= other,
                    CompareDir::Gt => self > other,
                    CompareDir::Ge => self >= other,
                }
            }
        }
    };
}

impl_compare_elem_ord!(i32);
impl_compare_elem_ord!(i64);
impl_compare_elem_ord!(bool);

macro_rules! impl_ordered_elem_ord {
    ($ty:ty) => {
        impl OrderedElem for $ty {
            fn max_elem(self, other: Self) -> Self {
                self.max(other)
            }

            fn min_elem(self, other: Self) -> Self {
                self.min(other)
            }
        }
    };
}

impl_ordered_elem_ord!(i32);
impl_ordered_elem_ord!(i64);

macro_rules! impl_wrapping_integer_elem {
    ($ty:ty) => {
        impl WrappingIntegerElem for $ty {
            fn wrapping_add_elem(self, other: Self) -> Self {
                self.wrapping_add(other)
            }

            fn wrapping_sub_elem(self, other: Self) -> Self {
                self.wrapping_sub(other)
            }

            fn wrapping_mul_elem(self, other: Self) -> Self {
                self.wrapping_mul(other)
            }

            fn wrapping_div_elem(self, other: Self) -> Self {
                self.wrapping_div(other)
            }

            fn wrapping_rem_elem(self, other: Self) -> Self {
                self.wrapping_rem(other)
            }

            fn wrapping_neg_elem(self) -> Self {
                self.wrapping_neg()
            }

            fn wrapping_abs_elem(self) -> Self {
                self.wrapping_abs()
            }

            fn signum_elem(self) -> Self {
                self.signum()
            }
        }
    };
}

impl_wrapping_integer_elem!(i32);
impl_wrapping_integer_elem!(i64);

fn strided_view_contains<T>(
    op: &'static str,
    view: &StridedView<'_, T>,
    pred: impl Fn(T) -> bool + Copy + Sync,
) -> crate::Result<bool>
where
    T: Copy + Send + Sync,
{
    reduce(view, pred, |lhs, rhs| lhs || rhs, false)
        .map_err(|err| crate::Error::backend_source(op, err))
}

fn ensure_no_zero_divisor<T>(op: &'static str, rhs: &StridedView<'_, T>) -> crate::Result<()>
where
    T: WrappingIntegerElem,
{
    if strided_view_contains(op, rhs, |value| value == T::zero())? {
        return Err(crate::cpu_division_by_zero(op, T::dtype()));
    }
    Ok(())
}

fn complex_scalar_tensor<T>(scalar: T) -> crate::Result<TypedTensor<Complex<T>>>
where
    T: Copy + Clone + Zero + tenferro_tensor::TensorScalar,
    Complex<T>: tenferro_tensor::TensorScalar,
{
    TypedTensor::from_vec_col_major(vec![], vec![Complex::new(scalar, T::zero())])
}

fn complex_scalar_tensor_from_tensor<T>(
    input: &TypedTensor<T>,
) -> crate::Result<TypedTensor<Complex<T>>>
where
    T: Copy + Clone + Zero + tenferro_tensor::TensorScalar,
    Complex<T>: tenferro_tensor::TensorScalar,
{
    complex_scalar_tensor(typed_host_data("add", input)?[0])
}

fn complex_scalar_tensor_from_view<T, R>(
    input: &TypedTensorView<'_, T, R>,
) -> crate::Result<TypedTensor<Complex<T>>>
where
    T: Copy + Clone + Zero + 'static + tenferro_tensor::TensorScalar,
    Complex<T>: tenferro_tensor::TensorScalar,
    R: TensorRank,
{
    complex_scalar_tensor(typed_view_from_view("add", input)?.get(&[]))
}

#[cfg(test)]
fn with_test_pool<T>(f: impl FnOnce(&mut BufferPool) -> T) -> T {
    let mut buffers = BufferPool::new();
    f(&mut buffers)
}

/// Add two CPU tensors elementwise.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::add;
/// use tenferro_tensor::Tensor;
///
/// let a = Tensor::from_vec_col_major(vec![2], vec![1.0_f64, 2.0])?;
/// let b = Tensor::from_vec_col_major(vec![2], vec![3.0_f64, 4.0])?;
/// let out = add(&a, &b)?;
/// assert_eq!(out.as_slice::<f64>().unwrap(), &[4.0, 6.0]);
/// # Ok::<(), tenferro_tensor::Error>(())
/// ```
#[cfg(test)]
#[doc(hidden)]
pub fn add(lhs: &Tensor, rhs: &Tensor) -> crate::Result<Tensor> {
    with_test_pool(|buffers| add_with_pool(buffers, lhs, rhs))
}

/// Dispatch a same-dtype binary pair to one typed kernel.
///
/// The arm is selected by the pair's dtypes, which `Tensor` derives from the same payload the
/// extraction reads, so `pair_operand` cannot fail inside it. Binding each fallible step to its
/// own `let` keeps every `?` on a line whose call executed, and keeping the body in one macro
/// means the per-preset binary tables share one covered definition instead of one closing
/// `)?))` line per arm.
macro_rules! same_dtype_binary {
    ($buffers:expr, $lhs:expr, $rhs:expr, $scalar:ty, $kernel:ident, $op:expr) => {{
        let a = pair_operand::<$scalar>($op, $lhs, $rhs, $lhs)?;
        let b = pair_operand::<$scalar>($op, $lhs, $rhs, $rhs)?;
        let out = $kernel($buffers, a, b)?;
        Ok(Tensor::from_typed::<$scalar>(out))
    }};
}

#[doc(hidden)]
pub fn add_with_pool(
    buffers: &mut BufferPool,
    lhs: &Tensor,
    rhs: &Tensor,
) -> crate::Result<Tensor> {
    match (lhs.dtype(), rhs.dtype()) {
        (DType::F32, DType::F32) => {
            same_dtype_binary!(buffers, lhs, rhs, f32, typed_add_with_pool, "add")
        }
        (DType::F64, DType::F64) => {
            same_dtype_binary!(buffers, lhs, rhs, f64, typed_add_with_pool, "add")
        }
        (DType::I32, DType::I32) => {
            same_dtype_binary!(buffers, lhs, rhs, i32, typed_wrapping_add_with_pool, "add")
        }
        (DType::I64, DType::I64) => {
            same_dtype_binary!(buffers, lhs, rhs, i64, typed_wrapping_add_with_pool, "add")
        }
        (DType::C32, DType::C32) => {
            same_dtype_binary!(buffers, lhs, rhs, Complex<f32>, typed_add_with_pool, "add")
        }
        (DType::C64, DType::C64) => {
            same_dtype_binary!(buffers, lhs, rhs, Complex<f64>, typed_add_with_pool, "add")
        }
        (DType::F32, DType::C32)
            if pair_operand::<f32>("add", lhs, rhs, lhs)?
                .shape()
                .is_empty() =>
        {
            let value = typed_host_data("add", pair_operand::<f32>("add", lhs, rhs, lhs)?)?[0];
            let scalar = complex_scalar_tensor(value)?;
            let other = pair_operand::<Complex<f32>>("add", lhs, rhs, rhs)?;
            let out = typed_add_with_pool(buffers, &scalar, other)?;
            Ok(Tensor::from_typed::<Complex<f32>>(out))
        }
        (DType::C32, DType::F32)
            if pair_operand::<f32>("add", lhs, rhs, rhs)?
                .shape()
                .is_empty() =>
        {
            let value = typed_host_data("add", pair_operand::<f32>("add", lhs, rhs, rhs)?)?[0];
            let scalar = complex_scalar_tensor(value)?;
            let first = pair_operand::<Complex<f32>>("add", lhs, rhs, lhs)?;
            let out = typed_add_with_pool(buffers, first, &scalar)?;
            Ok(Tensor::from_typed::<Complex<f32>>(out))
        }
        (DType::F64, DType::C64)
            if pair_operand::<f64>("add", lhs, rhs, lhs)?
                .shape()
                .is_empty() =>
        {
            let value = typed_host_data("add", pair_operand::<f64>("add", lhs, rhs, lhs)?)?[0];
            let scalar = complex_scalar_tensor(value)?;
            let other = pair_operand::<Complex<f64>>("add", lhs, rhs, rhs)?;
            let out = typed_add_with_pool(buffers, &scalar, other)?;
            Ok(Tensor::from_typed::<Complex<f64>>(out))
        }
        (DType::C64, DType::F64)
            if pair_operand::<f64>("add", lhs, rhs, rhs)?
                .shape()
                .is_empty() =>
        {
            let value = typed_host_data("add", pair_operand::<f64>("add", lhs, rhs, rhs)?)?[0];
            let scalar = complex_scalar_tensor(value)?;
            let first = pair_operand::<Complex<f64>>("add", lhs, rhs, lhs)?;
            let out = typed_add_with_pool(buffers, first, &scalar)?;
            Ok(Tensor::from_typed::<Complex<f64>>(out))
        }
        _ => Err(tensor_pair_error("add", lhs, rhs)),
    }
}

/// Dispatch a same-variant read pair to that variant's typed kernel.
///
/// Every erased elementwise operation reaches the same typed kernels through
/// these declarations, so an operation adds no matching code of its own.
macro_rules! dispatch_read_same_variant {
    ($buffers:expr, $lhs:expr, $rhs:expr, $variant:ident, $func:ident) => {
        match (&$lhs, &$rhs) {
            (TensorRead::Tensor(a), TensorRead::View(TensorView::$variant(b)))
                if a.dtype()
                    == <preset_scalar!($variant) as tenferro_tensor::TensorScalar>::dtype() =>
            {
                let a = a
                    .as_typed::<preset_scalar!($variant)>()
                    .expect("the dtype guard selects this arm");
                let a = a.as_view();
                return Ok(Tensor::from_typed::<preset_scalar!($variant)>($func(
                    $buffers, &a, b,
                )?));
            }
            (TensorRead::View(TensorView::$variant(a)), TensorRead::Tensor(b))
                if b.dtype()
                    == <preset_scalar!($variant) as tenferro_tensor::TensorScalar>::dtype() =>
            {
                let b = b
                    .as_typed::<preset_scalar!($variant)>()
                    .expect("the dtype guard selects this arm");
                let b = b.as_view();
                return Ok(Tensor::from_typed::<preset_scalar!($variant)>($func(
                    $buffers, a, &b,
                )?));
            }
            (
                TensorRead::View(TensorView::$variant(a)),
                TensorRead::View(TensorView::$variant(b)),
            ) => {
                return Ok(Tensor::from_typed::<preset_scalar!($variant)>($func(
                    $buffers, a, b,
                )?));
            }
            _ => {}
        }
    };
}

/// The single list of preset variants an erased elementwise operation reaches.
///
/// An operation supplies only the kernel that implements it for a float-like
/// variant and for an integer variant; the variant list itself exists once here,
/// so adding a preset scalar is a single edit rather than one edit per
/// operation.
macro_rules! dispatch_read_presets {
    ($buffers:expr, $lhs:expr, $rhs:expr, $float:ident, $integer:ident) => {
        dispatch_read_real_complex_scalar!($buffers, $lhs, $rhs, F32, C32, $float);
        dispatch_read_real_complex_scalar!($buffers, $lhs, $rhs, F64, C64, $float);

        dispatch_read_same_variant!($buffers, $lhs, $rhs, F32, $float);
        dispatch_read_same_variant!($buffers, $lhs, $rhs, F64, $float);
        dispatch_read_same_variant!($buffers, $lhs, $rhs, I32, $integer);
        dispatch_read_same_variant!($buffers, $lhs, $rhs, I64, $integer);
        dispatch_read_same_variant!($buffers, $lhs, $rhs, C32, $float);
        dispatch_read_same_variant!($buffers, $lhs, $rhs, C64, $float);
    };
}

macro_rules! dispatch_read_real_complex_scalar {
    ($buffers:expr, $lhs:expr, $rhs:expr, $real_variant:ident, $complex_variant:ident, $func:ident) => {
        match (&$lhs, &$rhs) {
            (TensorRead::Tensor(real), TensorRead::View(TensorView::$complex_variant(complex)))
                if real.dtype()
                    == <preset_scalar!($real_variant) as tenferro_tensor::TensorScalar>::dtype(
                    )
                    && real.shape().is_empty() =>
            {
                let real = real
                    .as_typed::<preset_scalar!($real_variant)>()
                    .expect("the dtype guard selects this arm");
                let scalar = complex_scalar_tensor_from_tensor(real)?;
                let scalar = scalar.as_view();
                return Ok(Tensor::from_typed::<preset_scalar!($complex_variant)>(
                    $func($buffers, &scalar, complex)?,
                ));
            }
            (TensorRead::View(TensorView::$real_variant(real)), TensorRead::Tensor(complex))
                if complex.dtype()
                    == <preset_scalar!($complex_variant) as tenferro_tensor::TensorScalar>::dtype()
                    && real.shape().is_empty() =>
            {
                let complex = complex
                    .as_typed::<preset_scalar!($complex_variant)>()
                    .expect("the dtype guard selects this arm");
                let scalar = complex_scalar_tensor_from_view(real)?;
                let scalar = scalar.as_view();
                let complex = complex.as_view();
                return Ok(Tensor::from_typed::<preset_scalar!($complex_variant)>(
                    $func($buffers, &scalar, &complex)?,
                ));
            }
            (
                TensorRead::View(TensorView::$real_variant(real)),
                TensorRead::View(TensorView::$complex_variant(complex)),
            ) if real.shape().is_empty() => {
                let scalar = complex_scalar_tensor_from_view(real)?;
                let scalar = scalar.as_view();
                return Ok(Tensor::from_typed::<preset_scalar!($complex_variant)>(
                    $func($buffers, &scalar, complex)?,
                ));
            }
            (TensorRead::Tensor(complex), TensorRead::View(TensorView::$real_variant(real)))
                if complex.dtype()
                    == <preset_scalar!($complex_variant) as tenferro_tensor::TensorScalar>::dtype()
                    && real.shape().is_empty() =>
            {
                let complex = complex
                    .as_typed::<preset_scalar!($complex_variant)>()
                    .expect("the dtype guard selects this arm");
                let complex = complex.as_view();
                let scalar = complex_scalar_tensor_from_view(real)?;
                let scalar = scalar.as_view();
                return Ok(Tensor::from_typed::<preset_scalar!($complex_variant)>(
                    $func($buffers, &complex, &scalar)?,
                ));
            }
            (TensorRead::View(TensorView::$complex_variant(complex)), TensorRead::Tensor(real))
                if real.dtype()
                    == <preset_scalar!($real_variant) as tenferro_tensor::TensorScalar>::dtype(
                    )
                    && real.shape().is_empty() =>
            {
                let real = real
                    .as_typed::<preset_scalar!($real_variant)>()
                    .expect("the dtype guard selects this arm");
                let scalar = complex_scalar_tensor_from_tensor(real)?;
                let scalar = scalar.as_view();
                return Ok(Tensor::from_typed::<preset_scalar!($complex_variant)>(
                    $func($buffers, complex, &scalar)?,
                ));
            }
            (
                TensorRead::View(TensorView::$complex_variant(complex)),
                TensorRead::View(TensorView::$real_variant(real)),
            ) if real.shape().is_empty() => {
                let scalar = complex_scalar_tensor_from_view(real)?;
                let scalar = scalar.as_view();
                return Ok(Tensor::from_typed::<preset_scalar!($complex_variant)>(
                    $func($buffers, complex, &scalar)?,
                ));
            }
            _ => {}
        }
    };
}

#[doc(hidden)]
pub fn add_read_with_pool(
    buffers: &mut BufferPool,
    lhs: TensorRead<'_>,
    rhs: TensorRead<'_>,
) -> crate::Result<Tensor> {
    if let (TensorRead::Tensor(lhs), TensorRead::Tensor(rhs)) = (&lhs, &rhs) {
        return add_with_pool(buffers, lhs, rhs);
    }

    dispatch_read_presets!(
        buffers,
        lhs,
        rhs,
        typed_add_view_with_pool,
        typed_wrapping_add_view_with_pool
    );

    Err(read_pair_error("add", lhs, rhs))
}

/// Subtract two CPU tensors elementwise.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::sub;
/// use tenferro_tensor::Tensor;
///
/// let a = Tensor::from_vec_col_major(vec![2], vec![5.0_f64, 2.0])?;
/// let b = Tensor::from_vec_col_major(vec![2], vec![3.0_f64, 4.0])?;
/// let out = sub(&a, &b)?;
/// assert_eq!(out.as_slice::<f64>().unwrap(), &[2.0, -2.0]);
/// # Ok::<(), tenferro_tensor::Error>(())
/// ```
#[cfg(test)]
#[doc(hidden)]
pub fn sub(lhs: &Tensor, rhs: &Tensor) -> crate::Result<Tensor> {
    with_test_pool(|buffers| sub_with_pool(buffers, lhs, rhs))
}

#[doc(hidden)]
pub fn sub_with_pool(
    buffers: &mut BufferPool,
    lhs: &Tensor,
    rhs: &Tensor,
) -> crate::Result<Tensor> {
    match (lhs.dtype(), rhs.dtype()) {
        (DType::F32, DType::F32) => {
            same_dtype_binary!(buffers, lhs, rhs, f32, typed_sub_with_pool, "sub")
        }
        (DType::F64, DType::F64) => {
            same_dtype_binary!(buffers, lhs, rhs, f64, typed_sub_with_pool, "sub")
        }
        (DType::I32, DType::I32) => {
            same_dtype_binary!(buffers, lhs, rhs, i32, typed_wrapping_sub_with_pool, "sub")
        }
        (DType::I64, DType::I64) => {
            same_dtype_binary!(buffers, lhs, rhs, i64, typed_wrapping_sub_with_pool, "sub")
        }
        (DType::C32, DType::C32) => {
            same_dtype_binary!(buffers, lhs, rhs, Complex<f32>, typed_sub_with_pool, "sub")
        }
        (DType::C64, DType::C64) => {
            same_dtype_binary!(buffers, lhs, rhs, Complex<f64>, typed_sub_with_pool, "sub")
        }
        (DType::F32, DType::C32)
            if pair_operand::<f32>("sub", lhs, rhs, lhs)?
                .shape()
                .is_empty() =>
        {
            let value = typed_host_data("sub", pair_operand::<f32>("sub", lhs, rhs, lhs)?)?[0];
            let scalar = complex_scalar_tensor(value)?;
            let other = pair_operand::<Complex<f32>>("sub", lhs, rhs, rhs)?;
            let out = typed_sub_with_pool(buffers, &scalar, other)?;
            Ok(Tensor::from_typed::<Complex<f32>>(out))
        }
        (DType::C32, DType::F32)
            if pair_operand::<f32>("sub", lhs, rhs, rhs)?
                .shape()
                .is_empty() =>
        {
            let value = typed_host_data("sub", pair_operand::<f32>("sub", lhs, rhs, rhs)?)?[0];
            let scalar = complex_scalar_tensor(value)?;
            let first = pair_operand::<Complex<f32>>("sub", lhs, rhs, lhs)?;
            let out = typed_sub_with_pool(buffers, first, &scalar)?;
            Ok(Tensor::from_typed::<Complex<f32>>(out))
        }
        (DType::F64, DType::C64)
            if pair_operand::<f64>("sub", lhs, rhs, lhs)?
                .shape()
                .is_empty() =>
        {
            let value = typed_host_data("sub", pair_operand::<f64>("sub", lhs, rhs, lhs)?)?[0];
            let scalar = complex_scalar_tensor(value)?;
            let other = pair_operand::<Complex<f64>>("sub", lhs, rhs, rhs)?;
            let out = typed_sub_with_pool(buffers, &scalar, other)?;
            Ok(Tensor::from_typed::<Complex<f64>>(out))
        }
        (DType::C64, DType::F64)
            if pair_operand::<f64>("sub", lhs, rhs, rhs)?
                .shape()
                .is_empty() =>
        {
            let value = typed_host_data("sub", pair_operand::<f64>("sub", lhs, rhs, rhs)?)?[0];
            let scalar = complex_scalar_tensor(value)?;
            let first = pair_operand::<Complex<f64>>("sub", lhs, rhs, lhs)?;
            let out = typed_sub_with_pool(buffers, first, &scalar)?;
            Ok(Tensor::from_typed::<Complex<f64>>(out))
        }
        _ => Err(tensor_pair_error("sub", lhs, rhs)),
    }
}

#[doc(hidden)]
pub fn sub_read_with_pool(
    buffers: &mut BufferPool,
    lhs: TensorRead<'_>,
    rhs: TensorRead<'_>,
) -> crate::Result<Tensor> {
    if let (TensorRead::Tensor(lhs), TensorRead::Tensor(rhs)) = (&lhs, &rhs) {
        return sub_with_pool(buffers, lhs, rhs);
    }

    dispatch_read_presets!(
        buffers,
        lhs,
        rhs,
        typed_sub_view_with_pool,
        typed_wrapping_sub_view_with_pool
    );

    Err(read_pair_error("sub", lhs, rhs))
}

/// Multiply two CPU tensors elementwise.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::mul;
/// use tenferro_tensor::Tensor;
///
/// let a = Tensor::from_vec_col_major(vec![2], vec![2.0_f64, 3.0])?;
/// let b = Tensor::from_vec_col_major(vec![2], vec![4.0_f64, 5.0])?;
/// let out = mul(&a, &b)?;
/// assert_eq!(out.as_slice::<f64>().unwrap(), &[8.0, 15.0]);
/// # Ok::<(), tenferro_tensor::Error>(())
/// ```
#[cfg(test)]
#[doc(hidden)]
pub fn mul(lhs: &Tensor, rhs: &Tensor) -> crate::Result<Tensor> {
    with_test_pool(|buffers| mul_with_pool(buffers, lhs, rhs))
}

fn binary_read_with_pool(
    op: &'static str,
    buffers: &mut BufferPool,
    lhs: TensorRead<'_>,
    rhs: TensorRead<'_>,
    f: impl FnOnce(&mut BufferPool, &Tensor, &Tensor) -> crate::Result<Tensor>,
) -> crate::Result<Tensor> {
    if let (Some(lhs), Some(rhs)) = (lhs.as_tensor(), rhs.as_tensor()) {
        return f(buffers, lhs, rhs);
    }

    Err(read_pair_error(op, lhs, rhs))
}

#[doc(hidden)]
pub fn mul_with_pool(
    buffers: &mut BufferPool,
    lhs: &Tensor,
    rhs: &Tensor,
) -> crate::Result<Tensor> {
    match (lhs.dtype(), rhs.dtype()) {
        (DType::F32, DType::F32) => {
            same_dtype_binary!(buffers, lhs, rhs, f32, typed_mul_with_pool, "mul")
        }
        (DType::F64, DType::F64) => {
            same_dtype_binary!(buffers, lhs, rhs, f64, typed_mul_with_pool, "mul")
        }
        (DType::I32, DType::I32) => {
            same_dtype_binary!(buffers, lhs, rhs, i32, typed_wrapping_mul_with_pool, "mul")
        }
        (DType::I64, DType::I64) => {
            same_dtype_binary!(buffers, lhs, rhs, i64, typed_wrapping_mul_with_pool, "mul")
        }
        (DType::C32, DType::C32) => {
            same_dtype_binary!(buffers, lhs, rhs, Complex<f32>, typed_mul_with_pool, "mul")
        }
        (DType::C64, DType::C64) => {
            same_dtype_binary!(buffers, lhs, rhs, Complex<f64>, typed_mul_with_pool, "mul")
        }
        (DType::F32, DType::C32)
            if pair_operand::<f32>("mul", lhs, rhs, lhs)?
                .shape()
                .is_empty() =>
        {
            let value = typed_host_data("mul", pair_operand::<f32>("mul", lhs, rhs, lhs)?)?[0];
            let scalar = complex_scalar_tensor(value)?;
            let other = pair_operand::<Complex<f32>>("mul", lhs, rhs, rhs)?;
            let out = typed_mul_with_pool(buffers, &scalar, other)?;
            Ok(Tensor::from_typed::<Complex<f32>>(out))
        }
        (DType::C32, DType::F32)
            if pair_operand::<f32>("mul", lhs, rhs, rhs)?
                .shape()
                .is_empty() =>
        {
            let value = typed_host_data("mul", pair_operand::<f32>("mul", lhs, rhs, rhs)?)?[0];
            let scalar = complex_scalar_tensor(value)?;
            let first = pair_operand::<Complex<f32>>("mul", lhs, rhs, lhs)?;
            let out = typed_mul_with_pool(buffers, first, &scalar)?;
            Ok(Tensor::from_typed::<Complex<f32>>(out))
        }
        (DType::F64, DType::C64)
            if pair_operand::<f64>("mul", lhs, rhs, lhs)?
                .shape()
                .is_empty() =>
        {
            let value = typed_host_data("mul", pair_operand::<f64>("mul", lhs, rhs, lhs)?)?[0];
            let scalar = complex_scalar_tensor(value)?;
            let other = pair_operand::<Complex<f64>>("mul", lhs, rhs, rhs)?;
            let out = typed_mul_with_pool(buffers, &scalar, other)?;
            Ok(Tensor::from_typed::<Complex<f64>>(out))
        }
        (DType::C64, DType::F64)
            if pair_operand::<f64>("mul", lhs, rhs, rhs)?
                .shape()
                .is_empty() =>
        {
            let value = typed_host_data("mul", pair_operand::<f64>("mul", lhs, rhs, rhs)?)?[0];
            let scalar = complex_scalar_tensor(value)?;
            let first = pair_operand::<Complex<f64>>("mul", lhs, rhs, lhs)?;
            let out = typed_mul_with_pool(buffers, first, &scalar)?;
            Ok(Tensor::from_typed::<Complex<f64>>(out))
        }
        _ => Err(tensor_pair_error("mul", lhs, rhs)),
    }
}

#[doc(hidden)]
pub fn mul_read_with_pool(
    buffers: &mut BufferPool,
    lhs: TensorRead<'_>,
    rhs: TensorRead<'_>,
) -> crate::Result<Tensor> {
    if let (TensorRead::Tensor(lhs), TensorRead::Tensor(rhs)) = (&lhs, &rhs) {
        return mul_with_pool(buffers, lhs, rhs);
    }

    dispatch_read_presets!(
        buffers,
        lhs,
        rhs,
        typed_mul_view_with_pool,
        typed_wrapping_mul_view_with_pool
    );

    binary_read_with_pool("mul", buffers, lhs, rhs, mul_with_pool)
}

enum CpuReadView<'a> {
    F32(TypedTensorView<'a, f32>),
    F64(TypedTensorView<'a, f64>),
    I32(TypedTensorView<'a, i32>),
    I64(TypedTensorView<'a, i64>),
    Bool(TypedTensorView<'a, bool>),
    C32(TypedTensorView<'a, Complex<f32>>),
    C64(TypedTensorView<'a, Complex<f64>>),
}

/// The typed tensor behind `input`, or this module's refusal when the dtype is a caller-owned payload.
///
/// The read view has no externally defined variant, so a payload the caller owns is reported instead of
/// being unwrapped.
fn read_typed<'a, T: tenferro_tensor::TensorScalar>(
    input: &'a Tensor,
) -> crate::Result<TypedTensorView<'a, T>> {
    let dtype = input.dtype();
    input
        .as_typed::<T>()
        .map(|tensor| tensor.as_view())
        .ok_or_else(|| {
            crate::Error::unsupported_dtype(
                "read_as_cpu_view",
                dtype,
                "the CPU read view covers the preset scalars",
            )
        })
}

fn read_as_cpu_view(input: TensorRead<'_>) -> crate::Result<CpuReadView<'_>> {
    match input {
        TensorRead::Tensor(tensor) => match tensor.dtype() {
            DType::F32 => Ok(CpuReadView::F32(read_typed::<f32>(tensor)?)),
            DType::F64 => Ok(CpuReadView::F64(read_typed::<f64>(tensor)?)),
            DType::I32 => Ok(CpuReadView::I32(read_typed::<i32>(tensor)?)),
            DType::I64 => Ok(CpuReadView::I64(read_typed::<i64>(tensor)?)),
            DType::Bool => Ok(CpuReadView::Bool(read_typed::<bool>(tensor)?)),
            DType::C32 => Ok(CpuReadView::C32(read_typed::<tenferro_tensor::Complex32>(
                tensor,
            )?)),
            DType::C64 => Ok(CpuReadView::C64(read_typed::<tenferro_tensor::Complex64>(
                tensor,
            )?)),
            DType::External(..) => Err(crate::Error::unsupported_dtype(
                "read_as_cpu_view",
                tensor.dtype(),
                "the CPU read view covers the preset scalars",
            )),
        },
        TensorRead::View(TensorView::F32(view)) => Ok(CpuReadView::F32(view)),
        TensorRead::View(TensorView::F64(view)) => Ok(CpuReadView::F64(view)),
        TensorRead::View(TensorView::I32(view)) => Ok(CpuReadView::I32(view)),
        TensorRead::View(TensorView::I64(view)) => Ok(CpuReadView::I64(view)),
        TensorRead::View(TensorView::Bool(view)) => Ok(CpuReadView::Bool(view)),
        TensorRead::View(TensorView::C32(view)) => Ok(CpuReadView::C32(view)),
        TensorRead::View(TensorView::C64(view)) => Ok(CpuReadView::C64(view)),
    }
}

// Owned/read entrypoints share these static replay bodies, not function pointers.
fn replay_unary<T: Copy + Send + Sync, O: Copy + PoolScalar>(
    op: &'static str,
    out: &mut strided_kernel::StridedViewMut<'_, MaybeUninit<O>>,
    input: &StridedView<'_, T>,
    f: impl Fn(T) -> O + Copy + Sync,
) -> crate::Result<()> {
    map_into(out, input, |x| MaybeUninit::new(f(x)))
        .map_err(|err| crate::Error::backend_source(op, err))
}

fn replay_select<T: Copy + PoolScalar>(
    out: &mut strided_kernel::StridedViewMut<'_, MaybeUninit<T>>,
    pred: &StridedView<'_, bool>,
    yes: &StridedView<'_, T>,
    no: &StridedView<'_, T>,
) -> crate::Result<()> {
    zip_map3_into(out, pred, yes, no, |p, t, f| {
        MaybeUninit::new(if p { t } else { f })
    })
    .map_err(|err| crate::Error::backend_source("select", err))
}

fn replay_clamp<T: OrderedElem + PoolScalar>(
    out: &mut strided_kernel::StridedViewMut<'_, MaybeUninit<T>>,
    input: &StridedView<'_, T>,
    lower: &StridedView<'_, T>,
    upper: &StridedView<'_, T>,
) -> crate::Result<()> {
    zip_map3_into(out, input, lower, upper, |x, lo, hi| {
        MaybeUninit::new(hi.min_elem(lo.max_elem(x)))
    })
    .map_err(|err| crate::Error::backend_source("clamp", err))
}

fn replay_binary<T: Copy + Send + Sync, O: Copy + Send + Sync>(
    op: &'static str,
    out: &mut strided_kernel::StridedViewMut<'_, MaybeUninit<O>>,
    lhs: &StridedView<'_, T>,
    rhs: &StridedView<'_, T>,
    f: impl Fn(T, T) -> O + Copy + Sync,
) -> crate::Result<()> {
    zip_map2_into(out, lhs, rhs, |a, b| MaybeUninit::new(f(a, b)))
        .map_err(|err| crate::Error::backend_source(op, err))
}

fn replay_scalar_left<T: Copy + Send + Sync>(
    op: &'static str,
    out: &mut strided_kernel::StridedViewMut<'_, MaybeUninit<T>>,
    input: &StridedView<'_, T>,
    scalar: T,
    f: impl Fn(T, T) -> T + Copy + Sync,
) -> crate::Result<()> {
    map_into(out, input, |x| MaybeUninit::new(f(scalar, x)))
        .map_err(|err| crate::Error::backend_source(op, err))
}

fn replay_scalar_right<T: Copy + Send + Sync>(
    op: &'static str,
    out: &mut strided_kernel::StridedViewMut<'_, MaybeUninit<T>>,
    input: &StridedView<'_, T>,
    scalar: T,
    f: impl Fn(T, T) -> T + Copy + Sync,
) -> crate::Result<()> {
    map_into(out, input, |x| MaybeUninit::new(f(x, scalar)))
        .map_err(|err| crate::Error::backend_source(op, err))
}

fn typed_binary_view_with_pool<T, L, R>(
    op: &'static str,
    buffers: &mut BufferPool,
    lhs: &TypedTensorView<'_, T, L>,
    rhs: &TypedTensorView<'_, T, R>,
    f: impl Fn(T, T) -> T + Copy + Sync,
) -> crate::Result<TypedTensor<T>>
where
    T: Copy + PoolScalar + 'static,
    L: TensorRank,
    R: TensorRank,
{
    if lhs.shape() == rhs.shape() {
        let mut out = PooledUninitOutput::<T>::new(buffers, lhs.shape().to_vec())?;
        replay_binary(
            op,
            &mut out.as_uninit_view_mut()?,
            &typed_view_from_view(op, lhs)?,
            &typed_view_from_view(op, rhs)?,
            f,
        )?;
        // SAFETY: the successful runtime-selected zip/map replay writes every logical destination element and retains no destination view.
        Ok(unsafe { out.assume_init()? })
    } else if lhs.shape().is_empty() {
        let scalar = typed_view_from_view(op, lhs)?.get(&[]);
        let mut out = PooledUninitOutput::<T>::new(buffers, rhs.shape().to_vec())?;
        replay_scalar_left(
            op,
            &mut out.as_uninit_view_mut()?,
            &typed_view_from_view(op, rhs)?,
            scalar,
            f,
        )?;
        // SAFETY: the successful runtime-selected scalar-map replay writes every logical destination element and retains no destination view.
        Ok(unsafe { out.assume_init()? })
    } else if rhs.shape().is_empty() {
        let scalar = typed_view_from_view(op, rhs)?.get(&[]);
        let mut out = PooledUninitOutput::<T>::new(buffers, lhs.shape().to_vec())?;
        replay_scalar_right(
            op,
            &mut out.as_uninit_view_mut()?,
            &typed_view_from_view(op, lhs)?,
            scalar,
            f,
        )?;
        // SAFETY: the successful scalar map replay writes every logical destination element and retains no destination view.
        Ok(unsafe { out.assume_init()? })
    } else {
        Err(crate::Error::shape_mismatch(
            op,
            lhs.shape().to_vec(),
            rhs.shape().to_vec(),
        ))
    }
}

fn typed_unary_view_with_pool<T, R>(
    op: &'static str,
    buffers: &mut BufferPool,
    input: &TypedTensorView<'_, T, R>,
    f: impl Fn(T) -> T + Copy + Sync,
) -> crate::Result<TypedTensor<T>>
where
    T: Copy + PoolScalar + 'static,
    R: TensorRank,
{
    let mut out = PooledUninitOutput::<T>::new(buffers, input.shape().to_vec())?;
    replay_unary(
        op,
        &mut out.as_uninit_view_mut()?,
        &typed_view_from_view(op, input)?,
        f,
    )?;
    // SAFETY: the successful map replay writes every logical destination element and retains no destination view.
    Ok(unsafe { out.assume_init()? })
}

fn typed_same_shape_binary_view_with_pool<T, O, L, R>(
    op: &'static str,
    buffers: &mut BufferPool,
    lhs: &TypedTensorView<'_, T, L>,
    rhs: &TypedTensorView<'_, T, R>,
    f: impl Fn(T, T) -> O + Copy + Sync,
) -> crate::Result<TypedTensor<O>>
where
    T: Copy + Send + Sync + TensorScalar + 'static,
    O: Copy + PoolScalar,
    L: TensorRank,
    R: TensorRank,
{
    if lhs.shape() != rhs.shape() {
        return Err(crate::Error::shape_mismatch(
            op,
            lhs.shape().to_vec(),
            rhs.shape().to_vec(),
        ));
    }
    let mut out = PooledUninitOutput::<O>::new(buffers, lhs.shape().to_vec())?;
    replay_binary(
        op,
        &mut out.as_uninit_view_mut()?,
        &typed_view_from_view(op, lhs)?,
        &typed_view_from_view(op, rhs)?,
        f,
    )?;
    // SAFETY: the successful zip replay writes every logical destination element and retains no destination view.
    Ok(unsafe { out.assume_init()? })
}

fn typed_ordered_compare_view_with_pool<T, L, R>(
    buffers: &mut BufferPool,
    lhs: &TypedTensorView<'_, T, L>,
    rhs: &TypedTensorView<'_, T, R>,
    dir: &CompareDir,
) -> crate::Result<TypedTensor<bool>>
where
    T: Copy + Send + Sync + PartialOrd + 'static,
    L: TensorRank,
    R: TensorRank,
{
    if lhs.shape() != rhs.shape() {
        return Err(crate::Error::shape_mismatch(
            "compare",
            lhs.shape().to_vec(),
            rhs.shape().to_vec(),
        ));
    }
    let op = match dir {
        CompareDir::Eq => CompareOp::Eq,
        CompareDir::Lt => CompareOp::Lt,
        CompareDir::Le => CompareOp::Le,
        CompareDir::Gt => CompareOp::Gt,
        CompareDir::Ge => CompareOp::Ge,
    };
    let mut out = PooledUninitOutput::<bool>::new(buffers, lhs.shape().to_vec())?;
    compare_into_uninit(
        &mut out.as_uninit_view_mut()?,
        &typed_view_from_view("compare", lhs)?,
        &typed_view_from_view("compare", rhs)?,
        op,
    )
    .map_err(|err| crate::Error::backend_source("compare", err))?;
    // SAFETY: the successful compare replay writes every logical destination element and retains no destination view.
    Ok(unsafe { out.assume_init()? })
}

fn typed_select_view_with_pool<T, P, A, B>(
    buffers: &mut BufferPool,
    pred: &TypedTensorView<'_, bool, P>,
    on_true: &TypedTensorView<'_, T, A>,
    on_false: &TypedTensorView<'_, T, B>,
) -> crate::Result<TypedTensor<T>>
where
    T: Copy + PoolScalar + 'static,
    P: TensorRank,
    A: TensorRank,
    B: TensorRank,
{
    if pred.shape() != on_true.shape() {
        return Err(crate::Error::shape_mismatch(
            "select",
            pred.shape().to_vec(),
            on_true.shape().to_vec(),
        ));
    }
    if pred.shape() != on_false.shape() {
        return Err(crate::Error::shape_mismatch(
            "select",
            pred.shape().to_vec(),
            on_false.shape().to_vec(),
        ));
    }
    let mut out = PooledUninitOutput::<T>::new(buffers, pred.shape().to_vec())?;
    replay_select(
        &mut out.as_uninit_view_mut()?,
        &typed_view_from_view("select", pred)?,
        &typed_view_from_view("select", on_true)?,
        &typed_view_from_view("select", on_false)?,
    )?;
    // SAFETY: the successful select replay writes every logical destination element and retains no destination view.
    Ok(unsafe { out.assume_init()? })
}

fn typed_clamp_view_with_pool<T, I, L, U>(
    buffers: &mut BufferPool,
    input: &TypedTensorView<'_, T, I>,
    lower: &TypedTensorView<'_, T, L>,
    upper: &TypedTensorView<'_, T, U>,
) -> crate::Result<TypedTensor<T>>
where
    T: OrderedElem + PoolScalar + 'static,
    I: TensorRank,
    L: TensorRank,
    U: TensorRank,
{
    if input.shape() != lower.shape() {
        return Err(crate::Error::shape_mismatch(
            "clamp",
            input.shape().to_vec(),
            lower.shape().to_vec(),
        ));
    }
    if input.shape() != upper.shape() {
        return Err(crate::Error::shape_mismatch(
            "clamp",
            input.shape().to_vec(),
            upper.shape().to_vec(),
        ));
    }
    let mut out = PooledUninitOutput::<T>::new(buffers, input.shape().to_vec())?;
    replay_clamp(
        &mut out.as_uninit_view_mut()?,
        &typed_view_from_view("clamp", input)?,
        &typed_view_from_view("clamp", lower)?,
        &typed_view_from_view("clamp", upper)?,
    )?;
    // SAFETY: the successful clamp replay writes every logical destination element and retains no destination view.
    Ok(unsafe { out.assume_init()? })
}

#[derive(Clone, Copy)]
enum SplitOuterProductLayout {
    LhsPrefix,
    RhsPrefix,
}

struct SplitOuterProductPlan {
    #[allow(dead_code)]
    rows: usize,
    #[allow(dead_code)]
    cols: usize,
    #[allow(dead_code)]
    batches: usize,
    layout: SplitOuterProductLayout,
    lhs_free_axes: Vec<usize>,
    rhs_free_axes: Vec<usize>,
    lhs_batch_axes: Vec<usize>,
    rhs_batch_axes: Vec<usize>,
}

struct OuterProductAxisPartition {
    lhs_free_output_axes: Vec<usize>,
    rhs_free_output_axes: Vec<usize>,
    batch_output_axes: Vec<usize>,
    lhs_free_axes: Vec<usize>,
    rhs_free_axes: Vec<usize>,
    lhs_batch_axes: Vec<usize>,
    rhs_batch_axes: Vec<usize>,
}

fn shape_matches_dims(source_shape: &[usize], output_shape: &[usize], dims: &[usize]) -> bool {
    source_shape.len() == dims.len()
        && source_shape
            .iter()
            .zip(dims.iter())
            .all(|(&dim, &axis)| output_shape.get(axis).copied() == Some(dim))
}

fn axes_by_output(dims: &[usize], output_rank: usize) -> Option<Vec<Option<usize>>> {
    let mut axes = vec![None; output_rank];
    for (src_axis, &dst_axis) in dims.iter().enumerate() {
        let slot = axes.get_mut(dst_axis)?;
        if slot.replace(src_axis).is_some() {
            return None;
        }
    }
    Some(axes)
}

fn axes_shape_product<T>(
    op: &'static str,
    view: &TypedTensorView<'_, T>,
    axes: &[usize],
) -> crate::Result<usize>
where
    T: 'static,
{
    axes.iter().try_fold(1usize, |acc, &axis| {
        acc.checked_mul(view.shape()[axis]).ok_or_else(|| {
            crate::Error::invalid_argument(op, "shape", "shape size overflows usize")
        })
    })
}

fn classify_outer_product_axes(
    lhs_dims: &[usize],
    rhs_dims: &[usize],
    output_rank: usize,
) -> Option<OuterProductAxisPartition> {
    let lhs_axes_by_output = axes_by_output(lhs_dims, output_rank)?;
    let rhs_axes_by_output = axes_by_output(rhs_dims, output_rank)?;

    let mut lhs_free_output_axes = Vec::new();
    let mut rhs_free_output_axes = Vec::new();
    let mut batch_output_axes = Vec::new();
    let mut lhs_free_axes = Vec::new();
    let mut rhs_free_axes = Vec::new();
    let mut lhs_batch_axes = Vec::new();
    let mut rhs_batch_axes = Vec::new();

    for output_axis in 0..output_rank {
        match (
            lhs_axes_by_output[output_axis],
            rhs_axes_by_output[output_axis],
        ) {
            (Some(lhs_axis), Some(rhs_axis)) => {
                batch_output_axes.push(output_axis);
                lhs_batch_axes.push(lhs_axis);
                rhs_batch_axes.push(rhs_axis);
            }
            (Some(lhs_axis), None) => {
                lhs_free_output_axes.push(output_axis);
                lhs_free_axes.push(lhs_axis);
            }
            (None, Some(rhs_axis)) => {
                rhs_free_output_axes.push(output_axis);
                rhs_free_axes.push(rhs_axis);
            }
            (None, None) => return None,
        }
    }

    Some(OuterProductAxisPartition {
        lhs_free_output_axes,
        rhs_free_output_axes,
        batch_output_axes,
        lhs_free_axes,
        rhs_free_axes,
        lhs_batch_axes,
        rhs_batch_axes,
    })
}

fn output_axes_match_partition(output_rank: usize, groups: &[&[usize]]) -> bool {
    groups
        .iter()
        .flat_map(|group| group.iter().copied())
        .eq(0..output_rank)
}

fn split_outer_product_plan<T>(
    lhs: &TypedTensorView<'_, T>,
    lhs_shape: &[usize],
    lhs_dims: &[usize],
    rhs: &TypedTensorView<'_, T>,
    rhs_shape: &[usize],
    rhs_dims: &[usize],
) -> crate::Result<Option<SplitOuterProductPlan>>
where
    T: 'static,
{
    let output_rank = lhs_shape.len();
    if lhs_shape != rhs_shape
        || !shape_matches_dims(lhs.shape(), lhs_shape, lhs_dims)
        || !shape_matches_dims(rhs.shape(), rhs_shape, rhs_dims)
        || lhs.backend_buffer().is_some()
        || rhs.backend_buffer().is_some()
        || lhs.offset() < 0
        || rhs.offset() < 0
        || lhs.strides().iter().any(|&stride| stride < 0)
        || rhs.strides().iter().any(|&stride| stride < 0)
    {
        return Ok(None);
    }

    let Some(partition) = classify_outer_product_axes(lhs_dims, rhs_dims, output_rank) else {
        return Ok(None);
    };

    let lhs_free_size = axes_shape_product("broadcast_multiply", lhs, &partition.lhs_free_axes)?;
    let rhs_free_size = axes_shape_product("broadcast_multiply", rhs, &partition.rhs_free_axes)?;
    if lhs_free_size <= 1 || rhs_free_size <= 1 {
        return Ok(None);
    }
    let batches = axes_shape_product("broadcast_multiply", lhs, &partition.lhs_batch_axes)?;

    let lhs_prefix = output_axes_match_partition(
        output_rank,
        &[
            &partition.lhs_free_output_axes,
            &partition.rhs_free_output_axes,
            &partition.batch_output_axes,
        ],
    );
    if lhs_prefix {
        return Ok(Some(SplitOuterProductPlan {
            rows: lhs_free_size,
            cols: rhs_free_size,
            batches,
            layout: SplitOuterProductLayout::LhsPrefix,
            lhs_free_axes: partition.lhs_free_axes,
            rhs_free_axes: partition.rhs_free_axes,
            lhs_batch_axes: partition.lhs_batch_axes,
            rhs_batch_axes: partition.rhs_batch_axes,
        }));
    }

    let rhs_prefix = output_axes_match_partition(
        output_rank,
        &[
            &partition.rhs_free_output_axes,
            &partition.lhs_free_output_axes,
            &partition.batch_output_axes,
        ],
    );
    if rhs_prefix {
        return Ok(Some(SplitOuterProductPlan {
            rows: rhs_free_size,
            cols: lhs_free_size,
            batches,
            layout: SplitOuterProductLayout::RhsPrefix,
            lhs_free_axes: partition.lhs_free_axes,
            rhs_free_axes: partition.rhs_free_axes,
            lhs_batch_axes: partition.lhs_batch_axes,
            rhs_batch_axes: partition.rhs_batch_axes,
        }));
    }

    Ok(None)
}

fn try_outer_product_with_pool<T>(
    buffers: &mut BufferPool,
    lhs: &TypedTensorView<'_, T>,
    lhs_shape: &[usize],
    lhs_dims: &[usize],
    rhs: &TypedTensorView<'_, T>,
    rhs_shape: &[usize],
    rhs_dims: &[usize],
) -> crate::Result<Option<TypedTensor<T>>>
where
    T: Copy + Clone + Mul<Output = T> + PoolScalar + 'static,
{
    let Some(plan) = split_outer_product_plan(lhs, lhs_shape, lhs_dims, rhs, rhs_shape, rhs_dims)?
    else {
        return Ok(None);
    };

    let mut out = PooledUninitOutput::<T>::new(buffers, lhs_shape.to_vec())?;
    let lhs_view = typed_view_from_view("broadcast_multiply", lhs)?;
    let rhs_view = typed_view_from_view("broadcast_multiply", rhs)?;
    match plan.layout {
        SplitOuterProductLayout::LhsPrefix => {
            let lhs_perm: Vec<_> = plan
                .lhs_free_axes
                .iter()
                .chain(plan.lhs_batch_axes.iter())
                .copied()
                .collect();
            let rhs_perm: Vec<_> = plan
                .rhs_free_axes
                .iter()
                .chain(plan.rhs_batch_axes.iter())
                .copied()
                .collect();
            let lhs_outer = lhs_view
                .permute(&lhs_perm)
                .map_err(|err| crate::Error::backend_source("broadcast_multiply", err))?;
            let rhs_outer = rhs_view
                .permute(&rhs_perm)
                .map_err(|err| crate::Error::backend_source("broadcast_multiply", err))?;
            batched_outer_product_into_uninit(
                &mut out.as_uninit_view_mut()?,
                &lhs_outer,
                &rhs_outer,
                plan.lhs_free_axes.len(),
                plan.rhs_free_axes.len(),
            )
            .map_err(|err| crate::Error::backend_source("broadcast_multiply", err))?;
        }
        SplitOuterProductLayout::RhsPrefix => {
            let lhs_perm: Vec<_> = plan
                .lhs_free_axes
                .iter()
                .chain(plan.lhs_batch_axes.iter())
                .copied()
                .collect();
            let rhs_perm: Vec<_> = plan
                .rhs_free_axes
                .iter()
                .chain(plan.rhs_batch_axes.iter())
                .copied()
                .collect();
            let lhs_outer = lhs_view
                .permute(&lhs_perm)
                .map_err(|err| crate::Error::backend_source("broadcast_multiply", err))?;
            let rhs_outer = rhs_view
                .permute(&rhs_perm)
                .map_err(|err| crate::Error::backend_source("broadcast_multiply", err))?;
            batched_outer_product_into_uninit(
                &mut out.as_uninit_view_mut()?,
                &rhs_outer,
                &lhs_outer,
                plan.rhs_free_axes.len(),
                plan.lhs_free_axes.len(),
            )
            .map_err(|err| crate::Error::backend_source("broadcast_multiply", err))?;
        }
    }
    // SAFETY: the successful outer-product replay writes every logical destination element and retains no destination view.
    Ok(Some(unsafe { out.assume_init()? }))
}

struct LazyOuterProduct<T> {
    base: TypedTensor<T>,
    shape: Vec<usize>,
    strides: Vec<isize>,
}

fn axes_by_physical_stride<T>(view: &TypedTensorView<'_, T>, axes: &[usize]) -> Vec<usize>
where
    T: 'static,
{
    let mut sorted = axes.to_vec();
    sorted.sort_by(|&lhs_axis, &rhs_axis| {
        view.strides()[lhs_axis]
            .cmp(&view.strides()[rhs_axis])
            .then_with(|| lhs_axis.cmp(&rhs_axis))
    });
    sorted
}

fn append_axis_shapes<T>(shape: &mut Vec<usize>, view: &TypedTensorView<'_, T>, axes: &[usize])
where
    T: 'static,
{
    shape.extend(axes.iter().map(|&axis| view.shape()[axis]));
}

fn set_lazy_stride(
    logical_strides: &mut [Option<isize>],
    output_axis: usize,
    stride: isize,
) -> crate::Result<()> {
    let rank = logical_strides.len();
    let slot = logical_strides
        .get_mut(output_axis)
        .ok_or(crate::Error::axis_out_of_bounds(
            "broadcast_multiply",
            output_axis,
            rank,
        ))?;
    if slot.replace(stride).is_some() {
        return Err(crate::Error::duplicate_axis(
            "broadcast_multiply",
            output_axis,
            "lazy output layout",
        ));
    }
    Ok(())
}

struct LazyOuterProductStrideSpec<'a> {
    output_shape: &'a [usize],
    base_shape: &'a [usize],
    leading_axes: &'a [usize],
    leading_dims: &'a [usize],
    trailing_axes: &'a [usize],
    trailing_dims: &'a [usize],
    lhs_batch_axes: &'a [usize],
    rhs_batch_axes: &'a [usize],
    lhs_dims: &'a [usize],
    rhs_dims: &'a [usize],
}

fn lazy_outer_product_strides(spec: LazyOuterProductStrideSpec<'_>) -> crate::Result<Vec<isize>> {
    let base_strides = col_major_strides(spec.base_shape)?;
    let mut logical_strides = vec![None; spec.output_shape.len()];
    let mut base_axis = 0usize;

    for &axis in spec.leading_axes {
        set_lazy_stride(
            &mut logical_strides,
            spec.leading_dims[axis],
            base_strides[base_axis],
        )?;
        base_axis += 1;
    }
    for &axis in spec.trailing_axes {
        set_lazy_stride(
            &mut logical_strides,
            spec.trailing_dims[axis],
            base_strides[base_axis],
        )?;
        base_axis += 1;
    }
    for (&lhs_axis, &rhs_axis) in spec.lhs_batch_axes.iter().zip(spec.rhs_batch_axes.iter()) {
        let output_axis = spec.lhs_dims[lhs_axis];
        if spec.rhs_dims[rhs_axis] != output_axis {
            return Err(crate::Error::invalid_argument(
                "broadcast_multiply",
                "dimensions",
                "batch axes disagree while building lazy outer-product layout",
            ));
        }
        set_lazy_stride(&mut logical_strides, output_axis, base_strides[base_axis])?;
        base_axis += 1;
    }

    logical_strides
        .into_iter()
        .collect::<Option<Vec<_>>>()
        .ok_or_else(|| {
            crate::Error::invalid_argument(
                "broadcast_multiply",
                "dimensions",
                "lazy outer-product layout did not cover every output axis",
            )
        })
}

fn lazy_outer_product_value(
    tensor: Tensor,
    shape: Vec<usize>,
    strides: Vec<isize>,
) -> crate::Result<TensorValue> {
    TensorValue::from_parts(tensor, shape, strides, 0)
}

fn try_lazy_outer_product_with_pool<T>(
    buffers: &mut BufferPool,
    lhs: &TypedTensorView<'_, T>,
    lhs_shape: &[usize],
    lhs_dims: &[usize],
    rhs: &TypedTensorView<'_, T>,
    rhs_shape: &[usize],
    rhs_dims: &[usize],
) -> crate::Result<Option<LazyOuterProduct<T>>>
where
    T: Copy + Clone + Mul<Output = T> + PoolScalar + 'static,
{
    let Some(plan) = split_outer_product_plan(lhs, lhs_shape, lhs_dims, rhs, rhs_shape, rhs_dims)?
    else {
        return Ok(None);
    };

    let lhs_free_axes = axes_by_physical_stride(lhs, &plan.lhs_free_axes);
    let rhs_free_axes = axes_by_physical_stride(rhs, &plan.rhs_free_axes);
    if lhs_free_axes == plan.lhs_free_axes && rhs_free_axes == plan.rhs_free_axes {
        return Ok(None);
    }

    let lhs_view = typed_view_from_view("broadcast_multiply", lhs)?;
    let rhs_view = typed_view_from_view("broadcast_multiply", rhs)?;

    match plan.layout {
        SplitOuterProductLayout::LhsPrefix => {
            let lhs_perm: Vec<_> = lhs_free_axes
                .iter()
                .chain(plan.lhs_batch_axes.iter())
                .copied()
                .collect();
            let rhs_perm: Vec<_> = rhs_free_axes
                .iter()
                .chain(plan.rhs_batch_axes.iter())
                .copied()
                .collect();
            let lhs_outer = lhs_view
                .permute(&lhs_perm)
                .map_err(|err| crate::Error::backend_source("broadcast_multiply", err))?;
            let rhs_outer = rhs_view
                .permute(&rhs_perm)
                .map_err(|err| crate::Error::backend_source("broadcast_multiply", err))?;

            let mut base_shape = Vec::with_capacity(lhs_shape.len());
            append_axis_shapes(&mut base_shape, lhs, &lhs_free_axes);
            append_axis_shapes(&mut base_shape, rhs, &rhs_free_axes);
            append_axis_shapes(&mut base_shape, lhs, &plan.lhs_batch_axes);
            let strides = lazy_outer_product_strides(LazyOuterProductStrideSpec {
                output_shape: lhs_shape,
                base_shape: &base_shape,
                leading_axes: &lhs_free_axes,
                leading_dims: lhs_dims,
                trailing_axes: &rhs_free_axes,
                trailing_dims: rhs_dims,
                lhs_batch_axes: &plan.lhs_batch_axes,
                rhs_batch_axes: &plan.rhs_batch_axes,
                lhs_dims,
                rhs_dims,
            })?;

            let mut base = PooledUninitOutput::<T>::new(buffers, base_shape.clone())?;
            batched_outer_product_into_uninit(
                &mut base.as_uninit_view_mut()?,
                &lhs_outer,
                &rhs_outer,
                lhs_free_axes.len(),
                rhs_free_axes.len(),
            )
            .map_err(|err| crate::Error::backend_source("broadcast_multiply", err))?;
            Ok(Some(LazyOuterProduct {
                // SAFETY: the successful outer-product replay writes every logical base element and retains no destination view.
                base: unsafe { base.assume_init()? },
                shape: lhs_shape.to_vec(),
                strides,
            }))
        }
        SplitOuterProductLayout::RhsPrefix => {
            let lhs_perm: Vec<_> = lhs_free_axes
                .iter()
                .chain(plan.lhs_batch_axes.iter())
                .copied()
                .collect();
            let rhs_perm: Vec<_> = rhs_free_axes
                .iter()
                .chain(plan.rhs_batch_axes.iter())
                .copied()
                .collect();
            let lhs_outer = lhs_view
                .permute(&lhs_perm)
                .map_err(|err| crate::Error::backend_source("broadcast_multiply", err))?;
            let rhs_outer = rhs_view
                .permute(&rhs_perm)
                .map_err(|err| crate::Error::backend_source("broadcast_multiply", err))?;

            let mut base_shape = Vec::with_capacity(lhs_shape.len());
            append_axis_shapes(&mut base_shape, rhs, &rhs_free_axes);
            append_axis_shapes(&mut base_shape, lhs, &lhs_free_axes);
            append_axis_shapes(&mut base_shape, lhs, &plan.lhs_batch_axes);
            let strides = lazy_outer_product_strides(LazyOuterProductStrideSpec {
                output_shape: lhs_shape,
                base_shape: &base_shape,
                leading_axes: &rhs_free_axes,
                leading_dims: rhs_dims,
                trailing_axes: &lhs_free_axes,
                trailing_dims: lhs_dims,
                lhs_batch_axes: &plan.lhs_batch_axes,
                rhs_batch_axes: &plan.rhs_batch_axes,
                lhs_dims,
                rhs_dims,
            })?;

            let mut base = PooledUninitOutput::<T>::new(buffers, base_shape.clone())?;
            batched_outer_product_into_uninit(
                &mut base.as_uninit_view_mut()?,
                &rhs_outer,
                &lhs_outer,
                rhs_free_axes.len(),
                lhs_free_axes.len(),
            )
            .map_err(|err| crate::Error::backend_source("broadcast_multiply", err))?;
            Ok(Some(LazyOuterProduct {
                // SAFETY: the successful outer-product replay writes every logical base element and retains no destination view.
                base: unsafe { base.assume_init()? },
                shape: lhs_shape.to_vec(),
                strides,
            }))
        }
    }
}

#[allow(clippy::too_many_arguments)]
fn typed_broadcast_mul_view_with_pool<T, L, R>(
    buffers: &mut BufferPool,
    lhs: &TypedTensorView<'_, T, L>,
    lhs_shape: &[usize],
    lhs_dims: &[usize],
    rhs: &TypedTensorView<'_, T, R>,
    rhs_shape: &[usize],
    rhs_dims: &[usize],
    mul: impl Fn(T, T) -> T + Copy + Sync,
) -> crate::Result<TypedTensor<T>>
where
    T: Copy + Clone + Zero + Mul<Output = T> + PoolScalar + 'static,
    L: TensorRank,
    R: TensorRank,
{
    if lhs_shape != rhs_shape {
        return Err(crate::Error::shape_mismatch(
            "broadcast_multiply",
            lhs_shape.to_vec(),
            rhs_shape.to_vec(),
        ));
    }
    let output_rank = lhs_shape.len();
    let lhs_is_scalar = lhs.shape().is_empty() && lhs_dims.is_empty();
    let rhs_is_scalar = rhs.shape().is_empty() && rhs_dims.is_empty();
    let lhs_is_full_output =
        lhs.shape() == lhs_shape && lhs_dims.iter().copied().eq(0..output_rank);
    let rhs_is_full_output =
        rhs.shape() == rhs_shape && rhs_dims.iter().copied().eq(0..output_rank);
    if lhs_is_scalar && rhs_is_scalar {
        let lhs_scalar = typed_view_from_view("broadcast_multiply", lhs)?.get(&[]);
        let rhs_scalar = typed_view_from_view("broadcast_multiply", rhs)?.get(&[]);
        return filled_broadcast_multiply_tensor(buffers, lhs_shape, mul(lhs_scalar, rhs_scalar));
    }
    if lhs_is_scalar && rhs_is_full_output {
        let scalar = typed_view_from_view("broadcast_multiply", lhs)?.get(&[]);
        let mut out = PooledUninitOutput::<T>::new(buffers, lhs_shape.to_vec())?;
        map_into(
            &mut out.as_uninit_view_mut()?,
            &typed_view_from_view("broadcast_multiply", rhs)?,
            |x| MaybeUninit::new(mul(scalar, x)),
        )
        .map_err(|err| crate::Error::backend_source("broadcast_multiply", err))?;
        // SAFETY: the successful broadcast map replay writes every logical destination element and retains no destination view.
        return Ok(unsafe { out.assume_init()? });
    }
    if rhs_is_scalar && lhs_is_full_output {
        let scalar = typed_view_from_view("broadcast_multiply", rhs)?.get(&[]);
        let mut out = PooledUninitOutput::<T>::new(buffers, lhs_shape.to_vec())?;
        map_into(
            &mut out.as_uninit_view_mut()?,
            &typed_view_from_view("broadcast_multiply", lhs)?,
            |x| MaybeUninit::new(mul(x, scalar)),
        )
        .map_err(|err| crate::Error::backend_source("broadcast_multiply", err))?;
        // SAFETY: the successful broadcast map replay writes every logical destination element and retains no destination view.
        return Ok(unsafe { out.assume_init()? });
    }

    let mut out = PooledUninitOutput::<T>::new(buffers, lhs_shape.to_vec())?;
    let lhs_view = typed_view_from_view("broadcast_multiply", lhs)?;
    let rhs_view = typed_view_from_view("broadcast_multiply", rhs)?;
    broadcast_mul_into_uninit(
        &mut out.as_uninit_view_mut()?,
        &lhs_view,
        lhs_dims,
        &rhs_view,
        rhs_dims,
    )
    .map_err(|err| crate::Error::backend_source("broadcast_multiply", err))?;
    // SAFETY: the successful broadcast replay writes every logical destination element and retains no destination view.
    Ok(unsafe { out.assume_init()? })
}

fn filled_broadcast_multiply_tensor<T>(
    buffers: &mut BufferPool,
    shape: &[usize],
    fill: T,
) -> crate::Result<TypedTensor<T>>
where
    T: Copy + Clone + PoolScalar + 'static,
{
    let mut out = PooledUninitOutput::<T>::new(buffers, shape.to_vec())?;
    for item in out.as_uninit_slice_mut().iter_mut() {
        item.write(fill);
    }
    // SAFETY: the fill replay writes every logical destination element and retains no destination view.
    unsafe { out.assume_init() }
}

#[allow(clippy::too_many_arguments)]
#[doc(hidden)]
pub fn broadcast_multiply_read_with_pool(
    buffers: &mut BufferPool,
    lhs: TensorRead<'_>,
    lhs_shape: &[usize],
    lhs_dims: &[usize],
    rhs: TensorRead<'_>,
    rhs_shape: &[usize],
    rhs_dims: &[usize],
) -> crate::Result<Option<Tensor>> {
    let lhs = read_as_cpu_view(lhs)?;
    let rhs = read_as_cpu_view(rhs)?;

    macro_rules! dispatch {
        ($variant:ident, $lhs:expr, $rhs:expr, $mul:expr) => {{
            if let Some(out) = try_outer_product_with_pool(
                buffers, &$lhs, lhs_shape, lhs_dims, &$rhs, rhs_shape, rhs_dims,
            )? {
                return Ok(Some(Tensor::from_typed::<preset_scalar!($variant)>(out)));
            }
            Ok(Some(Tensor::from_typed::<preset_scalar!($variant)>(
                typed_broadcast_mul_view_with_pool(
                    buffers, &$lhs, lhs_shape, lhs_dims, &$rhs, rhs_shape, rhs_dims, $mul,
                )?,
            )))
        }};
    }

    match (lhs, rhs) {
        (CpuReadView::F32(lhs), CpuReadView::F32(rhs)) => dispatch!(F32, lhs, rhs, |x, y| x * y),
        (CpuReadView::F64(lhs), CpuReadView::F64(rhs)) => dispatch!(F64, lhs, rhs, |x, y| x * y),
        (CpuReadView::I32(lhs), CpuReadView::I32(rhs)) => {
            dispatch!(I32, lhs, rhs, |x, y| x.wrapping_mul(y))
        }
        (CpuReadView::I64(lhs), CpuReadView::I64(rhs)) => {
            dispatch!(I64, lhs, rhs, |x, y| x.wrapping_mul(y))
        }
        (CpuReadView::C32(lhs), CpuReadView::C32(rhs)) => dispatch!(C32, lhs, rhs, |x, y| x * y),
        (CpuReadView::C64(lhs), CpuReadView::C64(rhs)) => dispatch!(C64, lhs, rhs, |x, y| x * y),
        _ => Ok(None),
    }
}

#[allow(clippy::too_many_arguments)]
#[doc(hidden)]
pub fn broadcast_multiply_value_with_pool(
    buffers: &mut BufferPool,
    lhs: TensorRead<'_>,
    lhs_shape: &[usize],
    lhs_dims: &[usize],
    rhs: TensorRead<'_>,
    rhs_shape: &[usize],
    rhs_dims: &[usize],
) -> crate::Result<Option<TensorValue>> {
    broadcast_multiply_value_with_pool_and_tag(
        buffers,
        lhs,
        lhs_shape,
        lhs_dims,
        rhs,
        rhs_shape,
        rhs_dims,
        |_| {},
    )
}

#[allow(clippy::too_many_arguments)]
#[doc(hidden)]
pub fn broadcast_multiply_value_with_pool_and_tag(
    buffers: &mut BufferPool,
    lhs: TensorRead<'_>,
    lhs_shape: &[usize],
    lhs_dims: &[usize],
    rhs: TensorRead<'_>,
    rhs_shape: &[usize],
    rhs_dims: &[usize],
    mut tag_output: impl FnMut(&mut Tensor),
) -> crate::Result<Option<TensorValue>> {
    let lhs_view = read_as_cpu_view(lhs.clone())?;
    let rhs_view = read_as_cpu_view(rhs.clone())?;

    macro_rules! dispatch_lazy {
        ($variant:ident, $lhs:expr, $rhs:expr) => {{
            if let Some(out) = try_lazy_outer_product_with_pool(
                buffers, &$lhs, lhs_shape, lhs_dims, &$rhs, rhs_shape, rhs_dims,
            )? {
                let mut base = Tensor::from_typed::<preset_scalar!($variant)>(out.base);
                tag_output(&mut base);
                return Ok(Some(lazy_outer_product_value(
                    base,
                    out.shape,
                    out.strides,
                )?));
            }
        }};
    }

    match (lhs_view, rhs_view) {
        (CpuReadView::F32(lhs_view), CpuReadView::F32(rhs_view)) => {
            dispatch_lazy!(F32, lhs_view, rhs_view);
        }
        (CpuReadView::F64(lhs_view), CpuReadView::F64(rhs_view)) => {
            dispatch_lazy!(F64, lhs_view, rhs_view);
        }
        (CpuReadView::I32(lhs_view), CpuReadView::I32(rhs_view)) => {
            dispatch_lazy!(I32, lhs_view, rhs_view);
        }
        (CpuReadView::I64(lhs_view), CpuReadView::I64(rhs_view)) => {
            dispatch_lazy!(I64, lhs_view, rhs_view);
        }
        (CpuReadView::C32(lhs_view), CpuReadView::C32(rhs_view)) => {
            dispatch_lazy!(C32, lhs_view, rhs_view);
        }
        (CpuReadView::C64(lhs_view), CpuReadView::C64(rhs_view)) => {
            dispatch_lazy!(C64, lhs_view, rhs_view);
        }
        _ => {}
    }

    let mut tensor = broadcast_multiply_read_with_pool(
        buffers, lhs, lhs_shape, lhs_dims, rhs, rhs_shape, rhs_dims,
    )?;
    if let Some(tensor) = &mut tensor {
        tag_output(tensor);
    }
    Ok(tensor.map(TensorValue::from_tensor))
}

/// Divide two CPU tensors elementwise.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::div;
/// use tenferro_tensor::Tensor;
///
/// let a = Tensor::from_vec_col_major(vec![2], vec![8.0_f64, 15.0])?;
/// let b = Tensor::from_vec_col_major(vec![2], vec![2.0_f64, 5.0])?;
/// let out = div(&a, &b)?;
/// assert_eq!(out.as_slice::<f64>().unwrap(), &[4.0, 3.0]);
/// # Ok::<(), tenferro_tensor::Error>(())
/// ```
#[cfg(test)]
#[doc(hidden)]
pub fn div(lhs: &Tensor, rhs: &Tensor) -> crate::Result<Tensor> {
    with_test_pool(|buffers| div_with_pool(buffers, lhs, rhs))
}

#[doc(hidden)]
pub fn div_with_pool(
    buffers: &mut BufferPool,
    lhs: &Tensor,
    rhs: &Tensor,
) -> crate::Result<Tensor> {
    match (lhs.dtype(), rhs.dtype()) {
        (DType::F32, DType::F32) => {
            same_dtype_binary!(buffers, lhs, rhs, f32, typed_div_with_pool, "div")
        }
        (DType::F64, DType::F64) => {
            same_dtype_binary!(buffers, lhs, rhs, f64, typed_div_with_pool, "div")
        }
        (DType::I32, DType::I32) => {
            same_dtype_binary!(buffers, lhs, rhs, i32, typed_integer_div_with_pool, "div")
        }
        (DType::I64, DType::I64) => {
            same_dtype_binary!(buffers, lhs, rhs, i64, typed_integer_div_with_pool, "div")
        }
        (DType::C32, DType::C32) => {
            same_dtype_binary!(buffers, lhs, rhs, Complex<f32>, typed_div_with_pool, "div")
        }
        (DType::C64, DType::C64) => {
            same_dtype_binary!(buffers, lhs, rhs, Complex<f64>, typed_div_with_pool, "div")
        }
        (DType::F32, DType::C32)
            if pair_operand::<f32>("div", lhs, rhs, lhs)?
                .shape()
                .is_empty() =>
        {
            let value = typed_host_data("div", pair_operand::<f32>("div", lhs, rhs, lhs)?)?[0];
            let scalar = complex_scalar_tensor(value)?;
            let other = pair_operand::<Complex<f32>>("div", lhs, rhs, rhs)?;
            let out = typed_div_with_pool(buffers, &scalar, other)?;
            Ok(Tensor::from_typed::<Complex<f32>>(out))
        }
        (DType::C32, DType::F32)
            if pair_operand::<f32>("div", lhs, rhs, rhs)?
                .shape()
                .is_empty() =>
        {
            let value = typed_host_data("div", pair_operand::<f32>("div", lhs, rhs, rhs)?)?[0];
            let scalar = complex_scalar_tensor(value)?;
            let first = pair_operand::<Complex<f32>>("div", lhs, rhs, lhs)?;
            let out = typed_div_with_pool(buffers, first, &scalar)?;
            Ok(Tensor::from_typed::<Complex<f32>>(out))
        }
        (DType::F64, DType::C64)
            if pair_operand::<f64>("div", lhs, rhs, lhs)?
                .shape()
                .is_empty() =>
        {
            let value = typed_host_data("div", pair_operand::<f64>("div", lhs, rhs, lhs)?)?[0];
            let scalar = complex_scalar_tensor(value)?;
            let other = pair_operand::<Complex<f64>>("div", lhs, rhs, rhs)?;
            let out = typed_div_with_pool(buffers, &scalar, other)?;
            Ok(Tensor::from_typed::<Complex<f64>>(out))
        }
        (DType::C64, DType::F64)
            if pair_operand::<f64>("div", lhs, rhs, rhs)?
                .shape()
                .is_empty() =>
        {
            let value = typed_host_data("div", pair_operand::<f64>("div", lhs, rhs, rhs)?)?[0];
            let scalar = complex_scalar_tensor(value)?;
            let first = pair_operand::<Complex<f64>>("div", lhs, rhs, lhs)?;
            let out = typed_div_with_pool(buffers, first, &scalar)?;
            Ok(Tensor::from_typed::<Complex<f64>>(out))
        }
        _ => Err(crate::Error::dtype_mismatch(
            "div",
            lhs.dtype(),
            rhs.dtype(),
        )),
    }
}

#[doc(hidden)]
pub fn div_read_with_pool(
    buffers: &mut BufferPool,
    lhs: TensorRead<'_>,
    rhs: TensorRead<'_>,
) -> crate::Result<Tensor> {
    let lhs_dtype = lhs.dtype();
    let rhs_dtype = rhs.dtype();
    match (read_as_cpu_view(lhs)?, read_as_cpu_view(rhs)?) {
        (CpuReadView::F32(a), CpuReadView::F32(b)) => Ok(Tensor::from_typed::<f32>(
            typed_binary_view_with_pool("div", buffers, &a, &b, Div::div)?,
        )),
        (CpuReadView::F64(a), CpuReadView::F64(b)) => Ok(Tensor::from_typed::<f64>(
            typed_binary_view_with_pool("div", buffers, &a, &b, Div::div)?,
        )),
        (CpuReadView::I32(a), CpuReadView::I32(b)) => Ok(Tensor::from_typed::<i32>(
            typed_integer_div_view_with_pool(buffers, &a, &b)?,
        )),
        (CpuReadView::I64(a), CpuReadView::I64(b)) => Ok(Tensor::from_typed::<i64>(
            typed_integer_div_view_with_pool(buffers, &a, &b)?,
        )),
        (CpuReadView::C32(a), CpuReadView::C32(b)) => Ok(Tensor::from_typed::<Complex<f32>>(
            typed_binary_view_with_pool("div", buffers, &a, &b, Div::div)?,
        )),
        (CpuReadView::C64(a), CpuReadView::C64(b)) => Ok(Tensor::from_typed::<Complex<f64>>(
            typed_binary_view_with_pool("div", buffers, &a, &b, Div::div)?,
        )),
        (CpuReadView::F32(real), CpuReadView::C32(complex)) if real.shape().is_empty() => {
            let scalar = complex_scalar_tensor_from_view(&real)?;
            let scalar = scalar.as_view();
            Ok(Tensor::from_typed::<Complex<f32>>(
                typed_binary_view_with_pool("div", buffers, &scalar, &complex, Div::div)?,
            ))
        }
        (CpuReadView::C32(complex), CpuReadView::F32(real)) if real.shape().is_empty() => {
            let scalar = complex_scalar_tensor_from_view(&real)?;
            let scalar = scalar.as_view();
            Ok(Tensor::from_typed::<Complex<f32>>(
                typed_binary_view_with_pool("div", buffers, &complex, &scalar, Div::div)?,
            ))
        }
        (CpuReadView::F64(real), CpuReadView::C64(complex)) if real.shape().is_empty() => {
            let scalar = complex_scalar_tensor_from_view(&real)?;
            let scalar = scalar.as_view();
            Ok(Tensor::from_typed::<Complex<f64>>(
                typed_binary_view_with_pool("div", buffers, &scalar, &complex, Div::div)?,
            ))
        }
        (CpuReadView::C64(complex), CpuReadView::F64(real)) if real.shape().is_empty() => {
            let scalar = complex_scalar_tensor_from_view(&real)?;
            let scalar = scalar.as_view();
            Ok(Tensor::from_typed::<Complex<f64>>(
                typed_binary_view_with_pool("div", buffers, &complex, &scalar, Div::div)?,
            ))
        }
        _ => Err(crate::Error::dtype_mismatch("div", lhs_dtype, rhs_dtype)),
    }
}

/// Compute elementwise remainders on CPU tensors.
///
/// Integer remainders use wrapping two's-complement arithmetic for the
/// `MIN % -1` edge and return a structured error on zero divisors.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::rem;
/// use tenferro_tensor::Tensor;
///
/// let a = Tensor::from_vec_col_major(vec![2], vec![7_i32, -7])?;
/// let b = Tensor::from_vec_col_major(vec![2], vec![3_i32, 3])?;
/// let out = rem(&a, &b)?;
/// assert_eq!(out.as_slice::<i32>().unwrap(), &[1, -1]);
/// # Ok::<(), tenferro_tensor::Error>(())
/// ```
#[cfg(test)]
#[doc(hidden)]
pub fn rem(lhs: &Tensor, rhs: &Tensor) -> crate::Result<Tensor> {
    with_test_pool(|buffers| rem_with_pool(buffers, lhs, rhs))
}

#[doc(hidden)]
pub fn rem_with_pool(
    buffers: &mut BufferPool,
    lhs: &Tensor,
    rhs: &Tensor,
) -> crate::Result<Tensor> {
    match (lhs.dtype(), rhs.dtype()) {
        (DType::F32, DType::F32) => {
            same_dtype_binary!(buffers, lhs, rhs, f32, typed_rem_with_pool, "rem")
        }
        (DType::F64, DType::F64) => {
            same_dtype_binary!(buffers, lhs, rhs, f64, typed_rem_with_pool, "rem")
        }
        (DType::I32, DType::I32) => {
            same_dtype_binary!(buffers, lhs, rhs, i32, typed_integer_rem_with_pool, "rem")
        }
        (DType::I64, DType::I64) => {
            same_dtype_binary!(buffers, lhs, rhs, i64, typed_integer_rem_with_pool, "rem")
        }
        _ => Err(tensor_pair_error("rem", lhs, rhs)),
    }
}

#[doc(hidden)]
pub fn rem_read_with_pool(
    buffers: &mut BufferPool,
    lhs: TensorRead<'_>,
    rhs: TensorRead<'_>,
) -> crate::Result<Tensor> {
    let lhs_dtype = lhs.dtype();
    let rhs_dtype = rhs.dtype();
    match (read_as_cpu_view(lhs)?, read_as_cpu_view(rhs)?) {
        (CpuReadView::F32(a), CpuReadView::F32(b)) => Ok(Tensor::from_typed::<f32>(
            typed_binary_view_with_pool("rem", buffers, &a, &b, StdRem::rem)?,
        )),
        (CpuReadView::F64(a), CpuReadView::F64(b)) => Ok(Tensor::from_typed::<f64>(
            typed_binary_view_with_pool("rem", buffers, &a, &b, StdRem::rem)?,
        )),
        (CpuReadView::I32(a), CpuReadView::I32(b)) => Ok(Tensor::from_typed::<i32>(
            typed_integer_rem_view_with_pool(buffers, &a, &b)?,
        )),
        (CpuReadView::I64(a), CpuReadView::I64(b)) => Ok(Tensor::from_typed::<i64>(
            typed_integer_rem_view_with_pool(buffers, &a, &b)?,
        )),
        _ => Err(dtype_pair_error("rem", lhs_dtype, rhs_dtype)),
    }
}

/// Negate a CPU tensor elementwise.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::neg;
/// use tenferro_tensor::Tensor;
///
/// let input = Tensor::from_vec_col_major(vec![2], vec![1.0_f64, -2.0])?;
/// let out = neg(&input)?;
/// assert_eq!(out.as_slice::<f64>().unwrap(), &[-1.0, 2.0]);
/// # Ok::<(), tenferro_tensor::Error>(())
/// ```
#[cfg(test)]
#[doc(hidden)]
pub fn neg(input: &Tensor) -> crate::Result<Tensor> {
    with_test_pool(|buffers| neg_with_pool(buffers, input))
}

#[doc(hidden)]
pub fn neg_with_pool(buffers: &mut BufferPool, input: &Tensor) -> crate::Result<Tensor> {
    match input.dtype() {
        DType::F32 => Ok(Tensor::from_typed::<f32>(typed_neg_with_pool(
            buffers,
            input.as_typed::<f32>().ok_or_else(|| {
                unary_dtype_error("neg", input.dtype(), "F32/F64/I32/I64/C32/C64", false)
            })?,
        )?)),
        DType::F64 => Ok(Tensor::from_typed::<f64>(typed_neg_with_pool(
            buffers,
            input.as_typed::<f64>().ok_or_else(|| {
                unary_dtype_error("neg", input.dtype(), "F32/F64/I32/I64/C32/C64", false)
            })?,
        )?)),
        DType::I32 => Ok(Tensor::from_typed::<i32>(typed_wrapping_neg_with_pool(
            buffers,
            input.as_typed::<i32>().ok_or_else(|| {
                unary_dtype_error("neg", input.dtype(), "F32/F64/I32/I64/C32/C64", false)
            })?,
        )?)),
        DType::I64 => Ok(Tensor::from_typed::<i64>(typed_wrapping_neg_with_pool(
            buffers,
            input.as_typed::<i64>().ok_or_else(|| {
                unary_dtype_error("neg", input.dtype(), "F32/F64/I32/I64/C32/C64", false)
            })?,
        )?)),
        DType::Bool | DType::External(_) => Err(unary_dtype_error(
            "neg",
            input.dtype(),
            "F32/F64/I32/I64/C32/C64",
            false,
        )),
        DType::C32 => Ok(Tensor::from_typed::<Complex<f32>>(typed_neg_with_pool(
            buffers,
            input.as_typed::<Complex<f32>>().ok_or_else(|| {
                unary_dtype_error("neg", input.dtype(), "F32/F64/I32/I64/C32/C64", false)
            })?,
        )?)),
        DType::C64 => Ok(Tensor::from_typed::<Complex<f64>>(typed_neg_with_pool(
            buffers,
            input.as_typed::<Complex<f64>>().ok_or_else(|| {
                unary_dtype_error("neg", input.dtype(), "F32/F64/I32/I64/C32/C64", false)
            })?,
        )?)),
    }
}

#[doc(hidden)]
pub fn neg_read_with_pool(
    buffers: &mut BufferPool,
    input: TensorRead<'_>,
) -> crate::Result<Tensor> {
    let dtype = input.dtype();
    match read_as_cpu_view(input)? {
        CpuReadView::F32(t) => Ok(Tensor::from_typed::<f32>(typed_unary_view_with_pool(
            "neg",
            buffers,
            &t,
            Neg::neg,
        )?)),
        CpuReadView::F64(t) => Ok(Tensor::from_typed::<f64>(typed_unary_view_with_pool(
            "neg",
            buffers,
            &t,
            Neg::neg,
        )?)),
        CpuReadView::I32(t) => Ok(Tensor::from_typed::<i32>(typed_unary_view_with_pool(
            "neg",
            buffers,
            &t,
            WrappingIntegerElem::wrapping_neg_elem,
        )?)),
        CpuReadView::I64(t) => Ok(Tensor::from_typed::<i64>(typed_unary_view_with_pool(
            "neg",
            buffers,
            &t,
            WrappingIntegerElem::wrapping_neg_elem,
        )?)),
        CpuReadView::C32(t) => Ok(Tensor::from_typed::<Complex<f32>>(
            typed_unary_view_with_pool("neg", buffers, &t, Neg::neg)?,
        )),
        CpuReadView::C64(t) => Ok(Tensor::from_typed::<Complex<f64>>(
            typed_unary_view_with_pool("neg", buffers, &t, Neg::neg)?,
        )),
        _ => Err(unary_dtype_error(
            "neg",
            dtype,
            "F32/F64/I32/I64/C32/C64",
            false,
        )),
    }
}

/// Conjugate a real or complex CPU tensor elementwise.
///
/// # Examples
///
/// ```
/// use num_complex::Complex64;
/// use tenferro_cpu::conj;
/// use tenferro_tensor::Tensor;
///
/// let input = Tensor::from_vec_col_major(vec![1], vec![Complex64::new(1.0, 2.0)])?;
/// let out = conj(&input)?;
/// assert_eq!(out.as_slice::<Complex<f64>>().unwrap(), &[Complex64::new(1.0, -2.0)]);
/// # Ok::<(), tenferro_tensor::Error>(())
/// ```
#[cfg(test)]
#[doc(hidden)]
pub fn conj(input: &Tensor) -> crate::Result<Tensor> {
    with_test_pool(|buffers| conj_with_pool(buffers, input))
}

#[doc(hidden)]
pub fn conj_with_pool(buffers: &mut BufferPool, input: &Tensor) -> crate::Result<Tensor> {
    match input.dtype() {
        DType::F32 => Ok(Tensor::from_typed::<f32>(typed_conj_with_pool(
            buffers,
            input
                .as_typed::<f32>()
                .ok_or_else(|| unary_dtype_error("conj", input.dtype(), "F32/F64/C32/C64", true))?,
        )?)),
        DType::F64 => Ok(Tensor::from_typed::<f64>(typed_conj_with_pool(
            buffers,
            input
                .as_typed::<f64>()
                .ok_or_else(|| unary_dtype_error("conj", input.dtype(), "F32/F64/C32/C64", true))?,
        )?)),
        DType::I32 | DType::I64 | DType::Bool | DType::External(_) => Err(unary_dtype_error(
            "conj",
            input.dtype(),
            "F32/F64/C32/C64",
            true,
        )),
        DType::C32 => Ok(Tensor::from_typed::<Complex<f32>>(typed_conj_with_pool(
            buffers,
            input
                .as_typed::<Complex<f32>>()
                .ok_or_else(|| unary_dtype_error("conj", input.dtype(), "F32/F64/C32/C64", true))?,
        )?)),
        DType::C64 => Ok(Tensor::from_typed::<Complex<f64>>(typed_conj_with_pool(
            buffers,
            input
                .as_typed::<Complex<f64>>()
                .ok_or_else(|| unary_dtype_error("conj", input.dtype(), "F32/F64/C32/C64", true))?,
        )?)),
    }
}

#[doc(hidden)]
pub fn conj_read_with_pool(
    buffers: &mut BufferPool,
    input: TensorRead<'_>,
) -> crate::Result<Tensor> {
    let dtype = input.dtype();
    match read_as_cpu_view(input)? {
        CpuReadView::F32(t) => Ok(Tensor::from_typed::<f32>(typed_unary_view_with_pool(
            "conj",
            buffers,
            &t,
            ConjElem::conj_elem,
        )?)),
        CpuReadView::F64(t) => Ok(Tensor::from_typed::<f64>(typed_unary_view_with_pool(
            "conj",
            buffers,
            &t,
            ConjElem::conj_elem,
        )?)),
        CpuReadView::C32(t) => Ok(Tensor::from_typed::<Complex<f32>>(
            typed_unary_view_with_pool("conj", buffers, &t, ConjElem::conj_elem)?,
        )),
        CpuReadView::C64(t) => Ok(Tensor::from_typed::<Complex<f64>>(
            typed_unary_view_with_pool("conj", buffers, &t, ConjElem::conj_elem)?,
        )),
        _ => Err(unary_dtype_error("conj", dtype, "F32/F64/C32/C64", true)),
    }
}

/// Compute elementwise absolute values.
///
/// Complex inputs return real magnitudes (`C32 -> F32`, `C64 -> F64`).
///
/// # Examples
///
/// ```
/// use tenferro_cpu::abs;
/// use tenferro_tensor::Tensor;
///
/// let input = Tensor::from_vec_col_major(vec![2], vec![-3.0_f64, 4.0])?;
/// let out = abs(&input)?;
/// assert_eq!(out.as_slice::<f64>().unwrap(), &[3.0, 4.0]);
/// # Ok::<(), tenferro_tensor::Error>(())
/// ```
#[cfg(test)]
#[doc(hidden)]
pub fn abs(input: &Tensor) -> crate::Result<Tensor> {
    with_test_pool(|buffers| abs_with_pool(buffers, input))
}

#[doc(hidden)]
pub fn abs_with_pool(buffers: &mut BufferPool, input: &Tensor) -> crate::Result<Tensor> {
    match input.dtype() {
        DType::F32 => Ok(Tensor::from_typed::<f32>(typed_abs_with_pool(
            buffers,
            input.as_typed::<f32>().ok_or_else(|| {
                unary_dtype_error("abs", input.dtype(), "F32/F64/I32/I64/C32/C64", false)
            })?,
        )?)),
        DType::F64 => Ok(Tensor::from_typed::<f64>(typed_abs_with_pool(
            buffers,
            input.as_typed::<f64>().ok_or_else(|| {
                unary_dtype_error("abs", input.dtype(), "F32/F64/I32/I64/C32/C64", false)
            })?,
        )?)),
        DType::I32 => Ok(Tensor::from_typed::<i32>(typed_wrapping_abs_with_pool(
            buffers,
            input.as_typed::<i32>().ok_or_else(|| {
                unary_dtype_error("abs", input.dtype(), "F32/F64/I32/I64/C32/C64", false)
            })?,
        )?)),
        DType::I64 => Ok(Tensor::from_typed::<i64>(typed_wrapping_abs_with_pool(
            buffers,
            input.as_typed::<i64>().ok_or_else(|| {
                unary_dtype_error("abs", input.dtype(), "F32/F64/I32/I64/C32/C64", false)
            })?,
        )?)),
        DType::Bool | DType::External(_) => Err(unary_dtype_error(
            "abs",
            input.dtype(),
            "F32/F64/I32/I64/C32/C64",
            false,
        )),
        DType::C32 => Ok(Tensor::from_typed::<f32>(typed_complex_abs_with_pool(
            buffers,
            input.as_typed::<Complex<f32>>().ok_or_else(|| {
                unary_dtype_error("abs", input.dtype(), "F32/F64/I32/I64/C32/C64", false)
            })?,
        )?)),
        DType::C64 => Ok(Tensor::from_typed::<f64>(typed_complex_abs_with_pool(
            buffers,
            input.as_typed::<Complex<f64>>().ok_or_else(|| {
                unary_dtype_error("abs", input.dtype(), "F32/F64/I32/I64/C32/C64", false)
            })?,
        )?)),
    }
}

#[doc(hidden)]
pub fn abs_read_with_pool(
    buffers: &mut BufferPool,
    input: TensorRead<'_>,
) -> crate::Result<Tensor> {
    let dtype = input.dtype();
    match read_as_cpu_view(input)? {
        CpuReadView::F32(t) => Ok(Tensor::from_typed::<f32>(typed_unary_view_with_pool(
            "abs",
            buffers,
            &t,
            Tier2Elem::abs_elem,
        )?)),
        CpuReadView::F64(t) => Ok(Tensor::from_typed::<f64>(typed_unary_view_with_pool(
            "abs",
            buffers,
            &t,
            Tier2Elem::abs_elem,
        )?)),
        CpuReadView::I32(t) => Ok(Tensor::from_typed::<i32>(typed_unary_view_with_pool(
            "abs",
            buffers,
            &t,
            WrappingIntegerElem::wrapping_abs_elem,
        )?)),
        CpuReadView::I64(t) => Ok(Tensor::from_typed::<i64>(typed_unary_view_with_pool(
            "abs",
            buffers,
            &t,
            WrappingIntegerElem::wrapping_abs_elem,
        )?)),
        CpuReadView::C32(t) => Ok(Tensor::from_typed::<f32>(typed_complex_abs_view_with_pool(
            buffers, &t,
        )?)),
        CpuReadView::C64(t) => Ok(Tensor::from_typed::<f64>(typed_complex_abs_view_with_pool(
            buffers, &t,
        )?)),
        _ => Err(unary_dtype_error(
            "abs",
            dtype,
            "F32/F64/I32/I64/C32/C64",
            false,
        )),
    }
}

/// Compute elementwise signs.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::sign;
/// use tenferro_tensor::Tensor;
///
/// let input = Tensor::from_vec_col_major(vec![3], vec![-2.0_f64, 0.0, 3.0])?;
/// let out = sign(&input)?;
/// assert_eq!(out.as_slice::<f64>().unwrap(), &[-1.0, 0.0, 1.0]);
/// # Ok::<(), tenferro_tensor::Error>(())
/// ```
#[cfg(test)]
#[doc(hidden)]
pub fn sign(input: &Tensor) -> crate::Result<Tensor> {
    with_test_pool(|buffers| sign_with_pool(buffers, input))
}

#[doc(hidden)]
pub fn sign_with_pool(buffers: &mut BufferPool, input: &Tensor) -> crate::Result<Tensor> {
    match input.dtype() {
        DType::F32 => Ok(Tensor::from_typed::<f32>(typed_sign_with_pool(
            buffers,
            input.as_typed::<f32>().ok_or_else(|| {
                unary_dtype_error("sign", input.dtype(), "F32/F64/I32/I64/C32/C64", false)
            })?,
        )?)),
        DType::F64 => Ok(Tensor::from_typed::<f64>(typed_sign_with_pool(
            buffers,
            input.as_typed::<f64>().ok_or_else(|| {
                unary_dtype_error("sign", input.dtype(), "F32/F64/I32/I64/C32/C64", false)
            })?,
        )?)),
        DType::I32 => Ok(Tensor::from_typed::<i32>(typed_integer_sign_with_pool(
            buffers,
            input.as_typed::<i32>().ok_or_else(|| {
                unary_dtype_error("sign", input.dtype(), "F32/F64/I32/I64/C32/C64", false)
            })?,
        )?)),
        DType::I64 => Ok(Tensor::from_typed::<i64>(typed_integer_sign_with_pool(
            buffers,
            input.as_typed::<i64>().ok_or_else(|| {
                unary_dtype_error("sign", input.dtype(), "F32/F64/I32/I64/C32/C64", false)
            })?,
        )?)),
        DType::Bool | DType::External(_) => Err(unary_dtype_error(
            "sign",
            input.dtype(),
            "F32/F64/I32/I64/C32/C64",
            false,
        )),
        DType::C32 => Ok(Tensor::from_typed::<Complex<f32>>(typed_sign_with_pool(
            buffers,
            input.as_typed::<Complex<f32>>().ok_or_else(|| {
                unary_dtype_error("sign", input.dtype(), "F32/F64/I32/I64/C32/C64", false)
            })?,
        )?)),
        DType::C64 => Ok(Tensor::from_typed::<Complex<f64>>(typed_sign_with_pool(
            buffers,
            input.as_typed::<Complex<f64>>().ok_or_else(|| {
                unary_dtype_error("sign", input.dtype(), "F32/F64/I32/I64/C32/C64", false)
            })?,
        )?)),
    }
}

#[doc(hidden)]
pub fn sign_read_with_pool(
    buffers: &mut BufferPool,
    input: TensorRead<'_>,
) -> crate::Result<Tensor> {
    let dtype = input.dtype();
    match read_as_cpu_view(input)? {
        CpuReadView::F32(t) => Ok(Tensor::from_typed::<f32>(typed_unary_view_with_pool(
            "sign",
            buffers,
            &t,
            Tier2Elem::sign_elem,
        )?)),
        CpuReadView::F64(t) => Ok(Tensor::from_typed::<f64>(typed_unary_view_with_pool(
            "sign",
            buffers,
            &t,
            Tier2Elem::sign_elem,
        )?)),
        CpuReadView::I32(t) => Ok(Tensor::from_typed::<i32>(typed_unary_view_with_pool(
            "sign",
            buffers,
            &t,
            WrappingIntegerElem::signum_elem,
        )?)),
        CpuReadView::I64(t) => Ok(Tensor::from_typed::<i64>(typed_unary_view_with_pool(
            "sign",
            buffers,
            &t,
            WrappingIntegerElem::signum_elem,
        )?)),
        CpuReadView::C32(t) => Ok(Tensor::from_typed::<Complex<f32>>(
            typed_unary_view_with_pool("sign", buffers, &t, Tier2Elem::sign_elem)?,
        )),
        CpuReadView::C64(t) => Ok(Tensor::from_typed::<Complex<f64>>(
            typed_unary_view_with_pool("sign", buffers, &t, Tier2Elem::sign_elem)?,
        )),
        _ => Err(unary_dtype_error(
            "sign",
            dtype,
            "F32/F64/I32/I64/C32/C64",
            false,
        )),
    }
}

/// Compute elementwise maximum values.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::maximum;
/// use tenferro_tensor::Tensor;
///
/// let a = Tensor::from_vec_col_major(vec![2], vec![1.0_f64, 5.0])?;
/// let b = Tensor::from_vec_col_major(vec![2], vec![3.0_f64, 4.0])?;
/// let out = maximum(&a, &b)?;
/// assert_eq!(out.as_slice::<f64>().unwrap(), &[3.0, 5.0]);
/// # Ok::<(), tenferro_tensor::Error>(())
/// ```
#[cfg(test)]
#[doc(hidden)]
pub fn maximum(lhs: &Tensor, rhs: &Tensor) -> crate::Result<Tensor> {
    with_test_pool(|buffers| maximum_with_pool(buffers, lhs, rhs))
}

#[doc(hidden)]
pub fn maximum_with_pool(
    buffers: &mut BufferPool,
    lhs: &Tensor,
    rhs: &Tensor,
) -> crate::Result<Tensor> {
    reject_complex_ordered_dtypes("maximum", &[lhs.dtype(), rhs.dtype()])?;

    match (lhs.dtype(), rhs.dtype()) {
        (DType::F32, DType::F32) => {
            same_dtype_binary!(buffers, lhs, rhs, f32, typed_maximum_with_pool, "maximum")
        }
        (DType::F64, DType::F64) => {
            same_dtype_binary!(buffers, lhs, rhs, f64, typed_maximum_with_pool, "maximum")
        }
        (DType::I32, DType::I32) => {
            same_dtype_binary!(buffers, lhs, rhs, i32, typed_maximum_with_pool, "maximum")
        }
        (DType::I64, DType::I64) => {
            same_dtype_binary!(buffers, lhs, rhs, i64, typed_maximum_with_pool, "maximum")
        }
        _ => Err(tensor_pair_error("maximum", lhs, rhs)),
    }
}

#[doc(hidden)]
pub fn maximum_read_with_pool(
    buffers: &mut BufferPool,
    lhs: TensorRead<'_>,
    rhs: TensorRead<'_>,
) -> crate::Result<Tensor> {
    let lhs_dtype = lhs.dtype();
    let rhs_dtype = rhs.dtype();
    reject_complex_ordered_dtypes("maximum", &[lhs_dtype, rhs_dtype])?;

    match (read_as_cpu_view(lhs)?, read_as_cpu_view(rhs)?) {
        (CpuReadView::F32(a), CpuReadView::F32(b)) => Ok(Tensor::from_typed::<f32>(
            typed_same_shape_binary_view_with_pool(
                "maximum",
                buffers,
                &a,
                &b,
                OrderedElem::max_elem,
            )?,
        )),
        (CpuReadView::F64(a), CpuReadView::F64(b)) => Ok(Tensor::from_typed::<f64>(
            typed_same_shape_binary_view_with_pool(
                "maximum",
                buffers,
                &a,
                &b,
                OrderedElem::max_elem,
            )?,
        )),
        (CpuReadView::I32(a), CpuReadView::I32(b)) => Ok(Tensor::from_typed::<i32>(
            typed_same_shape_binary_view_with_pool(
                "maximum",
                buffers,
                &a,
                &b,
                OrderedElem::max_elem,
            )?,
        )),
        (CpuReadView::I64(a), CpuReadView::I64(b)) => Ok(Tensor::from_typed::<i64>(
            typed_same_shape_binary_view_with_pool(
                "maximum",
                buffers,
                &a,
                &b,
                OrderedElem::max_elem,
            )?,
        )),
        _ => Err(dtype_pair_error("maximum", lhs_dtype, rhs_dtype)),
    }
}

/// Compute elementwise minimum values.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::minimum;
/// use tenferro_tensor::Tensor;
///
/// let a = Tensor::from_vec_col_major(vec![2], vec![1.0_f64, 5.0])?;
/// let b = Tensor::from_vec_col_major(vec![2], vec![3.0_f64, 4.0])?;
/// let out = minimum(&a, &b)?;
/// assert_eq!(out.as_slice::<f64>().unwrap(), &[1.0, 4.0]);
/// # Ok::<(), tenferro_tensor::Error>(())
/// ```
#[cfg(test)]
#[doc(hidden)]
pub fn minimum(lhs: &Tensor, rhs: &Tensor) -> crate::Result<Tensor> {
    with_test_pool(|buffers| minimum_with_pool(buffers, lhs, rhs))
}

#[doc(hidden)]
pub fn minimum_with_pool(
    buffers: &mut BufferPool,
    lhs: &Tensor,
    rhs: &Tensor,
) -> crate::Result<Tensor> {
    reject_complex_ordered_dtypes("minimum", &[lhs.dtype(), rhs.dtype()])?;

    match (lhs.dtype(), rhs.dtype()) {
        (DType::F32, DType::F32) => {
            same_dtype_binary!(buffers, lhs, rhs, f32, typed_minimum_with_pool, "minimum")
        }
        (DType::F64, DType::F64) => {
            same_dtype_binary!(buffers, lhs, rhs, f64, typed_minimum_with_pool, "minimum")
        }
        (DType::I32, DType::I32) => {
            same_dtype_binary!(buffers, lhs, rhs, i32, typed_minimum_with_pool, "minimum")
        }
        (DType::I64, DType::I64) => {
            same_dtype_binary!(buffers, lhs, rhs, i64, typed_minimum_with_pool, "minimum")
        }
        _ => Err(tensor_pair_error("minimum", lhs, rhs)),
    }
}

#[doc(hidden)]
pub fn minimum_read_with_pool(
    buffers: &mut BufferPool,
    lhs: TensorRead<'_>,
    rhs: TensorRead<'_>,
) -> crate::Result<Tensor> {
    let lhs_dtype = lhs.dtype();
    let rhs_dtype = rhs.dtype();
    reject_complex_ordered_dtypes("minimum", &[lhs_dtype, rhs_dtype])?;

    match (read_as_cpu_view(lhs)?, read_as_cpu_view(rhs)?) {
        (CpuReadView::F32(a), CpuReadView::F32(b)) => Ok(Tensor::from_typed::<f32>(
            typed_same_shape_binary_view_with_pool(
                "minimum",
                buffers,
                &a,
                &b,
                OrderedElem::min_elem,
            )?,
        )),
        (CpuReadView::F64(a), CpuReadView::F64(b)) => Ok(Tensor::from_typed::<f64>(
            typed_same_shape_binary_view_with_pool(
                "minimum",
                buffers,
                &a,
                &b,
                OrderedElem::min_elem,
            )?,
        )),
        (CpuReadView::I32(a), CpuReadView::I32(b)) => Ok(Tensor::from_typed::<i32>(
            typed_same_shape_binary_view_with_pool(
                "minimum",
                buffers,
                &a,
                &b,
                OrderedElem::min_elem,
            )?,
        )),
        (CpuReadView::I64(a), CpuReadView::I64(b)) => Ok(Tensor::from_typed::<i64>(
            typed_same_shape_binary_view_with_pool(
                "minimum",
                buffers,
                &a,
                &b,
                OrderedElem::min_elem,
            )?,
        )),
        _ => Err(dtype_pair_error("minimum", lhs_dtype, rhs_dtype)),
    }
}

/// Compare two CPU tensors elementwise.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::compare;
/// use tenferro_tensor::{CompareDir, Tensor};
///
/// let a = Tensor::from_vec_col_major(vec![2], vec![1.0_f64, 5.0])?;
/// let b = Tensor::from_vec_col_major(vec![2], vec![3.0_f64, 4.0])?;
/// let out = compare(&a, &b, &CompareDir::Gt)?;
/// assert_eq!(out.as_slice::<bool>().unwrap(), &[false, true]);
/// # Ok::<(), tenferro_tensor::Error>(())
/// ```
#[cfg(test)]
#[doc(hidden)]
pub fn compare(lhs: &Tensor, rhs: &Tensor, dir: &CompareDir) -> crate::Result<Tensor> {
    with_test_pool(|buffers| compare_with_pool(buffers, lhs, rhs, dir))
}

#[doc(hidden)]
pub fn compare_with_pool(
    buffers: &mut BufferPool,
    lhs: &Tensor,
    rhs: &Tensor,
    dir: &CompareDir,
) -> crate::Result<Tensor> {
    reject_complex_unsupported_compare_dtypes(dir, &[lhs.dtype(), rhs.dtype()])?;

    match (lhs.dtype(), rhs.dtype()) {
        (DType::F32, DType::F32) => Ok(Tensor::from_typed::<bool>(
            typed_ordered_compare_view_with_pool(
                buffers,
                &pair_operand::<f32>("compare", lhs, rhs, lhs)?.as_view(),
                &pair_operand::<f32>("compare", lhs, rhs, rhs)?.as_view(),
                dir,
            )?,
        )),
        (DType::F64, DType::F64) => Ok(Tensor::from_typed::<bool>(
            typed_ordered_compare_view_with_pool(
                buffers,
                &pair_operand::<f64>("compare", lhs, rhs, lhs)?.as_view(),
                &pair_operand::<f64>("compare", lhs, rhs, rhs)?.as_view(),
                dir,
            )?,
        )),
        (DType::I32, DType::I32) => Ok(Tensor::from_typed::<bool>(
            typed_ordered_compare_view_with_pool(
                buffers,
                &pair_operand::<i32>("compare", lhs, rhs, lhs)?.as_view(),
                &pair_operand::<i32>("compare", lhs, rhs, rhs)?.as_view(),
                dir,
            )?,
        )),
        (DType::I64, DType::I64) => Ok(Tensor::from_typed::<bool>(
            typed_ordered_compare_view_with_pool(
                buffers,
                &pair_operand::<i64>("compare", lhs, rhs, lhs)?.as_view(),
                &pair_operand::<i64>("compare", lhs, rhs, rhs)?.as_view(),
                dir,
            )?,
        )),
        (DType::Bool, DType::Bool) => Ok(Tensor::from_typed::<bool>(
            typed_ordered_compare_view_with_pool(
                buffers,
                &pair_operand::<bool>("compare", lhs, rhs, lhs)?.as_view(),
                &pair_operand::<bool>("compare", lhs, rhs, rhs)?.as_view(),
                dir,
            )?,
        )),
        (DType::C32, DType::C32) => Ok(Tensor::from_typed::<bool>(typed_compare_with_pool(
            buffers,
            pair_operand::<Complex<f32>>("compare", lhs, rhs, lhs)?,
            pair_operand::<Complex<f32>>("compare", lhs, rhs, rhs)?,
            dir,
        )?)),
        (DType::C64, DType::C64) => Ok(Tensor::from_typed::<bool>(typed_compare_with_pool(
            buffers,
            pair_operand::<Complex<f64>>("compare", lhs, rhs, lhs)?,
            pair_operand::<Complex<f64>>("compare", lhs, rhs, rhs)?,
            dir,
        )?)),
        _ => Err(crate::Error::dtype_mismatch(
            "compare",
            lhs.dtype(),
            rhs.dtype(),
        )),
    }
}

#[doc(hidden)]
pub fn compare_read_with_pool(
    buffers: &mut BufferPool,
    lhs: TensorRead<'_>,
    rhs: TensorRead<'_>,
    dir: &CompareDir,
) -> crate::Result<Tensor> {
    let lhs_dtype = lhs.dtype();
    let rhs_dtype = rhs.dtype();
    reject_complex_unsupported_compare_dtypes(dir, &[lhs_dtype, rhs_dtype])?;

    match (read_as_cpu_view(lhs)?, read_as_cpu_view(rhs)?) {
        (CpuReadView::F32(a), CpuReadView::F32(b)) => Ok(Tensor::from_typed::<bool>(
            typed_ordered_compare_view_with_pool(buffers, &a, &b, dir)?,
        )),
        (CpuReadView::F64(a), CpuReadView::F64(b)) => Ok(Tensor::from_typed::<bool>(
            typed_ordered_compare_view_with_pool(buffers, &a, &b, dir)?,
        )),
        (CpuReadView::I32(a), CpuReadView::I32(b)) => Ok(Tensor::from_typed::<bool>(
            typed_ordered_compare_view_with_pool(buffers, &a, &b, dir)?,
        )),
        (CpuReadView::I64(a), CpuReadView::I64(b)) => Ok(Tensor::from_typed::<bool>(
            typed_ordered_compare_view_with_pool(buffers, &a, &b, dir)?,
        )),
        (CpuReadView::Bool(a), CpuReadView::Bool(b)) => Ok(Tensor::from_typed::<bool>(
            typed_ordered_compare_view_with_pool(buffers, &a, &b, dir)?,
        )),
        (CpuReadView::C32(a), CpuReadView::C32(b)) => Ok(Tensor::from_typed::<bool>(
            typed_same_shape_binary_view_with_pool("compare", buffers, &a, &b, |x, y| {
                x.compare_elem(y, dir)
            })?,
        )),
        (CpuReadView::C64(a), CpuReadView::C64(b)) => Ok(Tensor::from_typed::<bool>(
            typed_same_shape_binary_view_with_pool("compare", buffers, &a, &b, |x, y| {
                x.compare_elem(y, dir)
            })?,
        )),
        _ => Err(crate::Error::dtype_mismatch(
            "compare", lhs_dtype, rhs_dtype,
        )),
    }
}

/// Select values from two tensors using a boolean predicate tensor.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::select;
/// use tenferro_tensor::Tensor;
///
/// let pred = Tensor::from_vec_col_major(vec![2], vec![true, false])?;
/// let on_true = Tensor::from_vec_col_major(vec![2], vec![1.0_f64, 2.0])?;
/// let on_false = Tensor::from_vec_col_major(vec![2], vec![3.0_f64, 4.0])?;
/// let out = select(&pred, &on_true, &on_false)?;
/// assert_eq!(out.as_slice::<f64>().unwrap(), &[1.0, 4.0]);
/// # Ok::<(), tenferro_tensor::Error>(())
/// ```
#[cfg(test)]
#[doc(hidden)]
pub fn select(pred: &Tensor, on_true: &Tensor, on_false: &Tensor) -> crate::Result<Tensor> {
    with_test_pool(|buffers| select_with_pool(buffers, pred, on_true, on_false))
}

#[doc(hidden)]
pub fn select_with_pool(
    buffers: &mut BufferPool,
    pred: &Tensor,
    on_true: &Tensor,
    on_false: &Tensor,
) -> crate::Result<Tensor> {
    match (pred.dtype(), on_true.dtype(), on_false.dtype()) {
        (DType::Bool, DType::F32, DType::F32) => {
            let (p, t, f) = select_operands::<f32>("select", pred, on_true, on_false)?;
            Ok(Tensor::from_typed::<f32>(typed_select_with_pool(
                buffers, p, t, f,
            )?))
        }
        (DType::Bool, DType::F64, DType::F64) => {
            let (p, t, f) = select_operands::<f64>("select", pred, on_true, on_false)?;
            Ok(Tensor::from_typed::<f64>(typed_select_with_pool(
                buffers, p, t, f,
            )?))
        }
        (DType::Bool, DType::I32, DType::I32) => {
            let (p, t, f) = select_operands::<i32>("select", pred, on_true, on_false)?;
            Ok(Tensor::from_typed::<i32>(typed_select_with_pool(
                buffers, p, t, f,
            )?))
        }
        (DType::Bool, DType::I64, DType::I64) => {
            let (p, t, f) = select_operands::<i64>("select", pred, on_true, on_false)?;
            Ok(Tensor::from_typed::<i64>(typed_select_with_pool(
                buffers, p, t, f,
            )?))
        }
        (DType::Bool, DType::Bool, DType::Bool) => {
            let (p, t, f) = select_operands::<bool>("select", pred, on_true, on_false)?;
            Ok(Tensor::from_typed::<bool>(typed_select_with_pool(
                buffers, p, t, f,
            )?))
        }
        (DType::Bool, DType::C32, DType::C32) => {
            let (p, t, f) = select_operands::<Complex<f32>>("select", pred, on_true, on_false)?;
            Ok(Tensor::from_typed::<Complex<f32>>(typed_select_with_pool(
                buffers, p, t, f,
            )?))
        }
        (DType::Bool, DType::C64, DType::C64) => {
            let (p, t, f) = select_operands::<Complex<f64>>("select", pred, on_true, on_false)?;
            Ok(Tensor::from_typed::<Complex<f64>>(typed_select_with_pool(
                buffers, p, t, f,
            )?))
        }
        (DType::Bool, _, _) => Err(crate::Error::dtype_mismatch(
            "select",
            on_true.dtype(),
            on_false.dtype(),
        )),
        _ => Err(crate::Error::dtype_mismatch(
            "select",
            pred.dtype(),
            crate::DType::Bool,
        )),
    }
}

/// The predicate and value pair behind a select, or the refusal this table reports.
///
/// Callers reach this from a match on the three dtypes, so `None` means the tags and the runtime
/// dtypes disagree rather than a caller mistake.
fn select_operands<'a, T: TensorScalar>(
    op: &'static str,
    pred: &'a Tensor,
    on_true: &'a Tensor,
    on_false: &'a Tensor,
) -> crate::Result<(
    &'a TypedTensor<bool>,
    &'a TypedTensor<T>,
    &'a TypedTensor<T>,
)> {
    let p = pred
        .as_typed::<bool>()
        .ok_or_else(|| select_error(op, pred, on_true, on_false))?;
    let t = on_true
        .as_typed::<T>()
        .ok_or_else(|| select_error(op, pred, on_true, on_false))?;
    let f = on_false
        .as_typed::<T>()
        .ok_or_else(|| select_error(op, pred, on_true, on_false))?;
    Ok((p, t, f))
}

/// The refusal the select table reports, which is the pair's or the predicate's depending on which
/// arm would have answered.
fn select_error(
    op: &'static str,
    pred: &Tensor,
    on_true: &Tensor,
    on_false: &Tensor,
) -> crate::Error {
    if pred.dtype() != DType::Bool {
        crate::Error::dtype_mismatch(op, pred.dtype(), DType::Bool)
    } else {
        crate::Error::dtype_mismatch(op, on_true.dtype(), on_false.dtype())
    }
}

#[doc(hidden)]
pub fn select_read_with_pool(
    buffers: &mut BufferPool,
    pred: TensorRead<'_>,
    on_true: TensorRead<'_>,
    on_false: TensorRead<'_>,
) -> crate::Result<Tensor> {
    let pred_dtype = pred.dtype();
    let true_dtype = on_true.dtype();
    let false_dtype = on_false.dtype();
    match (
        read_as_cpu_view(pred)?,
        read_as_cpu_view(on_true)?,
        read_as_cpu_view(on_false)?,
    ) {
        (CpuReadView::Bool(p), CpuReadView::F32(t), CpuReadView::F32(f)) => Ok(
            Tensor::from_typed::<f32>(typed_select_view_with_pool(buffers, &p, &t, &f)?),
        ),
        (CpuReadView::Bool(p), CpuReadView::F64(t), CpuReadView::F64(f)) => Ok(
            Tensor::from_typed::<f64>(typed_select_view_with_pool(buffers, &p, &t, &f)?),
        ),
        (CpuReadView::Bool(p), CpuReadView::I32(t), CpuReadView::I32(f)) => Ok(
            Tensor::from_typed::<i32>(typed_select_view_with_pool(buffers, &p, &t, &f)?),
        ),
        (CpuReadView::Bool(p), CpuReadView::I64(t), CpuReadView::I64(f)) => Ok(
            Tensor::from_typed::<i64>(typed_select_view_with_pool(buffers, &p, &t, &f)?),
        ),
        (CpuReadView::Bool(p), CpuReadView::Bool(t), CpuReadView::Bool(f)) => Ok(
            Tensor::from_typed::<bool>(typed_select_view_with_pool(buffers, &p, &t, &f)?),
        ),
        (CpuReadView::Bool(p), CpuReadView::C32(t), CpuReadView::C32(f)) => Ok(
            Tensor::from_typed::<Complex<f32>>(typed_select_view_with_pool(buffers, &p, &t, &f)?),
        ),
        (CpuReadView::Bool(p), CpuReadView::C64(t), CpuReadView::C64(f)) => Ok(
            Tensor::from_typed::<Complex<f64>>(typed_select_view_with_pool(buffers, &p, &t, &f)?),
        ),
        (CpuReadView::Bool(_), _, _) => Err(crate::Error::dtype_mismatch(
            "select",
            true_dtype,
            false_dtype,
        )),
        _ => Err(crate::Error::dtype_mismatch(
            "select",
            pred_dtype,
            crate::DType::Bool,
        )),
    }
}

/// Clamp CPU tensor values elementwise between lower and upper bounds.
///
/// # Examples
///
/// ```
/// use tenferro_cpu::clamp;
/// use tenferro_tensor::Tensor;
///
/// let input = Tensor::from_vec_col_major(vec![3], vec![-1.0_f64, 2.0, 8.0])?;
/// let lower = Tensor::from_vec_col_major(vec![3], vec![0.0_f64, 0.0, 0.0])?;
/// let upper = Tensor::from_vec_col_major(vec![3], vec![5.0_f64, 5.0, 5.0])?;
/// let out = clamp(&input, &lower, &upper)?;
/// assert_eq!(out.as_slice::<f64>().unwrap(), &[0.0, 2.0, 5.0]);
/// # Ok::<(), tenferro_tensor::Error>(())
/// ```
#[cfg(test)]
#[doc(hidden)]
pub fn clamp(input: &Tensor, lower: &Tensor, upper: &Tensor) -> crate::Result<Tensor> {
    with_test_pool(|buffers| clamp_with_pool(buffers, input, lower, upper))
}

#[doc(hidden)]
pub fn clamp_with_pool(
    buffers: &mut BufferPool,
    input: &Tensor,
    lower: &Tensor,
    upper: &Tensor,
) -> crate::Result<Tensor> {
    reject_complex_ordered_dtypes("clamp", &[input.dtype(), lower.dtype(), upper.dtype()])?;

    dispatch_ternary_result_with_pool!("clamp", input, lower, upper, |x, lo, hi| {
        typed_clamp_with_pool(buffers, x, lo, hi)
    })
}

#[doc(hidden)]
pub fn clamp_read_with_pool(
    buffers: &mut BufferPool,
    input: TensorRead<'_>,
    lower: TensorRead<'_>,
    upper: TensorRead<'_>,
) -> crate::Result<Tensor> {
    let input_dtype = input.dtype();
    let lower_dtype = lower.dtype();
    let upper_dtype = upper.dtype();
    reject_complex_ordered_dtypes("clamp", &[input_dtype, lower_dtype, upper_dtype])?;

    match (
        read_as_cpu_view(input)?,
        read_as_cpu_view(lower)?,
        read_as_cpu_view(upper)?,
    ) {
        (CpuReadView::F32(input), CpuReadView::F32(lower), CpuReadView::F32(upper)) => Ok(
            Tensor::from_typed::<f32>(typed_clamp_view_with_pool(buffers, &input, &lower, &upper)?),
        ),
        (CpuReadView::F64(input), CpuReadView::F64(lower), CpuReadView::F64(upper)) => Ok(
            Tensor::from_typed::<f64>(typed_clamp_view_with_pool(buffers, &input, &lower, &upper)?),
        ),
        _ => Err(crate::Error::dtype_mismatch(
            "clamp",
            input_dtype,
            lower_dtype,
        )),
    }
}

#[doc(hidden)]
pub fn typed_add_with_pool<T>(
    buffers: &mut BufferPool,
    lhs: &TypedTensor<T>,
    rhs: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: Copy + Clone + Zero + Add<Output = T> + PoolScalar,
{
    typed_binary_with_pool("add", buffers, lhs, rhs, Add::add)
}

fn typed_binary_with_pool<T>(
    op: &'static str,
    buffers: &mut BufferPool,
    lhs: &TypedTensor<T>,
    rhs: &TypedTensor<T>,
    f: impl Fn(T, T) -> T + Copy + Sync,
) -> crate::Result<TypedTensor<T>>
where
    T: Copy + PoolScalar + 'static,
{
    if lhs.shape() == rhs.shape() {
        let mut out = PooledUninitOutput::<T>::new(buffers, lhs.shape().to_vec())?;
        replay_binary(
            op,
            &mut out.as_uninit_view_mut()?,
            &typed_view(op, lhs)?,
            &typed_view(op, rhs)?,
            f,
        )?;
        // SAFETY: the successful binary zip/map replay writes every logical destination element and retains no destination view.
        Ok(unsafe { out.assume_init()? })
    } else if lhs.shape().is_empty() {
        let scalar = typed_host_data(op, lhs)?[0];
        let mut out = PooledUninitOutput::<T>::new(buffers, rhs.shape().to_vec())?;
        replay_scalar_left(
            op,
            &mut out.as_uninit_view_mut()?,
            &typed_view(op, rhs)?,
            scalar,
            f,
        )?;
        // SAFETY: the successful binary scalar map replay writes every logical destination element and retains no destination view.
        Ok(unsafe { out.assume_init()? })
    } else if rhs.shape().is_empty() {
        let scalar = typed_host_data(op, rhs)?[0];
        let mut out = PooledUninitOutput::<T>::new(buffers, lhs.shape().to_vec())?;
        replay_scalar_right(
            op,
            &mut out.as_uninit_view_mut()?,
            &typed_view(op, lhs)?,
            scalar,
            f,
        )?;
        // SAFETY: the successful binary scalar map replay writes every logical destination element and retains no destination view.
        Ok(unsafe { out.assume_init()? })
    } else {
        Err(crate::Error::shape_mismatch(
            op,
            lhs.shape().to_vec(),
            rhs.shape().to_vec(),
        ))
    }
}

fn typed_wrapping_add_with_pool<T>(
    buffers: &mut BufferPool,
    lhs: &TypedTensor<T>,
    rhs: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: WrappingIntegerElem + Mul<Output = T>,
{
    typed_binary_with_pool(
        "add",
        buffers,
        lhs,
        rhs,
        WrappingIntegerElem::wrapping_add_elem,
    )
}

fn typed_wrapping_add_view_with_pool<T, L, R>(
    buffers: &mut BufferPool,
    lhs: &TypedTensorView<'_, T, L>,
    rhs: &TypedTensorView<'_, T, R>,
) -> crate::Result<TypedTensor<T>>
where
    T: WrappingIntegerElem + Mul<Output = T>,
    L: TensorRank,
    R: TensorRank,
{
    typed_binary_view_with_pool(
        "add",
        buffers,
        lhs,
        rhs,
        WrappingIntegerElem::wrapping_add_elem,
    )
}

#[doc(hidden)]
pub fn typed_add_view_with_pool<T, L, R>(
    buffers: &mut BufferPool,
    lhs: &TypedTensorView<'_, T, L>,
    rhs: &TypedTensorView<'_, T, R>,
) -> crate::Result<TypedTensor<T>>
where
    T: Copy + Clone + Zero + Add<Output = T> + PoolScalar + 'static,
    L: TensorRank,
    R: TensorRank,
{
    typed_binary_view_with_pool("add", buffers, lhs, rhs, Add::add)
}

#[doc(hidden)]
pub fn typed_sub_with_pool<T>(
    buffers: &mut BufferPool,
    lhs: &TypedTensor<T>,
    rhs: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: Copy + PoolScalar + Sub<Output = T> + 'static,
{
    typed_binary_with_pool("sub", buffers, lhs, rhs, Sub::sub)
}

fn typed_wrapping_sub_with_pool<T>(
    buffers: &mut BufferPool,
    lhs: &TypedTensor<T>,
    rhs: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: WrappingIntegerElem,
{
    typed_binary_with_pool(
        "sub",
        buffers,
        lhs,
        rhs,
        WrappingIntegerElem::wrapping_sub_elem,
    )
}

fn typed_wrapping_sub_view_with_pool<T, L, R>(
    buffers: &mut BufferPool,
    lhs: &TypedTensorView<'_, T, L>,
    rhs: &TypedTensorView<'_, T, R>,
) -> crate::Result<TypedTensor<T>>
where
    T: WrappingIntegerElem,
    L: TensorRank,
    R: TensorRank,
{
    typed_binary_view_with_pool(
        "sub",
        buffers,
        lhs,
        rhs,
        WrappingIntegerElem::wrapping_sub_elem,
    )
}

#[doc(hidden)]
pub fn typed_sub_view_with_pool<T, L, R>(
    buffers: &mut BufferPool,
    lhs: &TypedTensorView<'_, T, L>,
    rhs: &TypedTensorView<'_, T, R>,
) -> crate::Result<TypedTensor<T>>
where
    T: Copy + PoolScalar + Sub<Output = T> + 'static,
    L: TensorRank,
    R: TensorRank,
{
    typed_binary_view_with_pool("sub", buffers, lhs, rhs, Sub::sub)
}

#[doc(hidden)]
pub fn typed_mul_with_pool<T>(
    buffers: &mut BufferPool,
    lhs: &TypedTensor<T>,
    rhs: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: Copy + Clone + Zero + Mul<Output = T> + PoolScalar + 'static,
{
    if lhs.shape() == rhs.shape() {
        let mut out = PooledUninitOutput::<T>::new(buffers, lhs.shape().to_vec())?;
        mul_into_uninit(
            &mut out.as_uninit_view_mut()?,
            &typed_view("mul", lhs)?,
            &typed_view("mul", rhs)?,
        )
        .map_err(|err| crate::Error::backend_source("mul", err))?;
        // SAFETY: the successful multiplication kernel writes every logical destination element and retains no destination view.
        Ok(unsafe { out.assume_init()? })
    } else {
        typed_binary_with_pool("mul", buffers, lhs, rhs, Mul::mul)
    }
}

fn typed_wrapping_mul_with_pool<T>(
    buffers: &mut BufferPool,
    lhs: &TypedTensor<T>,
    rhs: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: WrappingIntegerElem,
{
    if lhs.shape() == rhs.shape() {
        let mut out = PooledUninitOutput::<T>::new(buffers, lhs.shape().to_vec())?;
        mul_into_uninit(
            &mut out.as_uninit_view_mut()?,
            &typed_view("mul", lhs)?,
            &typed_view("mul", rhs)?,
        )
        .map_err(|err| crate::Error::backend_source("mul", err))?;
        // SAFETY: the successful wrapping multiplication kernel writes every logical destination element and retains no destination view.
        Ok(unsafe { out.assume_init()? })
    } else {
        typed_binary_with_pool(
            "mul",
            buffers,
            lhs,
            rhs,
            WrappingIntegerElem::wrapping_mul_elem,
        )
    }
}

fn typed_wrapping_mul_view_with_pool<T, L, R>(
    buffers: &mut BufferPool,
    lhs: &TypedTensorView<'_, T, L>,
    rhs: &TypedTensorView<'_, T, R>,
) -> crate::Result<TypedTensor<T>>
where
    T: WrappingIntegerElem,
    L: TensorRank,
    R: TensorRank,
{
    if lhs.shape() == rhs.shape() {
        let mut out = PooledUninitOutput::<T>::new(buffers, lhs.shape().to_vec())?;
        mul_into_uninit(
            &mut out.as_uninit_view_mut()?,
            &typed_view_from_view("mul", lhs)?,
            &typed_view_from_view("mul", rhs)?,
        )
        .map_err(|err| crate::Error::backend_source("mul", err))?;
        // SAFETY: the successful wrapping multiplication kernel writes every logical destination element and retains no destination view.
        Ok(unsafe { out.assume_init()? })
    } else {
        typed_binary_view_with_pool(
            "mul",
            buffers,
            lhs,
            rhs,
            WrappingIntegerElem::wrapping_mul_elem,
        )
    }
}

#[doc(hidden)]
pub fn typed_mul_view_with_pool<T, L, R>(
    buffers: &mut BufferPool,
    lhs: &TypedTensorView<'_, T, L>,
    rhs: &TypedTensorView<'_, T, R>,
) -> crate::Result<TypedTensor<T>>
where
    T: Copy + Clone + Zero + Mul<Output = T> + PoolScalar + 'static,
    L: TensorRank,
    R: TensorRank,
{
    if lhs.shape() == rhs.shape() {
        let mut out = PooledUninitOutput::<T>::new(buffers, lhs.shape().to_vec())?;
        mul_into_uninit(
            &mut out.as_uninit_view_mut()?,
            &typed_view_from_view("mul", lhs)?,
            &typed_view_from_view("mul", rhs)?,
        )
        .map_err(|err| crate::Error::backend_source("mul", err))?;
        // SAFETY: the successful multiplication kernel writes every logical destination element and retains no destination view.
        Ok(unsafe { out.assume_init()? })
    } else {
        typed_binary_view_with_pool("mul", buffers, lhs, rhs, Mul::mul)
    }
}

#[doc(hidden)]
pub fn typed_div_with_pool<T>(
    buffers: &mut BufferPool,
    lhs: &TypedTensor<T>,
    rhs: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: Copy + Clone + Zero + Div<Output = T> + PoolScalar + 'static,
{
    typed_binary_with_pool("div", buffers, lhs, rhs, Div::div)
}

fn typed_integer_div_with_pool<T>(
    buffers: &mut BufferPool,
    lhs: &TypedTensor<T>,
    rhs: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: WrappingIntegerElem,
{
    let rhs_view = typed_view("div", rhs)?;
    ensure_no_zero_divisor("div", &rhs_view)?;
    typed_binary_with_pool(
        "div",
        buffers,
        lhs,
        rhs,
        WrappingIntegerElem::wrapping_div_elem,
    )
}

fn typed_integer_div_view_with_pool<T, L, R>(
    buffers: &mut BufferPool,
    lhs: &TypedTensorView<'_, T, L>,
    rhs: &TypedTensorView<'_, T, R>,
) -> crate::Result<TypedTensor<T>>
where
    T: WrappingIntegerElem,
    L: TensorRank,
    R: TensorRank,
{
    let rhs_view = typed_view_from_view("div", rhs)?;
    ensure_no_zero_divisor("div", &rhs_view)?;
    typed_binary_view_with_pool(
        "div",
        buffers,
        lhs,
        rhs,
        WrappingIntegerElem::wrapping_div_elem,
    )
}

fn typed_rem_with_pool<T>(
    buffers: &mut BufferPool,
    lhs: &TypedTensor<T>,
    rhs: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: Copy + Clone + Zero + StdRem<Output = T> + PoolScalar + 'static,
{
    typed_binary_with_pool("rem", buffers, lhs, rhs, StdRem::rem)
}

fn typed_integer_rem_with_pool<T>(
    buffers: &mut BufferPool,
    lhs: &TypedTensor<T>,
    rhs: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: WrappingIntegerElem,
{
    let rhs_view = typed_view("rem", rhs)?;
    ensure_no_zero_divisor("rem", &rhs_view)?;
    typed_binary_with_pool(
        "rem",
        buffers,
        lhs,
        rhs,
        WrappingIntegerElem::wrapping_rem_elem,
    )
}

fn typed_integer_rem_view_with_pool<T, L, R>(
    buffers: &mut BufferPool,
    lhs: &TypedTensorView<'_, T, L>,
    rhs: &TypedTensorView<'_, T, R>,
) -> crate::Result<TypedTensor<T>>
where
    T: WrappingIntegerElem,
    L: TensorRank,
    R: TensorRank,
{
    let rhs_view = typed_view_from_view("rem", rhs)?;
    ensure_no_zero_divisor("rem", &rhs_view)?;
    typed_binary_view_with_pool(
        "rem",
        buffers,
        lhs,
        rhs,
        WrappingIntegerElem::wrapping_rem_elem,
    )
}

#[doc(hidden)]
fn typed_map_with_pool<T, O>(
    op: &'static str,
    buffers: &mut BufferPool,
    input: &TypedTensor<T>,
    f: impl Fn(T) -> O + Copy + Sync,
) -> crate::Result<TypedTensor<O>>
where
    T: Copy + Send + Sync + TensorScalar + 'static,
    O: Clone + PoolScalar,
{
    let mut out = PooledUninitOutput::<O>::new(buffers, input.shape().to_vec())?;
    replay_unary(
        op,
        &mut out.as_uninit_view_mut()?,
        &typed_view(op, input)?,
        f,
    )?;
    // SAFETY: the successful same-shape replay writes every logical destination element and retains no destination view.
    Ok(unsafe { out.assume_init()? })
}

#[doc(hidden)]
pub fn typed_neg_with_pool<T>(
    buffers: &mut BufferPool,
    input: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: Copy + Clone + Zero + Neg<Output = T> + PoolScalar + 'static,
{
    typed_map_with_pool("neg", buffers, input, Neg::neg)
}

fn typed_wrapping_unary_with_pool<T>(
    op: &'static str,
    buffers: &mut BufferPool,
    input: &TypedTensor<T>,
    f: impl Fn(T) -> T + Copy + Sync,
) -> crate::Result<TypedTensor<T>>
where
    T: WrappingIntegerElem,
{
    typed_map_with_pool(op, buffers, input, f)
}

fn typed_wrapping_neg_with_pool<T>(
    buffers: &mut BufferPool,
    input: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: WrappingIntegerElem,
{
    typed_wrapping_unary_with_pool(
        "neg",
        buffers,
        input,
        WrappingIntegerElem::wrapping_neg_elem,
    )
}

fn typed_wrapping_abs_with_pool<T>(
    buffers: &mut BufferPool,
    input: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: WrappingIntegerElem,
{
    typed_wrapping_unary_with_pool(
        "abs",
        buffers,
        input,
        WrappingIntegerElem::wrapping_abs_elem,
    )
}

#[doc(hidden)]
pub fn typed_conj_with_pool<T>(
    buffers: &mut BufferPool,
    input: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: Copy + Clone + Zero + ConjElem + PoolScalar + 'static,
{
    typed_map_with_pool("conj", buffers, input, ConjElem::conj_elem)
}

#[doc(hidden)]
pub fn typed_abs_with_pool<T>(
    buffers: &mut BufferPool,
    input: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: Tier2Elem + PoolScalar + 'static,
{
    typed_map_with_pool("abs", buffers, input, Tier2Elem::abs_elem)
}

fn typed_complex_abs_with_pool<T>(
    buffers: &mut BufferPool,
    input: &TypedTensor<Complex<T>>,
) -> crate::Result<TypedTensor<T>>
where
    T: num_traits::Float + PoolScalar + 'static,
    Complex<T>: TensorScalar,
{
    typed_map_with_pool("abs", buffers, input, Complex::norm)
}

fn typed_complex_abs_view_with_pool<T, R>(
    buffers: &mut BufferPool,
    input: &TypedTensorView<'_, Complex<T>, R>,
) -> crate::Result<TypedTensor<T>>
where
    T: num_traits::Float + PoolScalar + 'static,
    R: TensorRank,
{
    let mut out = PooledUninitOutput::<T>::new(buffers, input.shape().to_vec())?;
    replay_unary(
        "abs",
        &mut out.as_uninit_view_mut()?,
        &typed_view_from_view("abs", input)?,
        Complex::norm,
    )?;
    // SAFETY: the successful unary map replay writes every logical destination element and retains no destination view.
    Ok(unsafe { out.assume_init()? })
}

#[doc(hidden)]
pub fn typed_sign_with_pool<T>(
    buffers: &mut BufferPool,
    input: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: Tier2Elem + PoolScalar + 'static,
{
    typed_map_with_pool("sign", buffers, input, Tier2Elem::sign_elem)
}

fn typed_integer_sign_with_pool<T>(
    buffers: &mut BufferPool,
    input: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: WrappingIntegerElem,
{
    typed_wrapping_unary_with_pool("sign", buffers, input, WrappingIntegerElem::signum_elem)
}

#[doc(hidden)]
pub fn typed_maximum_with_pool<T>(
    buffers: &mut BufferPool,
    lhs: &TypedTensor<T>,
    rhs: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: OrderedElem + PoolScalar,
{
    if lhs.shape() != rhs.shape() {
        return Err(crate::Error::shape_mismatch(
            "maximum",
            lhs.shape().to_vec(),
            rhs.shape().to_vec(),
        ));
    }
    let mut out = PooledUninitOutput::<T>::new(buffers, lhs.shape().to_vec())?;
    replay_binary(
        "maximum",
        &mut out.as_uninit_view_mut()?,
        &typed_view("maximum", lhs)?,
        &typed_view("maximum", rhs)?,
        OrderedElem::max_elem,
    )?;
    // SAFETY: the successful binary zip replay writes every logical destination element and retains no destination view.
    Ok(unsafe { out.assume_init()? })
}

#[doc(hidden)]
pub fn typed_minimum_with_pool<T>(
    buffers: &mut BufferPool,
    lhs: &TypedTensor<T>,
    rhs: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: OrderedElem + PoolScalar,
{
    if lhs.shape() != rhs.shape() {
        return Err(crate::Error::shape_mismatch(
            "minimum",
            lhs.shape().to_vec(),
            rhs.shape().to_vec(),
        ));
    }
    let mut out = PooledUninitOutput::<T>::new(buffers, lhs.shape().to_vec())?;
    replay_binary(
        "minimum",
        &mut out.as_uninit_view_mut()?,
        &typed_view("minimum", lhs)?,
        &typed_view("minimum", rhs)?,
        OrderedElem::min_elem,
    )?;
    // SAFETY: the successful binary zip replay writes every logical destination element and retains no destination view.
    Ok(unsafe { out.assume_init()? })
}

#[doc(hidden)]
pub fn typed_compare_with_pool<T>(
    buffers: &mut BufferPool,
    lhs: &TypedTensor<T>,
    rhs: &TypedTensor<T>,
    dir: &CompareDir,
) -> crate::Result<TypedTensor<bool>>
where
    T: CompareElem + TensorScalar,
{
    if lhs.shape() != rhs.shape() {
        return Err(crate::Error::shape_mismatch(
            "compare",
            lhs.shape().to_vec(),
            rhs.shape().to_vec(),
        ));
    }
    let mut out = PooledUninitOutput::<bool>::new(buffers, lhs.shape().to_vec())?;
    zip_map2_into(
        &mut out.as_uninit_view_mut()?,
        &typed_view("compare", lhs)?,
        &typed_view("compare", rhs)?,
        |x, y| MaybeUninit::new(x.compare_elem(y, dir)),
    )
    .map_err(|err| crate::Error::backend_source("compare", err))?;
    // SAFETY: the successful compare replay writes every logical destination element and retains no destination view.
    Ok(unsafe { out.assume_init()? })
}

#[doc(hidden)]
pub fn typed_select_with_pool<T>(
    buffers: &mut BufferPool,
    pred: &TypedTensor<bool>,
    on_true: &TypedTensor<T>,
    on_false: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: Copy + PoolScalar,
{
    if pred.shape() != on_true.shape() {
        return Err(crate::Error::shape_mismatch(
            "select",
            pred.shape().to_vec(),
            on_true.shape().to_vec(),
        ));
    }
    if pred.shape() != on_false.shape() {
        return Err(crate::Error::shape_mismatch(
            "select",
            pred.shape().to_vec(),
            on_false.shape().to_vec(),
        ));
    }
    let mut out = PooledUninitOutput::<T>::new(buffers, pred.shape().to_vec())?;
    replay_select(
        &mut out.as_uninit_view_mut()?,
        &typed_view("select", pred)?,
        &typed_view("select", on_true)?,
        &typed_view("select", on_false)?,
    )?;
    // SAFETY: the successful select replay writes every logical destination element and retains no destination view.
    Ok(unsafe { out.assume_init()? })
}

#[doc(hidden)]
pub fn typed_clamp_with_pool<T>(
    buffers: &mut BufferPool,
    input: &TypedTensor<T>,
    lower: &TypedTensor<T>,
    upper: &TypedTensor<T>,
) -> crate::Result<TypedTensor<T>>
where
    T: OrderedElem + PoolScalar,
{
    if input.shape() != lower.shape() {
        return Err(crate::Error::shape_mismatch(
            "clamp",
            input.shape().to_vec(),
            lower.shape().to_vec(),
        ));
    }
    if input.shape() != upper.shape() {
        return Err(crate::Error::shape_mismatch(
            "clamp",
            input.shape().to_vec(),
            upper.shape().to_vec(),
        ));
    }
    let mut out = PooledUninitOutput::<T>::new(buffers, input.shape().to_vec())?;
    replay_clamp(
        &mut out.as_uninit_view_mut()?,
        &typed_view("clamp", input)?,
        &typed_view("clamp", lower)?,
        &typed_view("clamp", upper)?,
    )?;
    // SAFETY: the successful clamp replay writes every logical destination element and retains no destination view.
    Ok(unsafe { out.assume_init()? })
}

#[cfg(test)]
mod tests;