use crate::application::index::{line_elements_for, validate_mutable_output};
use crate::domain::scalar::Scalar;
use crate::infrastructure::cache::{cached_cache_geometry, CacheGeometry};
#[cfg(feature = "parallel")]
use crate::infrastructure::parallel::parallelize_bandwidth_bound;
use leto::{Array, ArrayView, ArrayViewMut, Result, VecStorage};
mod sealed {
pub trait Sealed {}
}
pub trait BinaryOp<T: Scalar>: sealed::Sealed + Copy + Send + Sync + 'static {
fn apply(lhs: T, rhs: T) -> T;
fn apply_slice(lhs: &[T], rhs: &[T], out: &mut [T]);
}
#[derive(Clone, Copy, Debug, Default)]
pub struct AddOp;
#[derive(Clone, Copy, Debug, Default)]
pub struct SubOp;
#[derive(Clone, Copy, Debug, Default)]
pub struct MulOp;
#[derive(Clone, Copy, Debug, Default)]
pub struct DivOp;
impl sealed::Sealed for AddOp {}
impl sealed::Sealed for SubOp {}
impl sealed::Sealed for MulOp {}
impl sealed::Sealed for DivOp {}
macro_rules! define_comparison_op {
($name:ident, $comparison:tt) => {
#[doc = concat!(stringify!($name), " applies the ", stringify!($comparison), " comparison and writes a zero-or-one mask.")]
#[derive(Clone, Copy, Debug, Default)]
pub struct $name;
impl sealed::Sealed for $name {}
impl<T: Scalar> BinaryOp<T> for $name {
#[inline(always)]
fn apply(lhs: T, rhs: T) -> T {
if lhs $comparison rhs {
T::ONE
} else {
T::ZERO
}
}
#[inline(always)]
fn apply_slice(lhs: &[T], rhs: &[T], out: &mut [T]) {
for ((out, lhs), rhs) in out.iter_mut().zip(lhs).zip(rhs) {
*out = Self::apply(*lhs, *rhs);
}
}
}
};
}
define_comparison_op!(EqOp, ==);
define_comparison_op!(NeOp, !=);
define_comparison_op!(LtOp, <);
define_comparison_op!(GtOp, >);
define_comparison_op!(LeOp, <=);
define_comparison_op!(GeOp, >=);
impl<T: Scalar> BinaryOp<T> for AddOp {
#[inline(always)]
fn apply(lhs: T, rhs: T) -> T {
lhs.add(rhs)
}
#[inline(always)]
fn apply_slice(lhs: &[T], rhs: &[T], out: &mut [T]) {
T::add_slice(lhs, rhs, out);
}
}
impl<T: Scalar> BinaryOp<T> for SubOp {
#[inline(always)]
fn apply(lhs: T, rhs: T) -> T {
lhs.sub(rhs)
}
#[inline(always)]
fn apply_slice(lhs: &[T], rhs: &[T], out: &mut [T]) {
T::sub_slice(lhs, rhs, out);
}
}
impl<T: Scalar> BinaryOp<T> for MulOp {
#[inline(always)]
fn apply(lhs: T, rhs: T) -> T {
lhs.mul(rhs)
}
#[inline(always)]
fn apply_slice(lhs: &[T], rhs: &[T], out: &mut [T]) {
T::mul_slice(lhs, rhs, out);
}
}
impl<T: Scalar> BinaryOp<T> for DivOp {
#[inline(always)]
fn apply(lhs: T, rhs: T) -> T {
lhs.div(rhs)
}
#[inline(always)]
fn apply_slice(lhs: &[T], rhs: &[T], out: &mut [T]) {
T::div_slice(lhs, rhs, out);
}
}
#[inline]
fn validate_binary_shapes<T, const N: usize>(
lhs: &ArrayView<'_, T, N>,
rhs: &ArrayView<'_, T, N>,
out: &ArrayViewMut<'_, T, N>,
) -> Result<()> {
lhs.layout().broadcast(out.shape())?;
rhs.layout().broadcast(out.shape())?;
Ok(())
}
#[inline]
fn validate_binary_storage<T, const N: usize>(
lhs: &ArrayView<'_, T, N>,
rhs: &ArrayView<'_, T, N>,
out: &ArrayViewMut<'_, T, N>,
) -> Result<()> {
lhs.layout().validate_storage_len(lhs.data().len())?;
rhs.layout().validate_storage_len(rhs.data().len())?;
validate_mutable_output(out, "binary map")?;
Ok(())
}
pub fn binary_map<Op, T, const N: usize>(
lhs: &ArrayView<'_, T, N>,
rhs: &ArrayView<'_, T, N>,
out: &mut ArrayViewMut<'_, T, N>,
) -> Result<()>
where
Op: BinaryOp<T>,
T: Scalar,
{
binary_map_with_cache_line::<Op, T, N>(lhs, rhs, out, None)
}
pub fn binary_map_with_cache_geometry<Op, T, const N: usize>(
lhs: &ArrayView<'_, T, N>,
rhs: &ArrayView<'_, T, N>,
out: &mut ArrayViewMut<'_, T, N>,
geometry: CacheGeometry,
) -> Result<()>
where
Op: BinaryOp<T>,
T: Scalar,
{
binary_map_with_cache_line::<Op, T, N>(lhs, rhs, out, Some(geometry.cache_line_bytes()))
}
fn binary_map_with_cache_line<Op, T, const N: usize>(
lhs: &ArrayView<'_, T, N>,
rhs: &ArrayView<'_, T, N>,
out: &mut ArrayViewMut<'_, T, N>,
cache_line_bytes: Option<usize>,
) -> Result<()>
where
Op: BinaryOp<T>,
T: Scalar,
{
validate_binary_shapes(lhs, rhs, out)?;
let out_shape = out.shape();
if lhs.shape() == out_shape
&& rhs.shape() == out_shape
&& lhs.strides() == out.strides()
&& rhs.strides() == out.strides()
{
if let (Some(lhs_slice), Some(rhs_slice), Some(out_slice)) = (
lhs.as_slice_memory_order(),
rhs.as_slice_memory_order(),
out.as_mut_slice_memory_order(),
) {
debug_assert_eq!(lhs_slice.len(), rhs_slice.len());
debug_assert_eq!(lhs_slice.len(), out_slice.len());
#[cfg(feature = "parallel")]
{
if parallelize_bandwidth_bound::<T>(lhs_slice.len(), 3) {
crate::application::strided::parallel_slice_into::<T, T, 2, _>(
[lhs_slice, rhs_slice],
out_slice,
3 * core::mem::size_of::<T>(),
|inputs, out| Op::apply_slice(inputs[0], inputs[1], out),
);
return Ok(());
}
}
Op::apply_slice(lhs_slice, rhs_slice, out_slice);
return Ok(());
}
}
validate_binary_storage(lhs, rhs, out)?;
let size = out.layout().checked_size()?;
let shape = out.shape();
let lhs_layout = lhs.layout().broadcast(shape)?;
let rhs_layout = rhs.layout().broadcast(shape)?;
let out_layout = out.layout();
let lhs_data = lhs.data();
let rhs_data = rhs.data();
let out_data = out.data_mut();
let cache_line_bytes =
cache_line_bytes.unwrap_or_else(|| cached_cache_geometry().cache_line_bytes());
let in_tile = line_elements_for::<T>(cache_line_bytes);
#[cfg(feature = "parallel")]
{
if parallelize_bandwidth_bound::<T>(size, 3) {
let lhs_ptr = lhs_data.as_ptr() as usize;
let rhs_ptr = rhs_data.as_ptr() as usize;
let out_ptr = out_data.as_mut_ptr() as usize;
crate::application::strided::strided_parallel::<N, 2, 1, _>(
crate::application::strided::StridedLayout::new(
size,
shape,
[lhs_layout, rhs_layout],
[out_layout],
in_tile,
in_tile,
),
3 * core::mem::size_of::<T>(),
move |in_off: [isize; 2], out_off: [isize; 1]| {
unsafe {
let lhs_value = *(lhs_ptr as *const T).offset(in_off[0]);
let rhs_value = *(rhs_ptr as *const T).offset(in_off[1]);
*(out_ptr as *mut T).offset(out_off[0]) = Op::apply(lhs_value, rhs_value);
}
},
);
return Ok(());
}
}
crate::application::strided::strided_serial::<N, 2, 1, _>(
crate::application::strided::StridedLayout::new(
size,
shape,
[lhs_layout, rhs_layout],
[out_layout],
in_tile,
in_tile,
),
|in_off: [isize; 2], out_off: [isize; 1]| {
out_data[out_off[0] as usize] =
Op::apply(lhs_data[in_off[0] as usize], rhs_data[in_off[1] as usize]);
},
)
}
#[inline]
pub fn add<T: Scalar, const N: usize>(
lhs: &ArrayView<'_, T, N>,
rhs: &ArrayView<'_, T, N>,
out: &mut ArrayViewMut<'_, T, N>,
) -> Result<()> {
binary_map::<AddOp, T, N>(lhs, rhs, out)
}
#[inline]
pub fn sub<T: Scalar, const N: usize>(
lhs: &ArrayView<'_, T, N>,
rhs: &ArrayView<'_, T, N>,
out: &mut ArrayViewMut<'_, T, N>,
) -> Result<()> {
binary_map::<SubOp, T, N>(lhs, rhs, out)
}
#[inline]
pub fn mul<T: Scalar, const N: usize>(
lhs: &ArrayView<'_, T, N>,
rhs: &ArrayView<'_, T, N>,
out: &mut ArrayViewMut<'_, T, N>,
) -> Result<()> {
binary_map::<MulOp, T, N>(lhs, rhs, out)
}
#[inline]
pub fn div<T: Scalar, const N: usize>(
lhs: &ArrayView<'_, T, N>,
rhs: &ArrayView<'_, T, N>,
out: &mut ArrayViewMut<'_, T, N>,
) -> Result<()> {
binary_map::<DivOp, T, N>(lhs, rhs, out)
}
#[inline]
pub fn scalar_map<Op, T, const N: usize>(
input: &ArrayView<'_, T, N>,
scalar: T,
) -> Result<Array<T, VecStorage<T>, N>>
where
Op: BinaryOp<T>,
T: Scalar,
{
crate::application::unary::mapv(input, move |x| Op::apply(x, scalar))
}
#[inline]
pub fn scalar_map_into<Op, T, const N: usize>(
input: &ArrayView<'_, T, N>,
scalar: T,
output: &mut ArrayViewMut<'_, T, N>,
) -> Result<()>
where
Op: BinaryOp<T>,
T: Scalar,
{
crate::application::unary::map_into_gated(input, output, move |x| Op::apply(x, scalar), false)
}
pub fn sum<T: Scalar, const N: usize>(arr: &ArrayView<'_, T, N>) -> T {
crate::application::reduction::reduce_all::<crate::application::reduction::SumAxis, T, N>(arr)
.expect("invariant: sum requires a view whose layout fits its storage")
}