use super::sources::{zip_offset, ZipSources};
use super::traversal::for_each_row_major_indexed;
use crate::application::index::{validate_mutable_output, RowMajorTraversal};
use leto::{ArrayViewMut, LetoError, Result};
mod sealed {
pub trait Sealed<S, F, const N: usize> {}
}
#[inline]
fn validate_output<T, const N: usize>(output: &ArrayViewMut<'_, T, N>) -> Result<()> {
validate_mutable_output(output, "zip mutable")
}
fn zip_one<T, S, F, const N: usize>(
output: &mut ArrayViewMut<'_, T, N>,
sources: S,
mut f: F,
) -> Result<()>
where
S: ZipSources<N>,
F: FnMut(&mut T, S::Values),
{
validate_output(output)?;
let shape = output.shape();
sources.validate(shape)?;
if let (Some(output_slice), Some(source_slices)) = (output.as_mut_slice(), sources.contiguous())
{
for (index, value) in output_slice.iter_mut().enumerate() {
f(value, S::contiguous_values(source_slices, index));
}
return Ok(());
}
let size = output.layout().checked_size()?;
let output_layout = output.layout();
let output_data = output.data_mut();
let Some(traversal) = RowMajorTraversal::new(size, shape) else {
return Ok(());
};
let output_step = traversal.last_axis_stride(output_layout);
let source_steps = sources.steps();
for row in 0..traversal.rows() {
let base = traversal.base_index(row);
let mut output_offset = zip_offset(output_layout, base)?;
let mut source_offsets = sources.offsets_at(base)?;
for _ in 0..traversal.inner() {
let output_index = usize::try_from(output_offset)
.expect("invariant: validated zip output offset is non-negative");
f(
&mut output_data[output_index],
sources.values(source_offsets),
);
output_offset += output_step;
sources.advance(&mut source_offsets, source_steps);
}
}
Ok(())
}
fn indexed_zip_one<T, S, F, const N: usize>(
output: &mut ArrayViewMut<'_, T, N>,
sources: S,
mut f: F,
) -> Result<()>
where
S: ZipSources<N>,
F: FnMut([usize; N], &mut T, S::Values),
{
validate_output(output)?;
let shape = output.shape();
sources.validate(shape)?;
let size = output.layout().checked_size()?;
let output_layout = output.layout();
let output_data = output.data_mut();
let Some(traversal) = RowMajorTraversal::new(size, shape) else {
return Ok(());
};
let output_step = traversal.last_axis_stride(output_layout);
let source_steps = sources.steps();
for_each_row_major_indexed(
traversal,
|index| {
Ok((
zip_offset(output_layout, index)?,
sources.offsets_at(index)?,
))
},
|index, offsets| {
let output_index = usize::try_from(offsets.0)
.expect("invariant: validated zip output offset is non-negative");
f(
index,
&mut output_data[output_index],
sources.values(offsets.1),
);
offsets.0 += output_step;
sources.advance(&mut offsets.1, source_steps);
},
)?;
Ok(())
}
pub trait ZipMutOutputs<S, F, const N: usize>: sealed::Sealed<S, F, N> {
fn zip(outputs: Self, sources: S, f: F) -> Result<()>;
}
pub trait IndexedZipMutOutputs<S, F, const N: usize>: sealed::Sealed<S, F, N> {
fn indexed_zip(outputs: Self, sources: S, f: F) -> Result<()>;
}
impl<'data, T, S, F, const N: usize> sealed::Sealed<S, F, N> for ArrayViewMut<'data, T, N> {}
impl<'data, T, S, F, const N: usize> ZipMutOutputs<S, F, N> for ArrayViewMut<'data, T, N>
where
S: ZipSources<N>,
F: FnMut(&mut T, S::Values),
{
fn zip(mut output: Self, sources: S, f: F) -> Result<()> {
zip_one(&mut output, sources, f)
}
}
macro_rules! impl_zip_mut_outputs_for_tuple {
($($type:ident => $data:ident : $index:tt),+ $(,)?) => {
impl<'data, $($type,)+ S, F, const N: usize>
sealed::Sealed<S, F, N>
for ($(ArrayViewMut<'data, $type, N>,)+)
{
}
impl<'data, $($type,)+ S, F, const N: usize>
ZipMutOutputs<S, F, N>
for ($(ArrayViewMut<'data, $type, N>,)+)
where
S: ZipSources<N>,
F: FnMut(($(&mut $type,)+), S::Values),
{
fn zip(mut outputs: Self, sources: S, mut f: F) -> Result<()> {
let shape = outputs.0.shape();
$(
if outputs.$index.shape() != shape {
return Err(LetoError::ShapeMismatch {
lhs: shape.to_vec(),
rhs: outputs.$index.shape().to_vec(),
});
}
validate_output(&outputs.$index)?;
)+
let size = outputs.0.layout().checked_size()?;
if let ($(Some($data),)+) = ($(outputs.$index.as_mut_slice(),)+) {
if let Some(source_slices) = sources.contiguous() {
for index in 0..size {
f(
($(&mut $data[index],)+),
S::contiguous_values(source_slices, index),
);
}
return Ok(());
}
}
let output_layouts = ($(outputs.$index.layout(),)+);
let ($($data,)+) = ($(outputs.$index.data_mut(),)+);
let Some(traversal) = RowMajorTraversal::new(size, shape) else {
return Ok(());
};
let output_steps = ($(traversal.last_axis_stride(output_layouts.$index),)+);
let source_steps = sources.steps();
for row in 0..traversal.rows() {
let base = traversal.base_index(row);
let mut output_offsets =
($(zip_offset(output_layouts.$index, base)?,)+);
let mut source_offsets = sources.offsets_at(base)?;
for _ in 0..traversal.inner() {
f(
($(&mut $data[usize::try_from(output_offsets.$index)
.expect("invariant: validated zip output offset is non-negative")],)+),
sources.values(source_offsets),
);
$(output_offsets.$index += output_steps.$index;)+
sources.advance(&mut source_offsets, source_steps);
}
}
Ok(())
}
}
};
}
impl_zip_mut_outputs_for_tuple!(A => a: 0, B => b: 1);
impl_zip_mut_outputs_for_tuple!(A => a: 0, B => b: 1, C => c: 2);
impl<'a, 'data, T, S, F, const N: usize> sealed::Sealed<S, F, N>
for &'a mut ArrayViewMut<'data, T, N>
{
}
impl<'a, 'data, T, S, F, const N: usize> ZipMutOutputs<S, F, N>
for &'a mut ArrayViewMut<'data, T, N>
where
S: ZipSources<N>,
F: FnMut(&mut T, S::Values),
{
fn zip(outputs: Self, sources: S, f: F) -> Result<()> {
zip_one(outputs, sources, f)
}
}
impl<'data, T, S, F, const N: usize> IndexedZipMutOutputs<S, F, N> for ArrayViewMut<'data, T, N>
where
S: ZipSources<N>,
F: FnMut([usize; N], &mut T, S::Values),
{
fn indexed_zip(mut output: Self, sources: S, f: F) -> Result<()> {
indexed_zip_one(&mut output, sources, f)
}
}
macro_rules! impl_indexed_zip_mut_outputs_for_tuple {
($($type:ident => $data:ident : $index:tt),+ $(,)?) => {
impl<'data, $($type,)+ S, F, const N: usize>
IndexedZipMutOutputs<S, F, N>
for ($(ArrayViewMut<'data, $type, N>,)+)
where
S: ZipSources<N>,
F: FnMut([usize; N], ($(&mut $type,)+), S::Values),
{
fn indexed_zip(mut outputs: Self, sources: S, mut f: F) -> Result<()> {
let shape = outputs.0.shape();
$(
if outputs.$index.shape() != shape {
return Err(LetoError::ShapeMismatch {
lhs: shape.to_vec(),
rhs: outputs.$index.shape().to_vec(),
});
}
validate_output(&outputs.$index)?;
)+
let size = outputs.0.layout().checked_size()?;
let output_layouts = ($(outputs.$index.layout(),)+);
let ($($data,)+) = ($(outputs.$index.data_mut(),)+);
let Some(traversal) = RowMajorTraversal::new(size, shape) else {
return Ok(());
};
let output_steps = ($(traversal.last_axis_stride(output_layouts.$index),)+);
let source_steps = sources.steps();
for_each_row_major_indexed(
traversal,
|index| {
Ok((
($(zip_offset(output_layouts.$index, index)?,)+),
sources.offsets_at(index)?,
))
},
|index, state| {
let (output_offsets, source_offsets) = state;
f(
index,
($(&mut $data[usize::try_from(output_offsets.$index)
.expect("invariant: validated zip output offset is non-negative")],)+),
sources.values(*source_offsets),
);
$(output_offsets.$index += output_steps.$index;)+
sources.advance(source_offsets, source_steps);
},
)?;
Ok(())
}
}
};
}
impl_indexed_zip_mut_outputs_for_tuple!(A => a: 0, B => b: 1);
impl_indexed_zip_mut_outputs_for_tuple!(A => a: 0, B => b: 1, C => c: 2);
impl<'a, 'data, T, S, F, const N: usize> IndexedZipMutOutputs<S, F, N>
for &'a mut ArrayViewMut<'data, T, N>
where
S: ZipSources<N>,
F: FnMut([usize; N], &mut T, S::Values),
{
fn indexed_zip(outputs: Self, sources: S, f: F) -> Result<()> {
indexed_zip_one(outputs, sources, f)
}
}
pub fn zip_mut_with<O, S, F, const N: usize>(outputs: O, sources: S, f: F) -> Result<()>
where
O: ZipMutOutputs<S, F, N>,
{
O::zip(outputs, sources, f)
}
pub fn indexed_zip_mut_with<O, S, F, const N: usize>(outputs: O, sources: S, f: F) -> Result<()>
where
O: IndexedZipMutOutputs<S, F, N>,
{
O::indexed_zip(outputs, sources, f)
}