use crate::application::index::{RowMajorTraversal, TileGeometry};
use leto::{Layout, Result};
#[derive(Clone, Copy)]
pub(crate) struct StridedLayout<const N: usize, const IN: usize, const OUT: usize> {
size: usize,
shape: [usize; N],
in_layouts: [Layout<N>; IN],
out_layouts: [Layout<N>; OUT],
in_tile: usize,
out_tile: usize,
}
impl<const N: usize, const IN: usize, const OUT: usize> StridedLayout<N, IN, OUT> {
pub(crate) const fn new(
size: usize,
shape: [usize; N],
in_layouts: [Layout<N>; IN],
out_layouts: [Layout<N>; OUT],
in_tile: usize,
out_tile: usize,
) -> Self {
Self {
size,
shape,
in_layouts,
out_layouts,
in_tile,
out_tile,
}
}
}
#[inline]
fn offsets<const M: usize, const N: usize>(
layouts: [Layout<N>; M],
index: [usize; N],
) -> Result<[isize; M]> {
let mut offsets = [0isize; M];
for (offset, layout) in offsets.iter_mut().zip(layouts) {
*offset = layout.offset_of(index)? as isize;
}
Ok(offsets)
}
#[cfg(feature = "parallel")]
#[inline]
fn offsets_expect<const M: usize, const N: usize>(
layouts: [Layout<N>; M],
index: [usize; N],
) -> [isize; M] {
let mut offsets = [0isize; M];
for (offset, layout) in offsets.iter_mut().zip(layouts) {
*offset = layout
.offset_of(index)
.expect("validated layout must map every logical index") as isize;
}
offsets
}
#[inline(always)]
fn advance<const M: usize>(offsets: &mut [isize; M], strides: &[isize; M]) {
for (offset, stride) in offsets.iter_mut().zip(strides) {
*offset += *stride;
}
}
#[inline]
fn is_column_walk<const IN: usize, const OUT: usize>(
in_step: &[isize; IN],
out_step: &[isize; OUT],
in_tile: usize,
out_tile: usize,
) -> bool {
in_step
.iter()
.any(|&stride| stride.unsigned_abs() >= in_tile)
|| out_step
.iter()
.any(|&stride| stride.unsigned_abs() >= out_tile)
}
#[inline]
pub(crate) fn strided_serial<const N: usize, const IN: usize, const OUT: usize, F>(
geometry: StridedLayout<N, IN, OUT>,
mut lane: F,
) -> Result<()>
where
F: FnMut([isize; IN], [isize; OUT]),
{
let StridedLayout {
size,
shape,
in_layouts,
out_layouts,
in_tile,
out_tile,
} = geometry;
let Some(traversal) = RowMajorTraversal::new(size, shape) else {
return Ok(());
};
let in_step: [isize; IN] = core::array::from_fn(|i| traversal.last_axis_stride(in_layouts[i]));
let out_step: [isize; OUT] =
core::array::from_fn(|j| traversal.last_axis_stride(out_layouts[j]));
if is_column_walk(&in_step, &out_step, in_tile, out_tile) {
if let Some(tile_geometry) = TileGeometry::new(size, shape, in_tile.min(out_tile)) {
let in_row_step: [isize; IN] = core::array::from_fn(|i| in_layouts[i].strides()[N - 2]);
let out_row_step: [isize; OUT] =
core::array::from_fn(|j| out_layouts[j].strides()[N - 2]);
for slab in 0..tile_geometry.slabs() {
let base_index = tile_geometry.slab_base_index(slab);
let in_base = offsets(in_layouts, base_index)?;
let out_base = offsets(out_layouts, base_index)?;
let mut row_block = 0;
while row_block < tile_geometry.height() {
let row_end = (row_block + tile_geometry.tile()).min(tile_geometry.height());
let mut col_block = 0;
while col_block < tile_geometry.width() {
let col_end = (col_block + tile_geometry.tile()).min(tile_geometry.width());
for row in row_block..row_end {
let r = row as isize;
let c0 = col_block as isize;
let mut in_off: [isize; IN] = core::array::from_fn(|i| {
in_base[i] + r * in_row_step[i] + c0 * in_step[i]
});
let mut out_off: [isize; OUT] = core::array::from_fn(|j| {
out_base[j] + r * out_row_step[j] + c0 * out_step[j]
});
for _ in col_block..col_end {
lane(in_off, out_off);
advance(&mut in_off, &in_step);
advance(&mut out_off, &out_step);
}
}
col_block = col_end;
}
row_block = row_end;
}
}
return Ok(());
}
}
for row in 0..traversal.rows() {
let base_index = traversal.base_index(row);
let mut in_off = offsets(in_layouts, base_index)?;
let mut out_off = offsets(out_layouts, base_index)?;
for _ in 0..traversal.inner() {
lane(in_off, out_off);
advance(&mut in_off, &in_step);
advance(&mut out_off, &out_step);
}
}
Ok(())
}
#[cfg(feature = "parallel")]
#[inline]
pub(crate) fn strided_parallel<const N: usize, const IN: usize, const OUT: usize, F>(
geometry: StridedLayout<N, IN, OUT>,
element_bytes: usize,
lane: F,
) where
F: Fn([isize; IN], [isize; OUT]) + Send + Sync,
{
use crate::infrastructure::parallel::for_each_unit_range;
let StridedLayout {
size,
shape,
in_layouts,
out_layouts,
in_tile,
out_tile,
} = geometry;
let Some(traversal) = RowMajorTraversal::new(size, shape) else {
return;
};
let in_step: [isize; IN] = core::array::from_fn(|i| traversal.last_axis_stride(in_layouts[i]));
let out_step: [isize; OUT] =
core::array::from_fn(|j| traversal.last_axis_stride(out_layouts[j]));
let lane = &lane;
if is_column_walk(&in_step, &out_step, in_tile, out_tile) {
if let Some(tile_geometry) = TileGeometry::new(size, shape, in_tile.min(out_tile)) {
let in_row_step: [isize; IN] = core::array::from_fn(|i| in_layouts[i].strides()[N - 2]);
let out_row_step: [isize; OUT] =
core::array::from_fn(|j| out_layouts[j].strides()[N - 2]);
let blocks = tile_geometry.slabs() * tile_geometry.row_blocks();
let block_bytes = tile_geometry
.tile()
.saturating_mul(tile_geometry.width())
.saturating_mul(element_bytes);
for_each_unit_range(blocks, block_bytes, move |first, count| {
for block in first..first + count {
let slab = block / tile_geometry.row_blocks();
let row_block = (block % tile_geometry.row_blocks()) * tile_geometry.tile();
let row_end = (row_block + tile_geometry.tile()).min(tile_geometry.height());
let base_index = tile_geometry.slab_base_index(slab);
let in_base = offsets_expect(in_layouts, base_index);
let out_base = offsets_expect(out_layouts, base_index);
let mut col_block = 0;
while col_block < tile_geometry.width() {
let col_end = (col_block + tile_geometry.tile()).min(tile_geometry.width());
for row in row_block..row_end {
let r = row as isize;
let c0 = col_block as isize;
let mut in_off: [isize; IN] = core::array::from_fn(|i| {
in_base[i] + r * in_row_step[i] + c0 * in_step[i]
});
let mut out_off: [isize; OUT] = core::array::from_fn(|j| {
out_base[j] + r * out_row_step[j] + c0 * out_step[j]
});
for _ in col_block..col_end {
lane(in_off, out_off);
advance(&mut in_off, &in_step);
advance(&mut out_off, &out_step);
}
}
col_block = col_end;
}
}
});
return;
}
}
let row_bytes = traversal.inner().saturating_mul(element_bytes);
for_each_unit_range(traversal.rows(), row_bytes, move |first, count| {
for row in first..first + count {
let base_index = traversal.base_index(row);
let mut in_off = offsets_expect(in_layouts, base_index);
let mut out_off = offsets_expect(out_layouts, base_index);
for _ in 0..traversal.inner() {
lane(in_off, out_off);
advance(&mut in_off, &in_step);
advance(&mut out_off, &out_step);
}
}
});
}
#[cfg(feature = "parallel")]
#[inline]
pub(crate) fn parallel_slice_into<T, U, const IN: usize, F>(
inputs: [&[T]; IN],
output: &mut [U],
element_bytes: usize,
lane: F,
) where
T: Send + Sync,
U: Send,
F: Fn([&[T]; IN], &mut [U]) + Send + Sync,
{
crate::infrastructure::parallel::for_each_unit_run_mut(output, element_bytes, |first, run| {
let end = first + run.len();
lane(core::array::from_fn(|i| &inputs[i][first..end]), run);
});
}