#![cfg_attr(test, allow(clippy::unwrap_used, reason = "test scope"))]
#[cfg(feature = "parallel")]
use leto::{TaskPartitionMut, TaskPartitionsMut};
#[cfg(feature = "parallel")]
pub(crate) const PARALLEL_MIN_BYTES: usize = 1024 * 1024;
#[cfg(feature = "parallel")]
pub(crate) const PARALLEL_MIN_ELEMENTS: usize = 65536;
#[cfg(feature = "parallel")]
pub(crate) const PARALLEL_MIN_REDUCTION_OUTPUTS: usize = 32768;
#[cfg(feature = "parallel")]
pub(crate) const PARALLEL_MIN_MATMUL_MACS: usize = 262_144;
#[cfg(feature = "parallel")]
pub(crate) const PARALLEL_MIN_MATMUL_ROWS: usize = 64;
#[cfg(feature = "parallel")]
#[inline]
fn dispatch_unit_task_mut_with<P, T, F>(
output: &mut [T],
unit_len: usize,
element_bytes: usize,
run: F,
) where
P: moirai::ExecutionPolicy,
T: Send,
F: Fn(usize, &mut [T]) + Send + Sync,
{
moirai::for_each_unit_task_mut_with::<P, _, _, _, _>(
output,
unit_len,
unit_len.saturating_mul(element_bytes),
|| (),
|(), first_unit, values| run(first_unit, values),
);
}
#[cfg(feature = "parallel")]
#[inline]
pub(crate) fn parallelize_bandwidth_bound<T>(len: usize, operands: usize) -> bool {
let working_set = len
.saturating_mul(operands)
.saturating_mul(core::mem::size_of::<T>());
crate::infrastructure::cache::working_set_spills_llc(working_set)
}
#[cfg(feature = "parallel")]
#[inline]
pub(crate) fn parallelize_compute_bound(len: usize) -> bool {
len >= PARALLEL_MIN_ELEMENTS
}
#[cfg(feature = "parallel")]
pub(crate) fn for_each_unit_run_mut<T, F>(output: &mut [T], element_bytes: usize, run: F)
where
T: Send,
F: Fn(usize, &mut [T]) + Send + Sync,
{
schedule_unit_runs_mut(output, element_bytes, &run);
}
#[cfg(feature = "parallel")]
fn schedule_unit_runs_mut<T: Send>(
output: &mut [T],
element_bytes: usize,
run: &(dyn Fn(usize, &mut [T]) + Sync),
) {
moirai::for_each_unit_task_mut_with::<moirai::Parallel, _, _, _, _>(
output,
1,
element_bytes,
|| (),
|(), first_index, values| run(first_index, values),
);
}
#[cfg(feature = "parallel")]
pub(crate) fn for_each_unit_range<F>(units: usize, unit_bytes: usize, run: F)
where
F: Fn(usize, usize) + Send + Sync,
{
schedule_unit_ranges(units, unit_bytes, &run);
}
#[cfg(feature = "parallel")]
fn schedule_unit_ranges(units: usize, unit_bytes: usize, run: &(dyn Fn(usize, usize) + Sync)) {
moirai::for_each_unit_task_range_with::<moirai::Parallel, _, _, _>(
units,
unit_bytes,
|| (),
|(), first_unit, count| run(first_unit, count),
);
}
pub(crate) fn for_each_plane_mut_many_with<T, S, I, F, const K: usize>(
outputs: [&mut [T]; K],
plane_len: usize,
#[cfg_attr(
not(feature = "parallel"),
expect(
unused_variables,
reason = "only the parallel arm sizes tasks by bytes"
)
)]
element_bytes: usize,
state: I,
plane: F,
) where
T: Send,
S: Send,
I: Fn() -> S + Send + Sync,
F: Fn(&mut S, usize, [&mut [T]; K]) + Send + Sync,
{
if plane_len == 0 {
return;
}
#[cfg(feature = "parallel")]
moirai::for_each_unit_task_many_mut_with::<moirai::WorkBytes<PARALLEL_MIN_BYTES>, _, _, _, _, K>(
outputs,
plane_len,
plane_len.saturating_mul(element_bytes),
state,
|task_state, first_plane, runs| {
let planes = runs.first().map_or(0, |run| run.len() / plane_len);
let mut cursors = runs.map(|run| run.chunks_exact_mut(plane_len));
for offset in 0..planes {
let lockstep = core::array::from_fn(|d| {
cursors[d]
.next()
.expect("invariant: every run holds the same plane count")
});
plane(task_state, first_plane + offset, lockstep);
}
},
);
#[cfg(not(feature = "parallel"))]
{
let mut task_state = state();
let planes = outputs.first().map_or(0, |output| output.len() / plane_len);
let mut cursors = outputs.map(|output| output.chunks_exact_mut(plane_len));
for index in 0..planes {
let lockstep = core::array::from_fn(|d| {
cursors[d]
.next()
.expect("invariant: every output holds the same plane count")
});
plane(&mut task_state, index, lockstep);
}
}
}
pub(crate) fn for_each_plane_mut<T, F>(
output: &mut [T],
plane_len: usize,
#[cfg_attr(
not(feature = "parallel"),
expect(
unused_variables,
reason = "only the parallel arm sizes tasks by bytes"
)
)]
element_bytes: usize,
plane: F,
) where
T: Send,
F: Fn(usize, &mut [T]) + Send + Sync,
{
if plane_len == 0 {
return;
}
#[cfg(feature = "parallel")]
dispatch_unit_task_mut_with::<moirai::WorkBytes<PARALLEL_MIN_BYTES>, _, _>(
output,
plane_len,
element_bytes,
|first_plane, planes| {
for (offset, values) in planes.chunks_exact_mut(plane_len).enumerate() {
plane(first_plane + offset, values);
}
},
);
#[cfg(not(feature = "parallel"))]
for (index, values) in output.chunks_exact_mut(plane_len).enumerate() {
plane(index, values);
}
}
#[cfg(feature = "parallel")]
pub fn for_each_task_partition_mut_with<'scope, P, T, F, const N: usize>(
runtime: &'scope moirai::Moirai,
partitions: TaskPartitionsMut<'scope, T, N>,
f: F,
) -> moirai::ExecutorResult<()>
where
P: moirai::ExecutionPolicy,
T: Send + 'scope,
F: Fn(TaskPartitionMut<'scope, T, N>) + Send + Sync + 'scope,
{
if !P::parallelize(partitions.len()) {
for partition in partitions {
f(partition);
}
return Ok(());
}
runtime.scope(|scope| {
let f = &f;
for partition in partitions {
scope.spawn(move |_| f(partition))?;
}
Ok(())
})
}
#[cfg(feature = "parallel")]
pub fn for_each_task_partition_mut<'scope, T, F, const N: usize>(
partitions: TaskPartitionsMut<'scope, T, N>,
f: F,
) -> moirai::ExecutorResult<()>
where
T: Send + 'scope,
F: Fn(TaskPartitionMut<'scope, T, N>) + Send + Sync + 'scope,
{
for_each_task_partition_mut_with::<moirai::Adaptive, _, _, _>(moirai::global(), partitions, f)
}
#[cfg(all(test, feature = "parallel"))]
mod tests {
use super::*;
use leto::{Array, ArrayViewMut, Layout, VecStorage};
use moirai::{Moirai, Parallel, Sequential};
fn values<const N: usize>(array: &Array<i32, VecStorage<i32>, N>) -> Vec<i32> {
array.iter().copied().collect()
}
#[test]
fn sequential_policy_consumes_partitions_in_logical_order() {
let runtime = Moirai::builder().worker_threads(1).build().unwrap();
let mut array = Array::from_shape_vec([2, 3], vec![0; 6]).unwrap();
let partitions = array.task_partitions_mut(2).unwrap();
for_each_task_partition_mut_with::<Sequential, _, _, _>(
&runtime,
partitions,
|partition| {
let start = partition.logical_range().start as i32;
for (offset, value) in partition.into_iter().enumerate() {
*value = start + offset as i32;
}
},
)
.unwrap();
assert_eq!(values(&array), vec![0, 1, 2, 3, 4, 5]);
runtime.shutdown();
}
#[test]
fn parallel_policy_updates_strided_negative_layout() {
let runtime = Moirai::builder().worker_threads(2).build().unwrap();
let mut storage = vec![-1; 6];
let layout =
Layout::try_new([2, 3], [3, -1], 2).expect("invariant: fixture layout is valid");
let view = ArrayViewMut::try_new(layout, &mut storage).unwrap();
let partitions = view.task_partitions_mut(2).unwrap();
for_each_task_partition_mut_with::<Parallel, _, _, _>(&runtime, partitions, |partition| {
let start = partition.logical_range().start as i32;
for (offset, value) in partition.into_iter().enumerate() {
*value = start + offset as i32;
}
})
.unwrap();
assert_eq!(storage, vec![2, 1, 0, 5, 4, 3]);
runtime.shutdown();
}
#[test]
fn shutdown_is_reported_before_partition_callback_runs() {
let runtime = Moirai::builder().worker_threads(1).build().unwrap();
let mut array = Array::from_shape_vec([4], vec![0; 4]).unwrap();
let partitions = array.task_partitions_mut(1).unwrap();
runtime.shutdown();
let result =
for_each_task_partition_mut_with::<Parallel, _, _, _>(&runtime, partitions, |_| {
panic!("shutdown must reject before callback")
});
assert_eq!(result, Err(moirai::ExecutorError::ShuttingDown));
}
}