use std::any::Any;
use std::panic::Location;
use crate::ctx::ThreadCtx;
use crate::diag::{Diagnostic, LaneProblem};
use crate::sched::{PHASE_ARRIVE, PHASE_DEPART};
pub const WARP_SIZE: u32 = 32;
pub const FULL_MASK: u32 = u32::MAX;
struct Lanes<'a> {
slots: &'a [Option<Box<dyn Any + Send>>],
op: &'static str,
}
impl Lanes<'_> {
fn get<T: Copy + 'static>(&self, lane: u32) -> T {
let slot = self
.slots
.get(lane as usize)
.and_then(|s| s.as_ref())
.unwrap_or_else(|| panic!("riri: lane {lane} published no value to `{}`", self.op));
*slot.downcast_ref::<T>().unwrap_or_else(|| {
panic!(
"riri: `{}` called with different value types on different lanes",
self.op
)
})
}
}
fn lane_error(
ctx: &ThreadCtx<'_>,
op: &'static str,
at: &'static Location<'static>,
mask: u32,
problem: LaneProblem,
) -> Diagnostic {
Diagnostic::WarpLaneError {
block: ctx.block_linear() as u32,
warp: ctx.warp_id(),
op,
at,
lane: ctx.lane_id(),
mask,
problem,
}
}
fn collective<T, R>(
ctx: &ThreadCtx<'_>,
mask: u32,
op: &'static str,
at: &'static Location<'static>,
value: T,
combine: impl FnOnce(&Lanes<'_>) -> R,
) -> R
where
T: Send + 'static,
{
ctx.schedule_point();
let lane = ctx.lane_id();
let valid = ctx.warp_valid_mask();
if mask & !valid != 0 {
ctx.trap(lane_error(
ctx,
op,
at,
mask,
LaneProblem::MaskOutsideBlock { valid },
));
}
if mask & (1u32 << lane) == 0 {
ctx.trap(lane_error(ctx, op, at, mask, LaneProblem::CallerNotInMask));
}
ctx.warp_slots().lock().unwrap()[lane as usize] = Some(Box::new(value));
ctx.warp_rendezvous(PHASE_ARRIVE, mask, at, op);
let result = {
let slots = ctx.warp_slots().lock().unwrap();
combine(&Lanes { slots: &slots, op })
};
ctx.warp_rendezvous(PHASE_DEPART, mask, at, op);
result
}
fn shuffle<T: Copy + Send + 'static>(
ctx: &ThreadCtx<'_>,
mask: u32,
op: &'static str,
at: &'static Location<'static>,
value: T,
src: Option<u32>,
) -> T {
collective(ctx, mask, op, at, value, |lanes| {
let Some(src) = src else { return value };
if mask & (1u32 << src) == 0 {
ctx.report(lane_error(
ctx,
op,
at,
mask,
LaneProblem::SourceLaneNotInMask { src },
));
return value;
}
lanes.get::<T>(src)
})
}
#[track_caller]
pub fn shfl_sync<T: Copy + Send + 'static>(
ctx: &ThreadCtx<'_>,
mask: u32,
value: T,
src_lane: u32,
) -> T {
shfl_sync_at(ctx, mask, value, src_lane, Location::caller())
}
pub(crate) fn shfl_sync_at<T: Copy + Send + 'static>(
ctx: &ThreadCtx<'_>,
mask: u32,
value: T,
src_lane: u32,
at: &'static Location<'static>,
) -> T {
let src = src_lane % ctx.warp_size();
shuffle(ctx, mask, "shfl_sync", at, value, Some(src))
}
#[track_caller]
pub fn shfl_up_sync<T: Copy + Send + 'static>(
ctx: &ThreadCtx<'_>,
mask: u32,
value: T,
delta: u32,
) -> T {
shfl_up_sync_at(ctx, mask, value, delta, Location::caller())
}
pub(crate) fn shfl_up_sync_at<T: Copy + Send + 'static>(
ctx: &ThreadCtx<'_>,
mask: u32,
value: T,
delta: u32,
at: &'static Location<'static>,
) -> T {
let src = ctx.lane_id().checked_sub(delta);
shuffle(ctx, mask, "shfl_up_sync", at, value, src)
}
#[track_caller]
pub fn shfl_down_sync<T: Copy + Send + 'static>(
ctx: &ThreadCtx<'_>,
mask: u32,
value: T,
delta: u32,
) -> T {
shfl_down_sync_at(ctx, mask, value, delta, Location::caller())
}
pub(crate) fn shfl_down_sync_at<T: Copy + Send + 'static>(
ctx: &ThreadCtx<'_>,
mask: u32,
value: T,
delta: u32,
at: &'static Location<'static>,
) -> T {
let src = ctx
.lane_id()
.checked_add(delta)
.filter(|&s| s < ctx.warp_size());
shuffle(ctx, mask, "shfl_down_sync", at, value, src)
}
#[track_caller]
pub fn shfl_xor_sync<T: Copy + Send + 'static>(
ctx: &ThreadCtx<'_>,
mask: u32,
value: T,
lane_mask: u32,
) -> T {
shfl_xor_sync_at(ctx, mask, value, lane_mask, Location::caller())
}
pub(crate) fn shfl_xor_sync_at<T: Copy + Send + 'static>(
ctx: &ThreadCtx<'_>,
mask: u32,
value: T,
lane_mask: u32,
at: &'static Location<'static>,
) -> T {
let src = Some(ctx.lane_id() ^ lane_mask).filter(|&s| s < ctx.warp_size());
shuffle(ctx, mask, "shfl_xor_sync", at, value, src)
}
pub(crate) fn ballot(
ctx: &ThreadCtx<'_>,
mask: u32,
pred: bool,
op: &'static str,
at: &'static Location<'static>,
) -> u32 {
collective(ctx, mask, op, at, pred, |lanes| {
(0..32u32)
.filter(|l| mask & (1u32 << l) != 0 && lanes.get::<bool>(*l))
.fold(0u32, |acc, l| acc | (1u32 << l))
})
}
#[track_caller]
pub fn ballot_sync(ctx: &ThreadCtx<'_>, mask: u32, pred: bool) -> u32 {
ballot(ctx, mask, pred, "ballot_sync", Location::caller())
}
#[track_caller]
pub fn any_sync(ctx: &ThreadCtx<'_>, mask: u32, pred: bool) -> bool {
ballot(ctx, mask, pred, "any_sync", Location::caller()) != 0
}
#[track_caller]
pub fn all_sync(ctx: &ThreadCtx<'_>, mask: u32, pred: bool) -> bool {
ballot(ctx, mask, pred, "all_sync", Location::caller()) == mask
}
#[track_caller]
pub fn sync_warp(ctx: &ThreadCtx<'_>, mask: u32) {
sync_warp_at(ctx, mask, Location::caller())
}
pub(crate) fn sync_warp_at(ctx: &ThreadCtx<'_>, mask: u32, at: &'static Location<'static>) {
collective(ctx, mask, "sync_warp", at, (), |_| ())
}