use std::cell::RefCell;
use std::marker::PhantomData;
use std::panic::Location;
use std::sync::Arc;
use crate::ctx::ThreadCtx;
use crate::diag::Report;
use crate::launch::{LaunchConfig, LaunchState};
use crate::mem::{ElemMut, GlobalBuf};
use crate::sched::{Choices, SplitMix64};
use crate::sync::Ordering;
struct Bound {
state: Arc<LaunchState>,
block: u32,
thread: u32,
gid: usize,
slot: usize,
}
thread_local! {
static CURRENT: RefCell<Option<Bound>> = const { RefCell::new(None) };
}
fn with_ctx<R>(f: impl FnOnce(&ThreadCtx<'_>) -> R) -> R {
CURRENT.with(|cell| {
let bound = cell.borrow();
let bound = bound
.as_ref()
.expect("riri: a cuda-oxide device function was called outside riri::oxide::launch");
let ctx = ThreadCtx {
launch: &bound.state,
block: bound.block,
thread: bound.thread,
gid: bound.gid,
slot: bound.slot,
};
f(&ctx)
})
}
struct Unbind;
impl Drop for Unbind {
fn drop(&mut self) {
CURRENT.with(|cell| *cell.borrow_mut() = None);
}
}
pub fn launch<F>(config: &LaunchConfig, kernel: F) -> Report
where
F: Fn() + Sync,
{
crate::launch::run_shared(
config,
Choices::Random(SplitMix64(config.seed)),
|state, ctx| {
CURRENT.with(|cell| {
*cell.borrow_mut() = Some(Bound {
state: Arc::clone(state),
block: ctx.block,
thread: ctx.thread,
gid: ctx.gid,
slot: ctx.slot,
});
});
let _unbind = Unbind;
kernel();
},
)
.0
}
pub struct ThreadIndex<'k> {
index: usize,
_not_send: PhantomData<*const ()>,
_scope: PhantomData<&'k ()>,
}
impl ThreadIndex<'_> {
pub fn get(&self) -> usize {
self.index
}
}
#[allow(non_snake_case)]
pub mod thread {
use super::{with_ctx, ThreadIndex};
use std::marker::PhantomData;
use std::panic::Location;
fn witness<'k>(index: usize) -> ThreadIndex<'k> {
ThreadIndex {
index,
_not_send: PhantomData,
_scope: PhantomData,
}
}
pub fn index_1d<'k>() -> ThreadIndex<'k> {
witness(with_ctx(|t| t.global_linear()))
}
pub fn threadIdx_x() -> u32 {
with_ctx(|t| t.thread_idx().x)
}
pub fn threadIdx_y() -> u32 {
with_ctx(|t| t.thread_idx().y)
}
pub fn threadIdx_z() -> u32 {
with_ctx(|t| t.thread_idx().z)
}
pub fn blockIdx_x() -> u32 {
with_ctx(|t| t.block_idx().x)
}
pub fn blockIdx_y() -> u32 {
with_ctx(|t| t.block_idx().y)
}
pub fn blockIdx_z() -> u32 {
with_ctx(|t| t.block_idx().z)
}
pub fn blockDim_x() -> u32 {
with_ctx(|t| t.block_dim().x)
}
pub fn blockDim_y() -> u32 {
with_ctx(|t| t.block_dim().y)
}
pub fn blockDim_z() -> u32 {
with_ctx(|t| t.block_dim().z)
}
pub fn gridDim_x() -> u32 {
with_ctx(|t| t.grid_dim().x)
}
pub fn warp_size() -> u32 {
with_ctx(|t| t.warp_size())
}
#[track_caller]
pub fn sync_threads() {
let at = Location::caller();
with_ctx(|t| t.sync_threads_at(at));
}
}
pub struct DisjointSlice<T> {
buf: GlobalBuf<T>,
}
impl<T> Clone for DisjointSlice<T> {
fn clone(&self) -> Self {
DisjointSlice {
buf: self.buf.clone(),
}
}
}
impl<T: Copy + Send + 'static> DisjointSlice<T> {
pub fn new(buf: &GlobalBuf<T>) -> Self {
DisjointSlice { buf: buf.clone() }
}
pub fn len(&self) -> usize {
self.buf.len()
}
pub fn is_empty(&self) -> bool {
self.buf.is_empty()
}
#[track_caller]
pub fn get_mut_indexed(&mut self) -> Option<(ElemMut<'_, T>, ThreadIndex<'_>)> {
let at = Location::caller();
let index = with_ctx(|t| t.global_linear());
let elem = with_ctx(|t| self.buf.elem_mut(t, index, true, at))?;
Some((
elem,
ThreadIndex {
index,
_not_send: PhantomData,
_scope: PhantomData,
},
))
}
#[track_caller]
pub fn get_mut(&mut self, idx: ThreadIndex<'_>) -> Option<ElemMut<'_, T>> {
let at = Location::caller();
let index = idx.get();
with_ctx(|t| self.buf.elem_mut(t, index, true, at))
}
#[track_caller]
pub fn get_unchecked_mut(&mut self, index: usize) -> ElemMut<'_, T> {
let at = Location::caller();
with_ctx(|t| self.buf.elem_mut(t, index, false, at))
.expect("riri: unchecked access is never None")
}
}
pub type AtomicOrdering = Ordering;
pub fn threadfence() {
with_ctx(|t| t.threadfence());
}
pub fn threadfence_system() {
with_ctx(|t| t.threadfence());
}
pub struct DeviceAtomic<T> {
buf: GlobalBuf<T>,
index: usize,
}
impl<T: Copy + Send + 'static> DeviceAtomic<T> {
pub fn at(buf: &GlobalBuf<T>, index: usize) -> Self {
DeviceAtomic {
buf: buf.clone(),
index,
}
}
#[track_caller]
pub fn load(&self, order: AtomicOrdering) -> T {
let at = Location::caller();
with_ctx(|t| self.buf.atomic_load_at(t, self.index, order, at))
}
#[track_caller]
pub fn store(&self, val: T, order: AtomicOrdering) {
let at = Location::caller();
with_ctx(|t| self.buf.atomic_store_at(t, self.index, val, order, at));
}
#[track_caller]
pub fn swap(&self, val: T, order: AtomicOrdering) -> T {
let at = Location::caller();
with_ctx(|t| self.buf.atomic_swap_at(t, self.index, val, order, at))
}
}
impl<T: Copy + Send + PartialEq + 'static> DeviceAtomic<T> {
#[track_caller]
pub fn compare_exchange(&self, current: T, new: T, order: AtomicOrdering) -> Result<T, T> {
let at = Location::caller();
with_ctx(|t| {
self.buf
.atomic_compare_exchange_at(t, self.index, current, new, order, at)
})
}
}
impl<T: Copy + Send + Ord + 'static> DeviceAtomic<T> {
#[track_caller]
pub fn fetch_min(&self, val: T, order: AtomicOrdering) -> T {
let at = Location::caller();
with_ctx(|t| self.buf.atomic_min_at(t, self.index, val, order, at))
}
#[track_caller]
pub fn fetch_max(&self, val: T, order: AtomicOrdering) -> T {
let at = Location::caller();
with_ctx(|t| self.buf.atomic_max_at(t, self.index, val, order, at))
}
}
impl<T: Copy + Send + std::ops::Add<Output = T> + 'static> DeviceAtomic<T> {
#[track_caller]
pub fn fetch_add(&self, val: T, order: AtomicOrdering) -> T {
let at = Location::caller();
with_ctx(|t| self.buf.atomic_add_at(t, self.index, val, order, at))
}
}
impl<T: Copy + Send + std::ops::Sub<Output = T> + 'static> DeviceAtomic<T> {
#[track_caller]
pub fn fetch_sub(&self, val: T, order: AtomicOrdering) -> T {
let at = Location::caller();
with_ctx(|t| self.buf.atomic_sub_at(t, self.index, val, order, at))
}
}
pub struct DeviceAtomicSlice<T> {
handles: Vec<DeviceAtomic<T>>,
}
impl<T: Copy + Send + 'static> DeviceAtomicSlice<T> {
pub fn new(buf: &GlobalBuf<T>) -> Self {
DeviceAtomicSlice {
handles: (0..buf.len()).map(|i| DeviceAtomic::at(buf, i)).collect(),
}
}
pub fn len(&self) -> usize {
self.handles.len()
}
pub fn is_empty(&self) -> bool {
self.handles.is_empty()
}
}
impl<T> std::ops::Index<usize> for DeviceAtomicSlice<T> {
type Output = DeviceAtomic<T>;
fn index(&self, index: usize) -> &DeviceAtomic<T> {
&self.handles[index]
}
}
pub mod warp {
use super::with_ctx;
use std::panic::Location;
pub fn lane_id() -> u32 {
with_ctx(|t| t.lane_id())
}
pub fn warp_id() -> u32 {
with_ctx(|t| t.warp_id())
}
#[track_caller]
pub fn shuffle(val: u32, src_lane: u32) -> u32 {
let at = Location::caller();
with_ctx(|t| crate::warp::shfl_sync_at(t, t.warp_valid_mask(), val, src_lane, at))
}
#[track_caller]
pub fn shuffle_f32(val: f32, src_lane: u32) -> f32 {
let at = Location::caller();
with_ctx(|t| crate::warp::shfl_sync_at(t, t.warp_valid_mask(), val, src_lane, at))
}
#[track_caller]
pub fn shuffle_up(val: u32, delta: u32) -> u32 {
let at = Location::caller();
with_ctx(|t| crate::warp::shfl_up_sync_at(t, t.warp_valid_mask(), val, delta, at))
}
#[track_caller]
pub fn shuffle_up_f32(val: f32, delta: u32) -> f32 {
let at = Location::caller();
with_ctx(|t| crate::warp::shfl_up_sync_at(t, t.warp_valid_mask(), val, delta, at))
}
#[track_caller]
pub fn shuffle_down(val: u32, delta: u32) -> u32 {
let at = Location::caller();
with_ctx(|t| crate::warp::shfl_down_sync_at(t, t.warp_valid_mask(), val, delta, at))
}
#[track_caller]
pub fn shuffle_down_f32(val: f32, delta: u32) -> f32 {
let at = Location::caller();
with_ctx(|t| crate::warp::shfl_down_sync_at(t, t.warp_valid_mask(), val, delta, at))
}
#[track_caller]
pub fn shuffle_xor(val: u32, lane_mask: u32) -> u32 {
let at = Location::caller();
with_ctx(|t| crate::warp::shfl_xor_sync_at(t, t.warp_valid_mask(), val, lane_mask, at))
}
#[track_caller]
pub fn shuffle_xor_f32(val: f32, lane_mask: u32) -> f32 {
let at = Location::caller();
with_ctx(|t| crate::warp::shfl_xor_sync_at(t, t.warp_valid_mask(), val, lane_mask, at))
}
#[track_caller]
pub fn all(predicate: bool) -> bool {
let at = Location::caller();
with_ctx(|t| {
crate::warp::ballot(t, t.warp_valid_mask(), predicate, "all", at) == t.warp_valid_mask()
})
}
#[track_caller]
pub fn any(predicate: bool) -> bool {
let at = Location::caller();
with_ctx(|t| crate::warp::ballot(t, t.warp_valid_mask(), predicate, "any", at) != 0)
}
#[track_caller]
pub fn ballot(predicate: bool) -> u32 {
let at = Location::caller();
with_ctx(|t| crate::warp::ballot(t, t.warp_valid_mask(), predicate, "ballot", at))
}
#[track_caller]
pub fn popc(predicate: bool) -> u32 {
ballot(predicate).count_ones()
}
}