riri 0.8.0

A Miri-style undefined-behaviour detector for Rust GPU kernels: run SIMT kernels on the CPU and catch data races, barrier and warp divergence, bad shuffle masks, and uninitialised reads of shared or global memory, understands release and acquire ordering between blocks, and shrinks a failing schedule to the few decisions that cause it.
Documentation
use std::any::Any;
use std::panic::Location;
use std::sync::{Arc, Mutex};

use crate::diag::{Access, AccessKind, Diagnostic};
use crate::dim::Dim3;
use crate::launch::LaunchState;
use crate::mem::SharedArray;
use crate::sched::AbortSignal;
use crate::sync::{key, Clock, Ordering};

/// Per-thread view of a running kernel: the equivalent of CUDA's built-in
/// `threadIdx`, `blockIdx`, `blockDim`, `gridDim`, plus `__syncthreads()`
/// and `__shared__` allocation.
pub struct ThreadCtx<'l> {
    pub(crate) launch: &'l LaunchState,
    pub(crate) block: u32,
    pub(crate) thread: u32,
    /// Global thread index, which the clocks and the kernel both use.
    pub(crate) gid: usize,
    /// Index within the resident wave, which is what the scheduler knows.
    pub(crate) slot: usize,
}

impl<'l> ThreadCtx<'l> {
    /// This thread's block as the current wave numbers it.
    fn wave_block(&self) -> usize {
        self.slot / self.launch.config.block.count() as usize
    }

    pub fn thread_idx(&self) -> Dim3 {
        Dim3::from_linear(self.thread, self.launch.config.block)
    }

    pub fn block_idx(&self) -> Dim3 {
        Dim3::from_linear(self.block, self.launch.config.grid)
    }

    pub fn block_dim(&self) -> Dim3 {
        self.launch.config.block
    }

    pub fn grid_dim(&self) -> Dim3 {
        self.launch.config.grid
    }

    /// Linear thread index within the block.
    pub fn thread_linear(&self) -> usize {
        self.thread as usize
    }

    /// Linear block index within the grid.
    pub fn block_linear(&self) -> usize {
        self.block as usize
    }

    /// `blockIdx * blockDim + threadIdx`, linearised over the whole launch.
    pub fn global_linear(&self) -> usize {
        self.gid
    }

    // ------------------------------------------------------------ warps ---

    /// Lanes per warp for this launch. 32 unless the launch config says
    /// otherwise.
    pub fn warp_size(&self) -> u32 {
        self.launch.config.warp_size
    }

    /// Index of this thread's warp within its block.
    pub fn warp_id(&self) -> u32 {
        self.thread / self.warp_size()
    }

    /// This thread's lane within its warp, in `0..warp_size()`.
    pub fn lane_id(&self) -> u32 {
        self.thread % self.warp_size()
    }

    /// Mask of the lanes of this thread's warp that actually exist.
    ///
    /// This is the full mask except in a block whose size is not a multiple
    /// of the warp size, where the last warp is short.
    pub fn warp_valid_mask(&self) -> u32 {
        self.launch.sched.valid_mask(self.warp_id() as usize)
    }

    /// `__syncthreads()`. Every thread of the block must reach the same
    /// barrier; otherwise Riri reports barrier divergence.
    #[track_caller]
    pub fn sync_threads(&self) {
        self.sync_threads_at(Location::caller());
    }

    /// `sync_threads` with the barrier's source location passed in, for
    /// callers reached through a closure, where `#[track_caller]` does not
    /// survive.
    pub(crate) fn sync_threads_at(&self, loc: &'static Location<'static>) {
        let block = self.block as usize;
        // Publish into the block before parking, take the block's out after
        // release. Every thread has arrived by then, so what comes back is
        // everything the block knows.
        self.launch
            .sync
            .lock()
            .unwrap()
            .barrier_arrive(self.gid, block);

        if self
            .launch
            .sched
            .barrier(self.slot, loc, &self.launch.reporter)
            .is_err()
        {
            std::panic::resume_unwind(Box::new(AbortSignal));
        }

        self.launch
            .sync
            .lock()
            .unwrap()
            .barrier_depart(self.gid, block);
    }

    /// `__threadfence()`: orders this thread's accesses against the atomics
    /// around it, without ordering the thread against anything on its own.
    ///
    /// Paired with relaxed atomics this is how blocks communicate. The
    /// producer writes its data, fences, then sets a flag; the consumer reads
    /// the flag, fences, then reads the data. Without the fences the two sides
    /// are unordered and Riri reports the data accesses as a race, which is
    /// what they are.
    pub fn threadfence(&self) {
        self.fence(Ordering::AcqRel);
    }

    /// A one-sided fence, for code that knows which side it needs.
    pub fn fence(&self, ordering: Ordering) {
        self.launch.sync.lock().unwrap().fence(self.gid, ordering);
    }

    /// A block-wide `__shared__` array. The first thread of the block to ask
    /// creates it; later calls with the same name return the same array.
    /// Elements start uninitialised: reading before writing is reported.
    ///
    /// Panics if the same name is reused with a different type or length.
    pub fn shared<T: Copy + Default + Send + 'static>(
        &self,
        name: &'static str,
        len: usize,
    ) -> SharedArray<T> {
        let mut map = self.launch.blocks[self.block as usize]
            .shared
            .lock()
            .unwrap();
        let entry = map
            .entry(name)
            .or_insert_with(|| {
                Arc::new(SharedArray::<T>::new_inner(self.block, name, len))
                    as Arc<dyn Any + Send + Sync>
            })
            .clone();
        drop(map);
        let arr = SharedArray::<T>::from_any(entry).unwrap_or_else(|| {
            panic!("riri: shared array `{name}` reused with a different element type")
        });
        assert_eq!(
            arr.len(),
            len,
            "riri: shared array `{name}` reused with a different length"
        );
        arr
    }

    // ----- crate-internal instrumentation hooks -----

    pub(crate) fn schedule_point(&self) {
        if self
            .launch
            .sched
            .yield_now(self.slot, &self.launch.reporter)
            .is_err()
        {
            std::panic::resume_unwind(Box::new(AbortSignal));
        }
    }

    /// Records one operation of this thread, returning both the access and
    /// the thread's clock as it now stands.
    ///
    /// The clock comes back with the access because deciding whether an
    /// earlier access is ordered against this one needs the state of the
    /// thread making it, not of the thread that made the earlier one.
    pub(crate) fn access(
        &self,
        kind: AccessKind,
        location: &'static Location<'static>,
    ) -> (Access, Clock) {
        let warp = self.warp_id();
        let (epoch, warp_epoch) = self.launch.sched.epochs(self.wave_block(), warp as usize);

        let mut sync = self.launch.sync.lock().unwrap();
        let seq = sync.tick(self.gid, key(self.block, self.thread));
        let now = sync.clock(self.gid).clone();

        let access = Access {
            block: self.block,
            thread: self.thread,
            epoch,
            warp,
            warp_epoch,
            seq,
            kind,
            location,
        };
        (access, now)
    }

    /// Publishes this thread's earlier work into `target`.
    pub(crate) fn release_into(&self, target: &mut Clock, ordering: Ordering) {
        self.launch
            .sync
            .lock()
            .unwrap()
            .release(self.gid, target, ordering);
    }

    /// Takes on whatever `source` published.
    pub(crate) fn acquire_from(&self, source: &Clock, ordering: Ordering) {
        self.launch
            .sync
            .lock()
            .unwrap()
            .acquire(self.gid, source, ordering);
    }

    /// The exchange slots this thread's warp uses to publish values to its
    /// mask-mates during a collective.
    pub(crate) fn warp_slots(&self) -> &Mutex<Vec<Option<Box<dyn Any + Send>>>> {
        let index = self.block as usize * self.launch.warps_per_block + self.warp_id() as usize;
        &self.launch.warps[index].slots
    }

    /// Parks at one half of a warp collective; unwinds if the launch aborts.
    pub(crate) fn warp_rendezvous(
        &self,
        phase: u8,
        mask: u32,
        at: &'static Location<'static>,
        op: &'static str,
    ) {
        let r = self.launch.sched.warp_rendezvous(
            self.slot,
            phase,
            mask,
            at,
            op,
            &self.launch.reporter,
        );
        if r.is_err() {
            std::panic::resume_unwind(Box::new(AbortSignal));
        }
    }

    pub(crate) fn report(&self, d: Diagnostic) {
        self.launch.reporter.push(d);
    }

    /// Reports a fatal error and aborts the launch, like a GPU trap.
    pub(crate) fn trap(&self, d: Diagnostic) -> ! {
        self.launch.reporter.push(d);
        self.launch.sched.abort();
        std::panic::resume_unwind(Box::new(AbortSignal));
    }

    pub(crate) fn launch_id(&self) -> u64 {
        self.launch.id
    }
}