use std::any::Any;
use std::collections::HashMap;
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use crate::ctx::ThreadCtx;
use crate::diag::{Diagnostic, Report, Reporter};
use crate::dim::Dim3;
use crate::sched::{AbortSignal, Choices, Scheduler, SplitMix64};
use crate::sync::SyncState;
use crate::warp::WARP_SIZE;
pub const MAX_THREADS: u32 = 16_384;
static NEXT_LAUNCH_ID: AtomicU64 = AtomicU64::new(1);
#[derive(Clone, Copy, Debug)]
pub struct LaunchConfig {
pub grid: Dim3,
pub block: Dim3,
pub seed: u64,
pub warp_size: u32,
pub resident_blocks: Option<u32>,
}
impl LaunchConfig {
pub fn new(grid: impl Into<Dim3>, block: impl Into<Dim3>) -> Self {
LaunchConfig {
grid: grid.into(),
block: block.into(),
seed: 0,
warp_size: WARP_SIZE,
resident_blocks: None,
}
}
pub fn seed(mut self, seed: u64) -> Self {
self.seed = seed;
self
}
pub fn warp_size(mut self, lanes: u32) -> Self {
assert!(
lanes.is_power_of_two() && lanes <= 32,
"riri: warp size must be a power of two no greater than 32, got {lanes}"
);
self.warp_size = lanes;
self
}
pub fn resident_blocks(mut self, blocks: u32) -> Self {
assert!(blocks > 0, "riri: at least one block must be resident");
self.resident_blocks = Some(blocks);
self
}
}
pub(crate) struct BlockState {
pub(crate) shared: Mutex<HashMap<&'static str, Arc<dyn Any + Send + Sync>>>,
}
pub(crate) struct WarpState {
pub(crate) slots: Mutex<Vec<Option<Box<dyn Any + Send>>>>,
}
pub(crate) struct LaunchState {
pub(crate) id: u64,
pub(crate) config: LaunchConfig,
pub(crate) sched: Scheduler,
pub(crate) first_block: u32,
pub(crate) reporter: Arc<Reporter>,
pub(crate) blocks: Arc<Vec<BlockState>>,
pub(crate) warps: Arc<Vec<WarpState>>,
pub(crate) warps_per_block: usize,
pub(crate) sync: Arc<Mutex<SyncState>>,
}
pub fn launch<F>(config: &LaunchConfig, kernel: F) -> Report
where
F: Fn(&ThreadCtx<'_>) + Sync,
{
run(config, Choices::Random(SplitMix64(config.seed)), kernel).0
}
pub(crate) fn run<F>(config: &LaunchConfig, choices: Choices, kernel: F) -> (Report, Vec<u32>, bool)
where
F: Fn(&ThreadCtx<'_>) + Sync,
{
run_shared(config, choices, |_, ctx| kernel(ctx))
}
pub(crate) fn run_shared<F>(
config: &LaunchConfig,
choices: Choices,
kernel: F,
) -> (Report, Vec<u32>, bool)
where
F: Fn(&Arc<LaunchState>, &ThreadCtx<'_>) + Sync,
{
let blocks = config.grid.count();
let tpb = config.block.count();
assert!(
blocks > 0 && tpb > 0,
"riri: grid and block must be non-empty"
);
assert!(
config.warp_size.is_power_of_two() && config.warp_size <= 32,
"riri: warp size must be a power of two no greater than 32, got {}",
config.warp_size
);
let total = blocks.checked_mul(tpb).expect("riri: launch too large");
let ws = config.warp_size as usize;
let warps_per_block = (tpb as usize).div_ceil(ws);
let resident = config.resident_blocks.unwrap_or(blocks).clamp(1, blocks);
let wave_threads = resident.checked_mul(tpb).expect("riri: wave too large");
assert!(
wave_threads <= MAX_THREADS,
"riri: {wave_threads} resident threads exceeds the limit of {MAX_THREADS}; lower resident_blocks or shrink the block"
);
let waves = blocks.div_ceil(resident);
let id = NEXT_LAUNCH_ID.fetch_add(1, Ordering::Relaxed);
let reporter = Arc::new(Reporter::default());
let block_state: Arc<Vec<BlockState>> = Arc::new(
(0..blocks)
.map(|_| BlockState {
shared: Mutex::new(HashMap::new()),
})
.collect(),
);
let warp_state: Arc<Vec<WarpState>> = Arc::new(
(0..blocks as usize * warps_per_block)
.map(|_| WarpState {
slots: Mutex::new((0..ws).map(|_| None).collect()),
})
.collect(),
);
let sync = Arc::new(Mutex::new(SyncState::new(total as usize, blocks as usize)));
let mut trace = Vec::new();
let mut complete = waves == 1;
let mut aborted = false;
let mut choices = Some(choices);
let mut first_block = 0u32;
for wave in 0..waves {
let wave_blocks = resident.min(blocks - first_block);
let wave_total = (wave_blocks * tpb) as usize;
let picks = choices
.take()
.unwrap_or_else(|| Choices::Random(SplitMix64(config.seed.wrapping_add(wave as u64))));
let state = Arc::new(LaunchState {
id,
config: *config,
sched: Scheduler::new(wave_blocks as usize, tpb as usize, ws, picks),
first_block,
reporter: Arc::clone(&reporter),
blocks: Arc::clone(&block_state),
warps: Arc::clone(&warp_state),
warps_per_block,
sync: Arc::clone(&sync),
});
std::thread::scope(|scope| {
for slot in 0..wave_total {
let state = &state;
let kernel = &kernel;
std::thread::Builder::new()
.name(format!(
"riri-{}",
first_block as usize * tpb as usize + slot
))
.stack_size(256 * 1024)
.spawn_scoped(scope, move || run_thread(state, kernel, slot))
.expect("riri: failed to spawn simulated thread");
}
});
if waves == 1 {
let (recorded, recorded_complete) = state.sched.take_trace();
trace = recorded;
complete = recorded_complete;
}
if state.sched.was_aborted() {
aborted = true;
break;
}
first_block += wave_blocks;
}
let report = Report {
seed: config.seed,
diagnostics: reporter.take(),
aborted,
};
(report, trace, complete)
}
fn run_thread<F>(state: &Arc<LaunchState>, kernel: &F, slot: usize)
where
F: Fn(&Arc<LaunchState>, &ThreadCtx<'_>) + Sync,
{
let tpb = state.config.block.count() as usize;
let block = state.first_block + (slot / tpb) as u32;
let thread = (slot % tpb) as u32;
let ctx = ThreadCtx {
launch: state,
block,
thread,
gid: block as usize * tpb + thread as usize,
slot,
};
if state.sched.wait_turn(slot).is_err() {
return;
}
match catch_unwind(AssertUnwindSafe(|| kernel(state, &ctx))) {
Ok(()) => state.sched.finish(slot, &state.reporter),
Err(payload) if payload.is::<AbortSignal>() => {}
Err(payload) => {
let message = payload
.downcast_ref::<&str>()
.map(|s| s.to_string())
.or_else(|| payload.downcast_ref::<String>().cloned())
.unwrap_or_else(|| "<non-string panic payload>".into());
state.reporter.push(Diagnostic::KernelPanic {
block: ctx.block,
thread: ctx.thread,
message,
});
state.sched.abort();
}
}
}