#![allow(clippy::missing_errors_doc)]
use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::thread::JoinHandle;
use cudarc::driver::{CudaContext, CudaFunction, CudaModule, CudaStream, DriverError, LaunchConfig, PushKernelArg};
use crate::sched::notify_ring::{NotifyHub, NotifySender};
use crate::backend::{
Backend, BackendCapabilities, BackendError, DispatchBackend, KernelArg, KernelHandle,
};
type WorkItem = Box<dyn FnOnce() + Send + 'static>;
pub struct CudaBackend {
device_id: u32,
context: Arc<CudaContext>,
stream: Arc<CudaStream>,
secondary_stream: Arc<CudaStream>,
caps: BackendCapabilities,
next_handle: AtomicU64,
modules: Mutex<HashMap<u64, Arc<CudaModule>>>,
functions: Mutex<HashMap<u64, CudaFunction>>,
worker_hub: NotifyHub<WorkItem>,
worker_tx: NotifySender<WorkItem>,
worker_handle: Mutex<Option<JoinHandle<()>>>,
}
impl std::fmt::Debug for CudaBackend {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("CudaBackend")
.field("device_id", &self.device_id)
.finish()
}
}
impl CudaBackend {
pub fn new() -> Result<Self, BackendError> {
Self::with_device(0)
}
pub fn with_device(device_id: u32) -> Result<Self, BackendError> {
let context =
CudaContext::new(device_id as usize).map_err(|e| map_driver_error(device_id, e))?;
let stream = context.default_stream();
let secondary_stream = context
.new_stream()
.map_err(|e| map_driver_error(device_id, e))?;
let caps = probe_capabilities();
const CUDA_WORKER_RING_CAPACITY: usize = 1024;
let worker_hub = NotifyHub::<WorkItem>::new(CUDA_WORKER_RING_CAPACITY, 1);
let worker_tx = worker_hub.sender();
let hub_for_worker = worker_hub.clone();
let worker_handle = std::thread::Builder::new()
.name(format!("flynnel-cuda-{device_id}"))
.spawn(move || {
let rx = hub_for_worker.register_consumer();
while let Some(work) = rx.recv() {
work();
}
})
.map_err(|e| {
BackendError::DeviceUnavailable(Backend::Cuda { device_id })
.map_io_context(format!("worker thread spawn: {e}"))
})?;
Ok(Self {
device_id,
context,
stream,
secondary_stream,
caps,
next_handle: AtomicU64::new(1),
modules: Mutex::new(HashMap::new()),
functions: Mutex::new(HashMap::new()),
worker_hub,
worker_tx,
worker_handle: Mutex::new(Some(worker_handle)),
})
}
pub fn context(&self) -> &Arc<CudaContext> {
&self.context
}
pub fn stream(&self) -> &Arc<CudaStream> {
&self.stream
}
pub fn secondary_stream(&self) -> &Arc<CudaStream> {
&self.secondary_stream
}
pub fn stream_for_slot(&self, slot: usize) -> &Arc<CudaStream> {
if slot & 1 == 0 {
&self.stream
} else {
&self.secondary_stream
}
}
pub fn dispatch_kernel_on_stream(
&self,
stream: &Arc<CudaStream>,
handle: KernelHandle,
count: u32,
args: &[KernelArg<'_>],
) -> Result<(), BackendError> {
let function = {
let guard = self
.functions
.lock()
.map_err(|_| BackendError::Launch("functions mutex poisoned".into()))?;
guard
.get(&handle.0)
.cloned()
.ok_or_else(|| BackendError::Launch(format!("unknown kernel handle {handle:?}")))?
};
let block = 256u32.min(count.max(1));
let grid = count.div_ceil(block);
let cfg = LaunchConfig {
grid_dim: (grid, 1, 1),
block_dim: (block, 1, 1),
shared_mem_bytes: 0,
};
let mut builder = stream.launch_builder(&function);
let mut i32s: Vec<i32> = Vec::new();
let mut i64s: Vec<i64> = Vec::new();
let mut u32s: Vec<u32> = Vec::new();
let mut u64s: Vec<u64> = Vec::new();
let mut f32s: Vec<f32> = Vec::new();
let mut f64s: Vec<f64> = Vec::new();
for arg in args {
match arg {
KernelArg::I32(v) => i32s.push(*v),
KernelArg::I64(v) => i64s.push(*v),
KernelArg::U32(v) => u32s.push(*v),
KernelArg::U64(v) => u64s.push(*v),
KernelArg::F32(v) => f32s.push(*v),
KernelArg::F64(v) => f64s.push(*v),
KernelArg::DevicePtr(p) => u64s.push(*p as u64),
KernelArg::HostSlice(_) => return Err(BackendError::NotSupported),
}
}
let (mut ii32, mut ii64, mut iu32, mut iu64, mut if32, mut if64) =
(0usize, 0usize, 0usize, 0usize, 0usize, 0usize);
for arg in args {
match arg {
KernelArg::I32(_) => {
builder.arg(&i32s[ii32]);
ii32 += 1;
}
KernelArg::I64(_) => {
builder.arg(&i64s[ii64]);
ii64 += 1;
}
KernelArg::U32(_) => {
builder.arg(&u32s[iu32]);
iu32 += 1;
}
KernelArg::U64(_) => {
builder.arg(&u64s[iu64]);
iu64 += 1;
}
KernelArg::F32(_) => {
builder.arg(&f32s[if32]);
if32 += 1;
}
KernelArg::F64(_) => {
builder.arg(&f64s[if64]);
if64 += 1;
}
KernelArg::DevicePtr(_) => {
builder.arg(&u64s[iu64]);
iu64 += 1;
}
KernelArg::HostSlice(_) => unreachable!("rejected in first pass"),
}
}
unsafe { builder.launch(cfg) }
.map_err(|e| BackendError::Launch(format!("{e:?}")))?;
Ok(())
}
}
impl DispatchBackend for CudaBackend {
fn id(&self) -> Backend {
Backend::Cuda {
device_id: self.device_id,
}
}
fn capabilities(&self) -> BackendCapabilities {
self.caps
}
fn dispatch_parallel_for(&self, count: u32, work: &(dyn Fn(u32) + Send + Sync)) {
if count == 0 {
return;
}
std::thread::scope(|scope| {
let threads = (count as usize).min(
std::thread::available_parallelism()
.map(std::num::NonZeroUsize::get)
.unwrap_or(1),
);
let chunks = count.div_ceil(threads as u32);
for t in 0..threads as u32 {
let lo = t.saturating_mul(chunks);
let hi = (lo + chunks).min(count);
if lo >= hi {
continue;
}
scope.spawn(move || {
for i in lo..hi {
work(i);
}
});
}
});
}
fn dispatch_one(&self, work: Box<dyn FnOnce() + Send>) {
drop(self.worker_tx.send(work));
}
fn register_kernel(&self, name: &str, source: &[u8]) -> Result<KernelHandle, BackendError> {
let ptx_text = std::str::from_utf8(source)
.map_err(|e| BackendError::KernelCompile(format!("PTX must be UTF-8: {e}")))?;
let ptx = cudarc::nvrtc::Ptx::from_src(ptx_text);
let module = self
.context
.load_module(ptx)
.map_err(|e| BackendError::KernelCompile(format!("{e:?}")))?;
let function = module
.load_function(name)
.map_err(|e| BackendError::KernelCompile(format!("function lookup `{name}`: {e:?}")))?;
let handle_id = self.next_handle.fetch_add(1, Ordering::Relaxed);
self.modules
.lock()
.map_err(|_| BackendError::KernelCompile("modules mutex poisoned".into()))?
.insert(handle_id, module);
self.functions
.lock()
.map_err(|_| BackendError::KernelCompile("functions mutex poisoned".into()))?
.insert(handle_id, function);
Ok(KernelHandle(handle_id))
}
fn dispatch_kernel(
&self,
handle: KernelHandle,
count: u32,
args: &[KernelArg<'_>],
) -> Result<(), BackendError> {
let function = {
let guard = self
.functions
.lock()
.map_err(|_| BackendError::Launch("functions mutex poisoned".into()))?;
guard
.get(&handle.0)
.cloned()
.ok_or_else(|| BackendError::Launch(format!("unknown kernel handle {handle:?}")))?
};
let block = 256u32.min(count.max(1));
let grid = count.div_ceil(block);
let cfg = LaunchConfig {
grid_dim: (grid, 1, 1),
block_dim: (block, 1, 1),
shared_mem_bytes: 0,
};
let mut builder = self.stream.launch_builder(&function);
let mut i32s: Vec<i32> = Vec::new();
let mut i64s: Vec<i64> = Vec::new();
let mut u32s: Vec<u32> = Vec::new();
let mut u64s: Vec<u64> = Vec::new();
let mut f32s: Vec<f32> = Vec::new();
let mut f64s: Vec<f64> = Vec::new();
for arg in args {
match arg {
KernelArg::I32(v) => i32s.push(*v),
KernelArg::I64(v) => i64s.push(*v),
KernelArg::U32(v) => u32s.push(*v),
KernelArg::U64(v) => u64s.push(*v),
KernelArg::F32(v) => f32s.push(*v),
KernelArg::F64(v) => f64s.push(*v),
KernelArg::DevicePtr(p) => u64s.push(*p as u64),
KernelArg::HostSlice(_) => return Err(BackendError::NotSupported),
}
}
let (mut ii32, mut ii64, mut iu32, mut iu64, mut if32, mut if64) =
(0usize, 0usize, 0usize, 0usize, 0usize, 0usize);
for arg in args {
match arg {
KernelArg::I32(_) => {
builder.arg(&i32s[ii32]);
ii32 += 1;
}
KernelArg::I64(_) => {
builder.arg(&i64s[ii64]);
ii64 += 1;
}
KernelArg::U32(_) => {
builder.arg(&u32s[iu32]);
iu32 += 1;
}
KernelArg::U64(_) => {
builder.arg(&u64s[iu64]);
iu64 += 1;
}
KernelArg::F32(_) => {
builder.arg(&f32s[if32]);
if32 += 1;
}
KernelArg::F64(_) => {
builder.arg(&f64s[if64]);
if64 += 1;
}
KernelArg::DevicePtr(_) => {
builder.arg(&u64s[iu64]);
iu64 += 1;
}
KernelArg::HostSlice(_) => unreachable!("rejected in first pass"),
}
}
unsafe { builder.launch(cfg) }
.map_err(|e| BackendError::Launch(format!("{e:?}")))?;
Ok(())
}
fn dispatch_kernel_sync(
&self,
handle: KernelHandle,
count: u32,
args: &[KernelArg<'_>],
) -> Result<(), BackendError> {
self.dispatch_kernel(handle, count, args)?;
self.stream
.synchronize()
.map_err(|e| BackendError::Launch(format!("stream synchronize: {e:?}")))
}
}
fn probe_capabilities() -> BackendCapabilities {
BackendCapabilities {
simt_width: 32,
max_threads_in_flight: 100_000,
launch_latency_ns: 10_000,
h2d_bw_bytes_per_sec: 25_000_000_000,
}
}
fn map_driver_error(device_id: u32, _e: DriverError) -> BackendError {
BackendError::DeviceUnavailable(Backend::Cuda { device_id })
}
impl Drop for CudaBackend {
fn drop(&mut self) {
self.worker_hub.shutdown();
if let Ok(mut guard) = self.worker_handle.lock()
&& let Some(handle) = guard.take()
{
drop(handle.join());
}
}
}
trait WithIoContext: Sized {
fn map_io_context(self, msg: String) -> Self;
}
impl WithIoContext for BackendError {
fn map_io_context(self, msg: String) -> Self {
match self {
BackendError::DeviceUnavailable(b) => {
eprintln!("[flynnel::cuda] {}: {msg}", b.name());
BackendError::DeviceUnavailable(b)
}
other => other,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::backend::detect::cuda_available;
#[test]
fn cuda_backend_construction_matches_availability() {
let res = CudaBackend::new();
if cuda_available() {
match res {
Ok(b) => assert_eq!(b.id(), Backend::Cuda { device_id: 0 }),
Err(BackendError::DeviceUnavailable(_)) => {}
Err(e) => panic!("unexpected CUDA construction error: {e}"),
}
} else {
assert!(matches!(res, Err(BackendError::DeviceUnavailable(_))));
}
}
#[test]
fn capabilities_report_warp_width_32() {
if let Ok(b) = CudaBackend::new() {
assert_eq!(b.capabilities().simt_width, 32);
}
}
#[test]
fn dispatch_parallel_for_invokes_each_index_on_host_fanout() {
let Ok(backend) = CudaBackend::new() else {
return;
};
use std::sync::atomic::AtomicU32;
let counters: Vec<AtomicU32> = (0..256).map(|_| AtomicU32::new(0)).collect();
let cref = &counters;
backend.dispatch_parallel_for(256, &|i| {
cref[i as usize].fetch_add(1, std::sync::atomic::Ordering::Relaxed);
});
for c in &counters {
assert_eq!(c.load(std::sync::atomic::Ordering::Relaxed), 1);
}
}
#[test]
fn register_and_dispatch_trivial_kernel() {
let Ok(backend) = CudaBackend::new() else {
return;
};
const TRIVIAL_PTX: &str = include_str!("../../kernels/mul3.ptx");
let handle = backend
.register_kernel("mul3", TRIVIAL_PTX.as_bytes())
.expect("registering trivial PTX kernel");
backend
.dispatch_kernel(handle, 32, &[KernelArg::U32(7)])
.expect("dispatching trivial PTX kernel");
backend
.stream()
.synchronize()
.expect("sync after trivial kernel dispatch");
}
}