#![allow(clippy::missing_errors_doc)]
use std::marker::PhantomData;
use std::ptr::NonNull;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicPtr, AtomicU64, Ordering};
use std::thread::JoinHandle;
use cudarc::driver::{
CudaContext, CudaEvent, CudaFunction, CudaModule, CudaSlice, CudaStream, DriverError,
LaunchArgs, LaunchConfig, PushKernelArg, result, sys,
};
use crate::sched::notify_ring::{NotifyHub, NotifySendResult, NotifySender};
use crate::backend::{
Backend, BackendCapabilities, BackendError, DeviceArg, DispatchBackend, KernelArg, KernelHandle,
};
type WorkItem = Box<dyn FnOnce() + Send + 'static>;
mod sealed {
pub trait Sealed:
cudarc::driver::DeviceRepr + cudarc::driver::ValidAsZeroBits + Copy + Send + Sync + 'static
{
}
impl Sealed for u8 {}
impl Sealed for i32 {}
impl Sealed for u32 {}
impl Sealed for i64 {}
impl Sealed for u64 {}
impl Sealed for f32 {}
impl Sealed for f64 {}
}
pub trait DeviceElement: sealed::Sealed {}
impl DeviceElement for u8 {}
impl DeviceElement for i32 {}
impl DeviceElement for u32 {}
impl DeviceElement for i64 {}
impl DeviceElement for u64 {}
impl DeviceElement for f32 {}
impl DeviceElement for f64 {}
pub struct DeviceBuffer<T: DeviceElement> {
slice: CudaSlice<T>,
}
impl<T: DeviceElement> DeviceBuffer<T> {
pub fn len(&self) -> usize {
self.slice.len()
}
pub fn is_empty(&self) -> bool {
self.slice.is_empty()
}
pub fn arg(&self) -> KernelArg<'_> {
KernelArg::Buffer(self)
}
}
impl<T: DeviceElement> DeviceArg for DeviceBuffer<T> {
fn as_any(&self) -> &dyn std::any::Any {
self
}
}
impl<T: DeviceElement> std::fmt::Debug for DeviceBuffer<T> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DeviceBuffer")
.field("element", &std::any::type_name::<T>())
.field("len", &self.slice.len())
.field("device", &self.slice.ordinal())
.finish()
}
}
pub struct MappedBuffer<T: DeviceElement> {
memory: MappedMemory,
len: usize,
element: PhantomData<T>,
}
impl<T: DeviceElement> MappedBuffer<T> {
pub fn len(&self) -> usize {
self.len
}
pub fn is_empty(&self) -> bool {
self.len == 0
}
pub fn arg(&self) -> KernelArg<'_> {
KernelArg::Buffer(&self.memory)
}
pub fn as_slice(&mut self) -> Result<&[T], BackendError> {
self.memory.settle()?;
Ok(unsafe { std::slice::from_raw_parts(self.memory.host.as_ptr().cast::<T>(), self.len) })
}
pub fn as_mut_slice(&mut self) -> Result<&mut [T], BackendError> {
self.memory.settle()?;
Ok(unsafe {
std::slice::from_raw_parts_mut(self.memory.host.as_ptr().cast::<T>(), self.len)
})
}
}
impl<T: DeviceElement> std::fmt::Debug for MappedBuffer<T> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("MappedBuffer")
.field("element", &std::any::type_name::<T>())
.field("len", &self.len)
.field("device", &self.memory.event.context().ordinal())
.finish()
}
}
const PER_THREAD_STREAM: u64 = 0x2;
struct MappedMemory {
host: NonNull<u8>,
device: u64,
bytes: usize,
event: CudaEvent,
stream: AtomicU64,
mixed: AtomicBool,
}
unsafe impl Send for MappedMemory {}
unsafe impl Sync for MappedMemory {}
impl MappedMemory {
fn note_launch(&self, stream: &CudaStream) -> Result<(), DriverError> {
let handle = stream.cu_stream() as usize as u64;
if handle == PER_THREAD_STREAM {
self.mixed.store(true, Ordering::Relaxed);
} else {
let tag = handle.wrapping_add(1);
if let Err(held) =
self.stream
.compare_exchange(0, tag, Ordering::Relaxed, Ordering::Relaxed)
&& held != tag
{
self.mixed.store(true, Ordering::Relaxed);
}
}
let recorded = self.event.record(stream);
if recorded.is_err() {
self.mixed.store(true, Ordering::Relaxed);
}
recorded
}
fn settle(&mut self) -> Result<(), BackendError> {
if *self.mixed.get_mut() {
self.event.context().synchronize().map_err(|e| {
memory_error(
"waiting for the device, which more than one stream used, before a \
mapped buffer's host slice"
.to_string(),
e,
)
})?;
} else {
self.event.synchronize().map_err(|e| {
memory_error("waiting for a mapped buffer's last launch".to_string(), e)
})?;
}
*self.mixed.get_mut() = false;
*self.stream.get_mut() = 0;
Ok(())
}
}
impl DeviceArg for MappedMemory {
fn as_any(&self) -> &dyn std::any::Any {
self
}
}
impl std::fmt::Debug for MappedMemory {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("MappedMemory")
.field("bytes", &self.bytes)
.field("device", &self.event.context().ordinal())
.finish()
}
}
impl Drop for MappedMemory {
fn drop(&mut self) {
if let Err(e) = self.settle() {
notice!(
"[flynnel::cuda] freeing {} bytes of mapped memory after its launches \
could not be waited for: {e}",
self.bytes
);
}
if let Err(e) = self.event.context().bind_to_thread() {
notice!("[flynnel::cuda] binding the context to free mapped memory: {e:?}");
}
if let Err(e) = unsafe { result::free_host(self.host.as_ptr().cast()) } {
notice!(
"[flynnel::cuda] the driver refused to free {} bytes of mapped memory: {e:?}",
self.bytes
);
}
}
}
enum BufferRef<'a> {
U8(&'a CudaSlice<u8>),
I32(&'a CudaSlice<i32>),
U32(&'a CudaSlice<u32>),
I64(&'a CudaSlice<i64>),
U64(&'a CudaSlice<u64>),
F32(&'a CudaSlice<f32>),
F64(&'a CudaSlice<f64>),
Mapped(&'a MappedMemory),
}
impl<'a> BufferRef<'a> {
fn of(arg: &'a dyn DeviceArg) -> Option<Self> {
let any = arg.as_any();
macro_rules! find {
($($t:ty => $variant:ident),*) => {
$(
if let Some(buffer) = any.downcast_ref::<DeviceBuffer<$t>>() {
return Some(BufferRef::$variant(&buffer.slice));
}
)*
};
}
find!(u8 => U8, i32 => I32, u32 => U32, i64 => I64, u64 => U64, f32 => F32, f64 => F64);
any.downcast_ref::<MappedMemory>().map(BufferRef::Mapped)
}
fn context(&self) -> &Arc<CudaContext> {
match self {
BufferRef::U8(s) => s.context(),
BufferRef::I32(s) => s.context(),
BufferRef::U32(s) => s.context(),
BufferRef::I64(s) => s.context(),
BufferRef::U64(s) => s.context(),
BufferRef::F32(s) => s.context(),
BufferRef::F64(s) => s.context(),
BufferRef::Mapped(m) => m.event.context(),
}
}
fn push_to<'b>(&self, builder: &mut LaunchArgs<'b>)
where
'a: 'b,
{
match *self {
BufferRef::U8(s) => builder.arg(s),
BufferRef::I32(s) => builder.arg(s),
BufferRef::U32(s) => builder.arg(s),
BufferRef::I64(s) => builder.arg(s),
BufferRef::U64(s) => builder.arg(s),
BufferRef::F32(s) => builder.arg(s),
BufferRef::F64(s) => builder.arg(s),
BufferRef::Mapped(m) => builder.arg(&m.device),
};
}
}
pub struct CudaBackend {
device_id: u32,
context: Arc<CudaContext>,
stream: Arc<CudaStream>,
secondary_stream: Arc<CudaStream>,
caps: BackendCapabilities,
next_handle: AtomicU64,
kernels: Box<[AtomicPtr<LoadedKernel>; MAX_KERNELS]>,
worker_hub: NotifyHub<WorkItem>,
worker_tx: NotifySender<WorkItem>,
worker_handle: Option<JoinHandle<()>>,
}
struct LoadedKernel {
_module: Arc<CudaModule>,
function: CudaFunction,
}
const MAX_KERNELS: usize = 1024;
const JIT_LOG_BYTES: usize = 16 * 1024;
impl std::fmt::Debug for CudaBackend {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("CudaBackend")
.field("device_id", &self.device_id)
.finish()
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
#[non_exhaustive]
pub struct DeviceInfo {
pub name: String,
pub capability: u32,
pub multiprocessors: u32,
pub clock_khz: u32,
pub fp64_ratio: u32,
pub memory_bytes: u64,
pub integrated: bool,
pub maps_host_memory: bool,
pub unified_addressing: bool,
pub host_native_atomics: bool,
pub watchdog: bool,
pub l2_cache_bytes: u32,
pub memory_clock_khz: u32,
pub memory_bus_bits: u32,
pub warp_size: u32,
pub max_threads_per_block: u32,
pub max_threads_per_multiprocessor: u32,
pub shared_memory_per_block: u32,
pub async_engines: u32,
}
impl DeviceInfo {
pub fn of_ordinal(ordinal: u32) -> Result<Self, BackendError> {
use sys::CUdevice_attribute as A;
driver_loadable(ordinal)?;
let unavailable = |what: String| {
BackendError::DeviceUnavailable(Backend::Cuda { device_id: ordinal })
.map_io_context(what)
};
result::init().map_err(|e| unavailable(format!("initializing the driver: {e:?}")))?;
let index = i32::try_from(ordinal).map_err(|e| {
unavailable(format!("device {ordinal} is past the driver's range: {e}"))
})?;
let device = result::device::get(index)
.map_err(|e| unavailable(format!("no device {ordinal}: {e:?}")))?;
let fact = |attribute: A| -> Result<u32, BackendError> {
let value = unsafe { result::device::get_attribute(device, attribute) }
.map_err(|e| unavailable(format!("reading {attribute:?}: {e:?}")))?;
u32::try_from(value)
.map_err(|e| unavailable(format!("{attribute:?} answered {value}: {e}")))
};
let flag = |attribute: A| fact(attribute).map(|value| value != 0);
let name = result::device::get_name(device)
.map_err(|e| unavailable(format!("reading the name: {e:?}")))?;
let memory_bytes = unsafe { result::device::total_mem(device) }
.map_err(|e| unavailable(format!("reading the total memory: {e:?}")))?;
Ok(Self {
name,
capability: fact(A::CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR)? * 10
+ fact(A::CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR)?,
multiprocessors: fact(A::CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT)?,
clock_khz: fact(A::CU_DEVICE_ATTRIBUTE_CLOCK_RATE)?,
fp64_ratio: fact(A::CU_DEVICE_ATTRIBUTE_SINGLE_TO_DOUBLE_PRECISION_PERF_RATIO)?,
memory_bytes: memory_bytes as u64,
integrated: flag(A::CU_DEVICE_ATTRIBUTE_INTEGRATED)?,
maps_host_memory: flag(A::CU_DEVICE_ATTRIBUTE_CAN_MAP_HOST_MEMORY)?,
unified_addressing: flag(A::CU_DEVICE_ATTRIBUTE_UNIFIED_ADDRESSING)?,
host_native_atomics: flag(A::CU_DEVICE_ATTRIBUTE_HOST_NATIVE_ATOMIC_SUPPORTED)?,
watchdog: flag(A::CU_DEVICE_ATTRIBUTE_KERNEL_EXEC_TIMEOUT)?,
l2_cache_bytes: fact(A::CU_DEVICE_ATTRIBUTE_L2_CACHE_SIZE)?,
memory_clock_khz: fact(A::CU_DEVICE_ATTRIBUTE_MEMORY_CLOCK_RATE)?,
memory_bus_bits: fact(A::CU_DEVICE_ATTRIBUTE_GLOBAL_MEMORY_BUS_WIDTH)?,
warp_size: fact(A::CU_DEVICE_ATTRIBUTE_WARP_SIZE)?,
max_threads_per_block: fact(A::CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK)?,
max_threads_per_multiprocessor: fact(
A::CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR,
)?,
shared_memory_per_block: fact(A::CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK)?,
async_engines: fact(A::CU_DEVICE_ATTRIBUTE_ASYNC_ENGINE_COUNT)?,
})
}
}
impl CudaBackend {
fn loaded(&self, handle: KernelHandle) -> Result<&LoadedKernel, BackendError> {
let unknown = || BackendError::Launch(format!("unknown kernel handle {handle:?}"));
let slot = self.kernels.get(handle.0 as usize).ok_or_else(unknown)?;
let published = slot.load(Ordering::Acquire);
if published.is_null() {
return Err(unknown());
}
Ok(unsafe { &*published })
}
pub fn new() -> Result<Self, BackendError> {
Self::with_device(0)
}
pub fn with_device(device_id: u32) -> Result<Self, BackendError> {
driver_loadable(device_id)?;
let context =
CudaContext::new(device_id as usize).map_err(|e| map_driver_error(device_id, e))?;
let stream = context.default_stream();
let secondary_stream = context
.new_stream()
.map_err(|e| map_driver_error(device_id, e))?;
let caps = probe_capabilities();
const CUDA_WORKER_RING_CAPACITY: usize = 1024;
let worker_hub = NotifyHub::<WorkItem>::new(CUDA_WORKER_RING_CAPACITY, 1);
let worker_tx = worker_hub.sender();
let hub_for_worker = worker_hub.clone();
let worker_handle = std::thread::Builder::new()
.name(format!("flynnel-cuda-{device_id}"))
.spawn(move || {
let rx = hub_for_worker.register_consumer();
while let Some(work) = rx.recv() {
work();
}
})
.map_err(|e| {
BackendError::DeviceUnavailable(Backend::Cuda { device_id })
.map_io_context(format!("worker thread spawn: {e}"))
})?;
Ok(Self {
device_id,
context,
stream,
secondary_stream,
caps,
next_handle: AtomicU64::new(1),
kernels: Box::new([const { AtomicPtr::new(core::ptr::null_mut()) }; MAX_KERNELS]),
worker_hub,
worker_tx,
worker_handle: Some(worker_handle),
})
}
pub fn context(&self) -> &Arc<CudaContext> {
&self.context
}
pub fn stream(&self) -> &Arc<CudaStream> {
&self.stream
}
pub fn secondary_stream(&self) -> &Arc<CudaStream> {
&self.secondary_stream
}
pub fn stream_for_slot(&self, slot: usize) -> &Arc<CudaStream> {
if slot & 1 == 0 {
&self.stream
} else {
&self.secondary_stream
}
}
pub fn device_name(&self) -> Result<String, BackendError> {
self.context
.name()
.map_err(|e| map_driver_error(self.device_id, e))
}
pub fn mem_info(&self) -> Result<(usize, usize), BackendError> {
self.context
.mem_get_info()
.map_err(|e| memory_error(format!("reading device {}'s memory", self.device_id), e))
}
pub fn device_count() -> Result<u32, BackendError> {
driver_loadable(0)?;
result::init().map_err(|e| map_driver_error(0, e))?;
let count = result::device::get_count().map_err(|e| map_driver_error(0, e))?;
u32::try_from(count).map_err(|e| {
BackendError::DeviceUnavailable(Backend::Cuda { device_id: 0 })
.map_io_context(format!("the driver counted {count} devices: {e}"))
})
}
pub fn device_info(&self) -> Result<DeviceInfo, BackendError> {
DeviceInfo::of_ordinal(self.device_id)
}
pub fn map_host<T: DeviceElement>(&self, len: usize) -> Result<MappedBuffer<T>, BackendError> {
if len == 0 {
return Err(BackendError::Memory(
"mapped memory needs at least one element".to_string(),
));
}
let what = || {
format!(
"mapping {len} {} for device {}",
std::any::type_name::<T>(),
self.device_id
)
};
let bytes = len
.checked_mul(std::mem::size_of::<T>())
.ok_or_else(|| BackendError::Memory(format!("{}: past the address space", what())))?;
let event = self
.context
.new_event(None)
.map_err(|e| memory_error(what(), e))?;
self.context
.bind_to_thread()
.map_err(|e| memory_error(what(), e))?;
let raw = unsafe { result::malloc_host(bytes, sys::CU_MEMHOSTALLOC_DEVICEMAP) }
.map_err(|e| memory_error(what(), e))?;
let Some(host) = NonNull::new(raw.cast::<u8>()) else {
return Err(BackendError::Memory(format!(
"{}: the driver answered a null address",
what()
)));
};
let mut device: sys::CUdeviceptr = 0;
let mapped =
unsafe { sys::cuMemHostGetDevicePointer_v2(&mut device, host.as_ptr().cast(), 0) }
.result();
if let Err(e) = mapped {
if let Err(freed) = unsafe { result::free_host(host.as_ptr().cast()) } {
notice!(
"[flynnel::cuda] freeing {bytes} bytes the device could not map: {freed:?}"
);
}
return Err(memory_error(format!("{}: no device address", what()), e));
}
unsafe { host.as_ptr().write_bytes(0, bytes) };
Ok(MappedBuffer {
memory: MappedMemory {
host,
device,
bytes,
event,
stream: AtomicU64::new(0),
mixed: AtomicBool::new(false),
},
len,
element: PhantomData,
})
}
pub fn alloc_zeroed<T: DeviceElement>(
&self,
len: usize,
) -> Result<DeviceBuffer<T>, BackendError> {
if len == 0 {
return Err(BackendError::Memory(
"a device buffer needs at least one element".to_string(),
));
}
let slice = self.stream.alloc_zeros::<T>(len).map_err(|e| {
memory_error(
format!(
"allocating {len} {} on device {}",
std::any::type_name::<T>(),
self.device_id
),
e,
)
})?;
Ok(DeviceBuffer { slice })
}
pub fn upload<T: DeviceElement>(&self, data: &[T]) -> Result<DeviceBuffer<T>, BackendError> {
if data.is_empty() {
return Err(BackendError::Memory(
"a device buffer needs at least one element".to_string(),
));
}
let slice = self.stream.clone_htod(data).map_err(|e| {
memory_error(
format!(
"uploading {} {} to device {}",
data.len(),
std::any::type_name::<T>(),
self.device_id
),
e,
)
})?;
Ok(DeviceBuffer { slice })
}
pub fn copy_in<T: DeviceElement>(
&self,
data: &[T],
buf: &mut DeviceBuffer<T>,
) -> Result<(), BackendError> {
self.check_owner(&buf.slice)?;
if data.len() > buf.len() {
return Err(BackendError::Memory(format!(
"{} elements do not fit a buffer of {}",
data.len(),
buf.len()
)));
}
if data.is_empty() {
return Ok(());
}
self.stream
.memcpy_htod(data, &mut buf.slice)
.map_err(|e| memory_error(format!("copying {} elements in", data.len()), e))
}
pub fn copy_out<T: DeviceElement>(
&self,
buf: &DeviceBuffer<T>,
out: &mut [T],
) -> Result<(), BackendError> {
if out.len() != buf.len() {
return Err(BackendError::Memory(format!(
"a buffer of {} elements read into a slice of {}",
buf.len(),
out.len()
)));
}
self.copy_out_range(buf, 0, out)
}
pub fn copy_out_range<T: DeviceElement>(
&self,
buf: &DeviceBuffer<T>,
offset: usize,
out: &mut [T],
) -> Result<(), BackendError> {
self.check_owner(&buf.slice)?;
let end = offset
.checked_add(out.len())
.filter(|&end| end <= buf.len())
.ok_or_else(|| {
BackendError::Memory(format!(
"{} elements from element {offset} of a buffer of {}",
out.len(),
buf.len()
))
})?;
if out.is_empty() {
return Ok(());
}
let view = buf.slice.slice(offset..end);
self.stream
.memcpy_dtoh(&view, out)
.map_err(|e| memory_error(format!("copying elements {offset}..{end} out"), e))?;
self.stream
.synchronize()
.map_err(|e| memory_error(format!("waiting for elements {offset}..{end}"), e))
}
fn check_owner<T>(&self, slice: &CudaSlice<T>) -> Result<(), BackendError> {
if Arc::ptr_eq(slice.context(), &self.context) {
Ok(())
} else {
Err(BackendError::Memory(format!(
"a buffer allocated by another CUDA backend was handed to device {}'s",
self.device_id
)))
}
}
pub fn register_ptx(&self, name: &str, ptx: &str) -> Result<KernelHandle, BackendError> {
if ptx.contains('\0') {
return Err(BackendError::KernelCompile(format!(
"the PTX for `{name}` holds a NUL byte, which ends the text the driver reads"
)));
}
let module = match self.context.load_module(cudarc::nvrtc::Ptx::from_src(ptx)) {
Ok(module) => module,
Err(e) => {
return Err(BackendError::KernelCompile(format!(
"the driver refused the PTX for `{name}`: {e:?}; {}",
jit_logs(&self.context, ptx)
)));
}
};
let function = module
.load_function(name)
.map_err(|e| BackendError::KernelCompile(format!("function lookup `{name}`: {e:?}")))?;
let handle_id = self.next_handle.fetch_add(1, Ordering::Relaxed);
let index = handle_id as usize;
if index >= MAX_KERNELS {
return Err(BackendError::KernelCompile(format!(
"this backend holds {MAX_KERNELS} kernels and `{name}` would be the {index}th"
)));
}
let entry = Box::into_raw(Box::new(LoadedKernel {
_module: module,
function,
}));
self.kernels[index].store(entry, Ordering::Release);
Ok(KernelHandle(handle_id))
}
pub fn dispatch_kernel_on_stream(
&self,
stream: &Arc<CudaStream>,
handle: KernelHandle,
count: u32,
args: &[KernelArg<'_>],
) -> Result<(), BackendError> {
let function = self.loaded(handle)?.function.clone();
if count == 0 {
return Ok(());
}
let mut i32s: Vec<i32> = Vec::new();
let mut i64s: Vec<i64> = Vec::new();
let mut u32s: Vec<u32> = Vec::new();
let mut u64s: Vec<u64> = Vec::new();
let mut f32s: Vec<f32> = Vec::new();
let mut f64s: Vec<f64> = Vec::new();
let mut staged: Vec<CudaSlice<u8>> = Vec::new();
let mut buffers: Vec<BufferRef<'_>> = Vec::new();
for arg in args {
match arg {
KernelArg::I32(v) => i32s.push(*v),
KernelArg::I64(v) => i64s.push(*v),
KernelArg::U32(v) => u32s.push(*v),
KernelArg::U64(v) => u64s.push(*v),
KernelArg::F32(v) => f32s.push(*v),
KernelArg::F64(v) => f64s.push(*v),
KernelArg::DevicePtr(p) => u64s.push(*p as u64),
KernelArg::HostSlice(bytes) => staged.push(self.stage(stream, bytes)?),
KernelArg::Buffer(buffer) => buffers.push(self.own_buffer(*buffer)?),
}
}
let block = 256u32.min(count);
let cfg = LaunchConfig {
grid_dim: (count.div_ceil(block), 1, 1),
block_dim: (block, 1, 1),
shared_mem_bytes: 0,
};
let mut builder = stream.launch_builder(&function);
let (mut ii32, mut ii64, mut iu32, mut iu64, mut if32, mut if64) =
(0usize, 0usize, 0usize, 0usize, 0usize, 0usize);
let (mut istaged, mut ibuffer) = (0usize, 0usize);
for arg in args {
match arg {
KernelArg::I32(_) => {
builder.arg(&i32s[ii32]);
ii32 += 1;
}
KernelArg::I64(_) => {
builder.arg(&i64s[ii64]);
ii64 += 1;
}
KernelArg::U32(_) => {
builder.arg(&u32s[iu32]);
iu32 += 1;
}
KernelArg::U64(_) | KernelArg::DevicePtr(_) => {
builder.arg(&u64s[iu64]);
iu64 += 1;
}
KernelArg::F32(_) => {
builder.arg(&f32s[if32]);
if32 += 1;
}
KernelArg::F64(_) => {
builder.arg(&f64s[if64]);
if64 += 1;
}
KernelArg::HostSlice(_) => {
builder.arg(&staged[istaged]);
istaged += 1;
}
KernelArg::Buffer(_) => {
buffers[ibuffer].push_to(&mut builder);
ibuffer += 1;
}
}
}
unsafe { builder.launch(cfg) }.map_err(|e| BackendError::Launch(format!("{e:?}")))?;
note_mapped(stream, &buffers)
}
fn stage(&self, stream: &Arc<CudaStream>, bytes: &[u8]) -> Result<CudaSlice<u8>, BackendError> {
if bytes.is_empty() {
return Err(BackendError::Memory(
"an empty host slice has no device copy to pass".to_string(),
));
}
stream.clone_htod(bytes).map_err(|e| {
memory_error(
format!(
"copying a {}-byte host slice to device {}",
bytes.len(),
self.device_id
),
e,
)
})
}
fn own_buffer<'a>(&self, arg: &'a dyn DeviceArg) -> Result<BufferRef<'a>, BackendError> {
let buffer = BufferRef::of(arg).ok_or(BackendError::NotSupported)?;
if Arc::ptr_eq(buffer.context(), &self.context) {
Ok(buffer)
} else {
Err(BackendError::Memory(format!(
"{arg:?} was allocated by another CUDA backend than device {}'s",
self.device_id
)))
}
}
}
impl DispatchBackend for CudaBackend {
fn id(&self) -> Backend {
Backend::Cuda {
device_id: self.device_id,
}
}
fn capabilities(&self) -> BackendCapabilities {
self.caps
}
fn dispatch_parallel_for(&self, count: u32, work: &(dyn Fn(u32) + Send + Sync)) {
if count == 0 {
return;
}
std::thread::scope(|scope| {
let threads = (count as usize).min(
std::thread::available_parallelism()
.map(std::num::NonZeroUsize::get)
.unwrap_or(1),
);
let chunks = count.div_ceil(threads as u32);
for t in 0..threads as u32 {
let lo = t.saturating_mul(chunks);
let hi = (lo + chunks).min(count);
if lo >= hi {
continue;
}
scope.spawn(move || {
for i in lo..hi {
work(i);
}
});
}
});
}
fn dispatch_one(&self, work: Box<dyn FnOnce() + Send>) {
if let NotifySendResult::Closed(_) = self.worker_tx.send(work) {
notice!(
"[flynnel::cuda] device {}: the worker has shut down, so the work item did not run",
self.device_id
);
}
}
fn register_kernel(&self, name: &str, source: &[u8]) -> Result<KernelHandle, BackendError> {
let ptx = std::str::from_utf8(source)
.map_err(|e| BackendError::KernelCompile(format!("PTX must be UTF-8: {e}")))?;
self.register_ptx(name, ptx)
}
fn dispatch_kernel(
&self,
handle: KernelHandle,
count: u32,
args: &[KernelArg<'_>],
) -> Result<(), BackendError> {
self.dispatch_kernel_on_stream(&self.stream, handle, count, args)
}
fn dispatch_kernel_sync(
&self,
handle: KernelHandle,
count: u32,
args: &[KernelArg<'_>],
) -> Result<(), BackendError> {
self.dispatch_kernel(handle, count, args)?;
self.stream
.synchronize()
.map_err(|e| BackendError::Launch(format!("stream synchronize: {e:?}")))
}
}
fn probe_capabilities() -> BackendCapabilities {
BackendCapabilities {
simt_width: 32,
max_threads_in_flight: 100_000,
launch_latency_ns: 10_000,
h2d_bw_bytes_per_sec: 25_000_000_000,
}
}
fn note_mapped(stream: &CudaStream, buffers: &[BufferRef<'_>]) -> Result<(), BackendError> {
let mut refused = Vec::new();
for buffer in buffers {
if let BufferRef::Mapped(memory) = buffer
&& let Err(e) = memory.note_launch(stream)
{
refused.push(format!("{e:?}"));
}
}
if refused.is_empty() {
return Ok(());
}
Err(BackendError::Launch(format!(
"the kernel was queued, and recording its launch on {} mapped buffer(s) failed: {}; \
each one's next host slice waits for the whole device",
refused.len(),
refused.join("; ")
)))
}
fn driver_loadable(device_id: u32) -> Result<(), BackendError> {
if !crate::backend::detect::cuda_available() {
return Err(BackendError::DeviceUnavailable(Backend::Cuda { device_id }));
}
if !unsafe { sys::is_culib_present() } {
return Err(BackendError::DeviceUnavailable(Backend::Cuda { device_id }));
}
Ok(())
}
fn map_driver_error(device_id: u32, e: DriverError) -> BackendError {
BackendError::DeviceUnavailable(Backend::Cuda { device_id }).map_io_context(format!("{e:?}"))
}
fn memory_error(what: String, e: DriverError) -> BackendError {
BackendError::Memory(format!("{what}: {e:?}"))
}
fn jit_logs(context: &CudaContext, ptx: &str) -> String {
let source = match std::ffi::CString::new(ptx) {
Ok(source) => source,
Err(nul) => return format!("the PTX holds a NUL byte at {}", nul.nul_position()),
};
if let Err(e) = context.bind_to_thread() {
return format!("no JIT log, the context would not bind: {e:?}");
}
let mut error_log = vec![0u8; JIT_LOG_BYTES];
let mut info_log = vec![0u8; JIT_LOG_BYTES];
let mut options = [
sys::CUjit_option::CU_JIT_ERROR_LOG_BUFFER,
sys::CUjit_option::CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES,
sys::CUjit_option::CU_JIT_INFO_LOG_BUFFER,
sys::CUjit_option::CU_JIT_INFO_LOG_BUFFER_SIZE_BYTES,
sys::CUjit_option::CU_JIT_LOG_VERBOSE,
];
let mut values: [*mut std::ffi::c_void; 5] = [
error_log.as_mut_ptr().cast(),
std::ptr::without_provenance_mut(JIT_LOG_BYTES),
info_log.as_mut_ptr().cast(),
std::ptr::without_provenance_mut(JIT_LOG_BYTES),
std::ptr::without_provenance_mut(1),
];
let mut module: sys::CUmodule = std::ptr::null_mut();
let loaded = unsafe {
sys::cuModuleLoadDataEx(
&mut module,
source.as_ptr().cast(),
options.len() as u32,
options.as_mut_ptr(),
values.as_mut_ptr(),
)
};
let mut text = format!(
"JIT {loaded:?}; error log: {}; information log: {}",
log_text(&error_log),
log_text(&info_log)
);
if !module.is_null() {
let unloaded = unsafe { cudarc::driver::result::module::unload(module) };
text.push_str(&format!(
"; that load succeeded, and unloading it answered {unloaded:?}"
));
}
text
}
fn log_text(buffer: &[u8]) -> String {
let end = buffer.iter().position(|&b| b == 0).unwrap_or(buffer.len());
let text = String::from_utf8_lossy(&buffer[..end]).trim().to_string();
if text.is_empty() {
"empty".to_string()
} else {
text
}
}
impl Drop for CudaBackend {
fn drop(&mut self) {
self.worker_hub.shutdown();
if let Some(handle) = self.worker_handle.take()
&& let Err(panicked) = handle.join()
{
notice!(
"flynnel: CUDA backend worker thread panicked: {}",
panic_message(&panicked)
);
}
for slot in self.kernels.iter() {
let published = slot.swap(core::ptr::null_mut(), Ordering::AcqRel);
if !published.is_null() {
drop(unsafe { Box::from_raw(published) });
}
}
}
}
fn panic_message(payload: &Box<dyn std::any::Any + Send>) -> String {
if let Some(s) = payload.downcast_ref::<&str>() {
return (*s).to_string();
}
match payload.downcast_ref::<String>() {
Some(s) => s.clone(),
None => "a payload of an unknown type".to_string(),
}
}
trait WithIoContext: Sized {
fn map_io_context(self, msg: String) -> Self;
}
impl WithIoContext for BackendError {
fn map_io_context(self, msg: String) -> Self {
match self {
BackendError::DeviceUnavailable(b) => {
notice!("[flynnel::cuda] {}: {msg}", b.name());
BackendError::DeviceUnavailable(b)
}
other => other,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::backend::detect::cuda_available;
const ADD_ONE_PTX: &str = include_str!("../../kernels/add_one.ptx");
fn backend() -> Option<CudaBackend> {
match CudaBackend::new() {
Ok(backend) => Some(backend),
Err(BackendError::DeviceUnavailable(_)) => None,
Err(other) => panic!("the backend refused for a reason other than no device: {other}"),
}
}
#[test]
fn cuda_backend_construction_matches_availability() {
let res = CudaBackend::new();
if cuda_available() {
match res {
Ok(b) => assert_eq!(b.id(), Backend::Cuda { device_id: 0 }),
Err(BackendError::DeviceUnavailable(_)) => {}
Err(e) => panic!("unexpected CUDA construction error: {e}"),
}
} else {
assert!(matches!(res, Err(BackendError::DeviceUnavailable(_))));
}
}
#[test]
fn capabilities_report_warp_width_32() {
if let Some(b) = backend() {
assert_eq!(b.capabilities().simt_width, 32);
}
}
#[test]
fn dispatch_parallel_for_invokes_each_index_on_host_fanout() {
let Some(backend) = backend() else {
return;
};
use std::sync::atomic::AtomicU32;
let counters: Vec<AtomicU32> = (0..256).map(|_| AtomicU32::new(0)).collect();
let cref = &counters;
backend.dispatch_parallel_for(256, &|i| {
cref[i as usize].fetch_add(1, std::sync::atomic::Ordering::Relaxed);
});
for c in &counters {
assert_eq!(c.load(std::sync::atomic::Ordering::Relaxed), 1);
}
}
#[test]
fn register_and_dispatch_trivial_kernel() {
let Some(backend) = backend() else {
return;
};
const TRIVIAL_PTX: &str = include_str!("../../kernels/mul3.ptx");
let handle = backend
.register_kernel("mul3", TRIVIAL_PTX.as_bytes())
.expect("registering trivial PTX kernel");
backend
.dispatch_kernel(handle, 32, &[KernelArg::U32(7)])
.expect("dispatching trivial PTX kernel");
backend
.stream()
.synchronize()
.expect("sync after trivial kernel dispatch");
}
#[test]
fn a_buffer_round_trips_through_a_kernel() {
let Some(backend) = backend() else {
return;
};
let handle = backend
.register_ptx("add_one", ADD_ONE_PTX)
.expect("add_one loads");
let data: Vec<u32> = (0..1000).map(|i| i * 3).collect();
let buf = backend.upload(&data).expect("upload");
let n = data.len() as u32;
backend
.dispatch_kernel(handle, n, &[buf.arg(), KernelArg::U32(n)])
.expect("launch");
let mut out = vec![0u32; data.len()];
backend.copy_out(&buf, &mut out).expect("copy out");
let want: Vec<u32> = data.iter().map(|v| v + 1).collect();
assert_eq!(out, want);
}
#[test]
fn a_ranged_readback_reads_only_its_range() {
let Some(backend) = backend() else {
return;
};
let data: Vec<f64> = (0..300).map(f64::from).collect();
let buf = backend.upload(&data).expect("upload");
let mut part = vec![0.0f64; 10];
backend
.copy_out_range(&buf, 100, &mut part)
.expect("ranged copy");
assert_eq!(part, data[100..110]);
let mut past = vec![0.0f64; 10];
assert!(matches!(
backend.copy_out_range(&buf, 295, &mut past),
Err(BackendError::Memory(_))
));
}
#[test]
fn a_zero_count_launches_nothing() {
let Some(backend) = backend() else {
return;
};
let handle = backend
.register_ptx("add_one", ADD_ONE_PTX)
.expect("add_one loads");
let data = vec![7u32; 64];
let buf = backend.upload(&data).expect("upload");
backend
.dispatch_kernel(handle, 0, &[buf.arg(), KernelArg::U32(64)])
.expect("a zero count is not an error");
let mut out = vec![0u32; 64];
backend.copy_out(&buf, &mut out).expect("copy out");
assert_eq!(out, data, "nothing ran, so nothing changed");
}
#[test]
fn a_host_slice_reaches_the_kernel_as_device_memory() {
let Some(backend) = backend() else {
return;
};
let handle = backend
.register_ptx("copy_add_one", ADD_ONE_PTX)
.expect("copy_add_one loads");
let src: Vec<u32> = (0..257).collect();
let bytes: Vec<u8> = src.iter().flat_map(|v| v.to_le_bytes()).collect();
let dst = backend.alloc_zeroed::<u32>(src.len()).expect("alloc");
let n = src.len() as u32;
backend
.dispatch_kernel(
handle,
n,
&[KernelArg::HostSlice(&bytes), dst.arg(), KernelArg::U32(n)],
)
.expect("launch");
let mut out = vec![0u32; src.len()];
backend.copy_out(&dst, &mut out).expect("copy out");
let want: Vec<u32> = src.iter().map(|v| v + 1).collect();
assert_eq!(out, want);
}
#[test]
fn copy_in_fills_a_prefix_and_leaves_the_rest() {
let Some(backend) = backend() else {
return;
};
let mut buf = backend.alloc_zeroed::<i64>(8).expect("alloc");
backend.copy_in(&[1, 2, 3], &mut buf).expect("copy in");
let mut out = vec![9i64; 8];
backend.copy_out(&buf, &mut out).expect("copy out");
assert_eq!(out, [1, 2, 3, 0, 0, 0, 0, 0]);
assert!(matches!(
backend.copy_in(&[0i64; 9], &mut buf),
Err(BackendError::Memory(_))
));
}
#[test]
fn bad_ptx_is_refused_with_the_jit_log() {
let Some(backend) = backend() else {
return;
};
let bad = ".version 7.0\n.target sm_70\n.address_size 64\n\
.visible .entry broken() { this is not an instruction; }\n";
match backend.register_ptx("broken", bad) {
Err(BackendError::KernelCompile(text)) => {
assert!(text.contains("error log"), "{text}");
assert!(text.contains("broken"), "{text}");
}
other => panic!("malformed PTX must be refused, got {other:?}"),
}
assert!(matches!(
backend.register_ptx("nul", "a\0b"),
Err(BackendError::KernelCompile(_))
));
}
#[test]
fn the_device_answers_its_name_and_memory() {
let Some(backend) = backend() else {
return;
};
let name = backend.device_name().expect("name");
assert!(!name.trim().is_empty());
let (free, total) = backend.mem_info().expect("mem_info");
assert!(total > 0);
assert!(free <= total, "{free} free of {total}");
eprintln!("device 0: {name}, {free} of {total} bytes free");
}
#[test]
fn another_backends_buffer_is_refused() {
let (Some(one), Some(two)) = (backend(), backend()) else {
return;
};
let handle = one
.register_ptx("add_one", ADD_ONE_PTX)
.expect("add_one loads");
let theirs = two.upload(&[1u32, 2, 3]).expect("upload");
assert!(matches!(
one.dispatch_kernel(handle, 3, &[theirs.arg(), KernelArg::U32(3)]),
Err(BackendError::Memory(_))
));
let mut out = [0u32; 3];
assert!(matches!(
one.copy_out(&theirs, &mut out),
Err(BackendError::Memory(_))
));
}
#[test]
fn a_kernel_reads_and_writes_mapped_memory_with_no_copy() {
let Some(backend) = backend() else {
return;
};
let handle = backend
.register_ptx("add_one", ADD_ONE_PTX)
.expect("add_one loads");
let mut mapped = backend.map_host::<u32>(1000).expect("map");
for (i, v) in mapped
.as_mut_slice()
.expect("host slice")
.iter_mut()
.enumerate()
{
*v = i as u32 * 3;
}
let n = mapped.len() as u32;
backend
.dispatch_kernel(handle, n, &[mapped.arg(), KernelArg::U32(n)])
.expect("launch");
let want: Vec<u32> = (0..n).map(|i| i * 3 + 1).collect();
assert_eq!(mapped.as_slice().expect("host slice"), &want[..]);
}
#[test]
fn mapped_memory_starts_zeroed_and_refuses_zero_elements() {
let Some(backend) = backend() else {
return;
};
let mut mapped = backend.map_host::<u64>(17).expect("map");
assert!(
mapped
.as_slice()
.expect("host slice")
.iter()
.all(|&v| v == 0)
);
assert!(matches!(
backend.map_host::<u8>(0),
Err(BackendError::Memory(_))
));
}
#[test]
fn a_host_slice_waits_for_the_last_launch_on_one_stream() {
let Some(backend) = backend() else {
return;
};
let handle = backend
.register_ptx("add_one", ADD_ONE_PTX)
.expect("add_one loads");
let mut mapped = backend.map_host::<u32>(4096).expect("map");
let n = mapped.len() as u32;
for _ in 0..3 {
backend
.dispatch_kernel(handle, n, &[mapped.arg(), KernelArg::U32(n)])
.expect("launch");
}
assert!(
!*mapped.memory.mixed.get_mut(),
"three launches on one stream wait on the event"
);
assert!(
mapped
.as_slice()
.expect("host slice")
.iter()
.all(|&v| v == 3)
);
assert_eq!(
*mapped.memory.stream.get_mut(),
0,
"a settle starts the account again"
);
}
#[test]
fn a_host_slice_waits_for_the_device_once_two_streams_used_the_buffer() {
let Some(backend) = backend() else {
return;
};
let handle = backend
.register_ptx("add_one", ADD_ONE_PTX)
.expect("add_one loads");
let mut mapped = backend.map_host::<u32>(4096).expect("map");
let n = mapped.len() as u32;
backend
.dispatch_kernel_on_stream(
backend.stream(),
handle,
n,
&[mapped.arg(), KernelArg::U32(n)],
)
.expect("first launch");
backend.stream().synchronize().expect("first launch done");
backend
.dispatch_kernel_on_stream(
backend.secondary_stream(),
handle,
n,
&[mapped.arg(), KernelArg::U32(n)],
)
.expect("second launch");
assert!(
*mapped.memory.mixed.get_mut(),
"a second stream marks the buffer mixed"
);
assert!(
mapped
.as_slice()
.expect("host slice")
.iter()
.all(|&v| v == 2)
);
assert!(!*mapped.memory.mixed.get_mut(), "a settle clears the mark");
}
#[test]
fn a_launch_on_the_per_thread_stream_counts_as_mixed() {
let Some(backend) = backend() else {
return;
};
let handle = backend
.register_ptx("add_one", ADD_ONE_PTX)
.expect("add_one loads");
let mut mapped = backend.map_host::<u32>(64).expect("map");
let per_thread = backend.context().per_thread_stream();
backend
.dispatch_kernel_on_stream(&per_thread, handle, 64, &[mapped.arg(), KernelArg::U32(64)])
.expect("launch");
assert!(
*mapped.memory.mixed.get_mut(),
"one handle names a stream per thread"
);
assert!(
mapped
.as_slice()
.expect("host slice")
.iter()
.all(|&v| v == 1)
);
}
#[test]
fn another_backends_mapped_buffer_is_refused() {
let (Some(one), Some(two)) = (backend(), backend()) else {
return;
};
let handle = one
.register_ptx("add_one", ADD_ONE_PTX)
.expect("add_one loads");
let theirs = two.map_host::<u32>(3).expect("map");
assert!(matches!(
one.dispatch_kernel(handle, 3, &[theirs.arg(), KernelArg::U32(3)]),
Err(BackendError::Memory(_))
));
}
#[test]
fn the_device_answers_every_fact_and_agrees_with_the_context() {
let Some(backend) = backend() else {
return;
};
let info = backend.device_info().expect("device info");
eprintln!("device 0: {info:?}");
assert_eq!(info.name, backend.device_name().expect("name"));
let (major, minor) = backend.context().compute_capability().expect("capability");
assert_eq!(
i64::from(info.capability),
i64::from(major) * 10 + i64::from(minor)
);
assert_eq!(
Some(info.multiprocessors),
crate::backend::detect::cuda_sm_count(0)
);
assert_eq!(
info.memory_bytes,
backend.mem_info().expect("mem_info").1 as u64
);
assert_eq!(info.warp_size, backend.capabilities().simt_width);
assert!(info.maps_host_memory, "map_host's tests ran on this device");
assert!(info.max_threads_per_block <= info.max_threads_per_multiprocessor);
assert!(info.clock_khz > 0 && info.memory_clock_khz > 0);
assert!(info.memory_bus_bits > 0 && info.l2_cache_bytes > 0);
assert!(info.fp64_ratio >= 1 && info.shared_memory_per_block > 0);
assert_eq!(
DeviceInfo::of_ordinal(0).expect("device 0 by ordinal"),
info
);
let count = CudaBackend::device_count().expect("count");
assert!(count >= 1);
assert!(
matches!(
DeviceInfo::of_ordinal(count),
Err(BackendError::DeviceUnavailable(_))
),
"no device past the count"
);
}
#[test]
fn zero_elements_are_refused_before_the_driver() {
let Some(backend) = backend() else {
return;
};
assert!(matches!(
backend.alloc_zeroed::<u8>(0),
Err(BackendError::Memory(_))
));
assert!(matches!(
backend.upload::<u8>(&[]),
Err(BackendError::Memory(_))
));
}
}