use std::fs::OpenOptions;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use cudarc::driver::CudaContext;
use cudarc::driver::sys as cu;
use memmap2::MmapMut;
use super::GpuPeerError;
use super::layout::{self, Geometry};
const REGISTER_DEVICEMAP: u32 = 0x02;
pub struct PeerRegion {
ctx: Arc<CudaContext>,
map: MmapMut,
dev_base: u64,
geometry: Geometry,
path: PathBuf,
remove_on_drop: bool,
registered: bool,
}
impl PeerRegion {
pub fn create(
ctx: &Arc<CudaContext>,
path: &Path,
geometry: Geometry,
remove_on_drop: bool,
) -> Result<Self, GpuPeerError> {
let bytes = geometry.region_bytes();
let file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(path)
.map_err(GpuPeerError::Io)?;
file.set_len(bytes as u64).map_err(GpuPeerError::Io)?;
let mut map = unsafe { MmapMut::map_mut(&file).map_err(GpuPeerError::Io)? };
for off in (0..bytes).step_by(4096) {
map[off] = 0;
}
let mut region = Self {
ctx: Arc::clone(ctx),
map,
dev_base: 0,
geometry,
path: path.to_path_buf(),
remove_on_drop,
registered: false,
};
region.write_header();
region.register()?;
Ok(region)
}
fn write_header(&mut self) {
self.store_u64(layout::HDR_MAGIC_OFF, layout::MAGIC);
self.store_u32(layout::HDR_VERSION_OFF, layout::VERSION);
self.store_u32(layout::HDR_LANES_OFF, self.geometry.lanes);
self.store_u32(layout::HDR_SLOT_BYTES_OFF, self.geometry.slot_bytes);
self.store_u32(layout::HDR_SLOTS_PER_OFF, self.geometry.slots_per_lane);
}
fn register(&mut self) -> Result<(), GpuPeerError> {
self.ctx
.bind_to_thread()
.map_err(|e| GpuPeerError::Driver(format!("bind_to_thread: {e:?}")))?;
let base = self.map.as_mut_ptr() as *mut core::ffi::c_void;
let bytes = self.geometry.region_bytes();
let r = unsafe { cu::cuMemHostRegister_v2(base, bytes, REGISTER_DEVICEMAP) };
if r != cu::CUresult::CUDA_SUCCESS {
return Err(GpuPeerError::Driver(format!("cuMemHostRegister_v2: {r:?}")));
}
self.registered = true;
let mut dptr: cu::CUdeviceptr = 0;
let r = unsafe { cu::cuMemHostGetDevicePointer_v2(&mut dptr, base, 0) };
if r != cu::CUresult::CUDA_SUCCESS {
return Err(GpuPeerError::Driver(format!(
"cuMemHostGetDevicePointer_v2: {r:?}"
)));
}
self.dev_base = dptr as u64;
Ok(())
}
#[inline]
pub fn dev_base(&self) -> u64 {
self.dev_base
}
#[inline]
pub fn geometry(&self) -> Geometry {
self.geometry
}
#[inline]
pub fn path(&self) -> &Path {
&self.path
}
#[inline]
pub fn context(&self) -> &Arc<CudaContext> {
&self.ctx
}
#[inline]
fn base_ptr(&self) -> *mut u8 {
self.map.as_ptr() as *mut u8
}
#[inline]
pub(crate) fn base_addr(&self) -> *mut u8 {
self.base_ptr()
}
#[inline]
pub fn load_u32(&self, off: usize) -> u32 {
debug_assert!(off + 4 <= self.geometry.region_bytes());
unsafe { (self.base_ptr().add(off) as *const u32).read_volatile() }
}
#[inline]
pub fn store_u32(&self, off: usize, v: u32) {
debug_assert!(off + 4 <= self.geometry.region_bytes());
unsafe { (self.base_ptr().add(off) as *mut u32).write_volatile(v) }
}
#[inline]
pub fn load_u64(&self, off: usize) -> u64 {
debug_assert!(off + 8 <= self.geometry.region_bytes());
unsafe { (self.base_ptr().add(off) as *const u64).read_volatile() }
}
#[inline]
pub fn store_u64(&self, off: usize, v: u64) {
debug_assert!(off + 8 <= self.geometry.region_bytes());
unsafe { (self.base_ptr().add(off) as *mut u64).write_volatile(v) }
}
#[inline]
pub fn load_i32(&self, off: usize) -> i32 {
self.load_u32(off) as i32
}
#[inline]
pub fn store_i32(&self, off: usize, v: i32) {
self.store_u32(off, v as u32);
}
#[inline]
pub fn write_bytes(&self, off: usize, src: &[u8]) {
debug_assert!(off + src.len() <= self.geometry.region_bytes());
unsafe {
core::ptr::copy_nonoverlapping(src.as_ptr(), self.base_ptr().add(off), src.len());
}
}
#[inline]
pub fn read_bytes(&self, off: usize, dst: &mut [u8]) {
debug_assert!(off + dst.len() <= self.geometry.region_bytes());
unsafe {
core::ptr::copy_nonoverlapping(self.base_ptr().add(off), dst.as_mut_ptr(), dst.len());
}
}
#[inline]
pub fn release_fence(&self) {
core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst);
}
}
impl Drop for PeerRegion {
fn drop(&mut self) {
if self.registered {
let base = self.map.as_mut_ptr() as *mut core::ffi::c_void;
let _rc: cu::CUresult = unsafe { cu::cuMemHostUnregister(base) };
self.registered = false;
}
if self.remove_on_drop {
drop(std::fs::remove_file(&self.path));
}
}
}