pub mod calibration;
pub mod group;
pub mod hybrid;
pub mod l2_persist;
pub mod lanes;
pub mod linalg;
pub mod ozaki;
pub mod layout;
pub mod region;
pub mod timed_lock;
pub mod vram;
pub mod watchdog;
pub mod wave;
mod poller;
use std::path::PathBuf;
use std::sync::atomic::{AtomicPtr, AtomicU64, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use cudarc::driver::sys as cu;
use cudarc::driver::{CudaContext, CudaFunction, CudaModule, CudaStream, LaunchConfig, PushKernelArg};
use cudarc::nvrtc::Ptx;
pub use calibration::PeerCalibration;
pub use lanes::{LaneSet, RegionWords, Ticket};
pub use layout::{
Geometry, OP_ADD1_F32, OP_ADD1_F32_V, OP_H2V, OP_NOP, OP_SUM_U32, OP_SUM_U32_V, OP_V2H,
RESIDENT_PARAMS_BYTES, STATUS_DONE, STATUS_ERR, STATUS_TEAM_INCOMPLETE, USER_OP_YIELD,
};
pub use group::{GroupHandle, PeerGroup};
pub use l2_persist::{L2BenchReport, L2Capability, L2Persist};
pub use region::PeerRegion;
pub use vram::VramPool;
pub(crate) const PEER_PTX: &str = include_str!("../../kernels/gpu_peer.ptx");
const PEER_CU: &str = include_str!("../../kernels/gpu_peer.cu");
#[derive(Debug)]
pub enum GpuPeerError {
NoDevice(String),
Driver(String),
Io(std::io::Error),
PayloadTooLarge {
len: usize,
max: usize,
},
ReapOutOfOrder {
lane: u32,
expected: u32,
got: u32,
},
Timeout,
Unavailable(&'static str),
}
impl core::fmt::Display for GpuPeerError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self {
Self::NoDevice(s) => write!(f, "no CUDA device: {s}"),
Self::Driver(s) => write!(f, "CUDA driver error: {s}"),
Self::Io(e) => write!(f, "region I/O error: {e}"),
Self::PayloadTooLarge { len, max } => {
write!(f, "payload {len} bytes exceeds slot capacity {max}")
}
Self::ReapOutOfOrder { lane, expected, got } => write!(
f,
"lane {lane} reap out of order: expected seq {expected}, got {got}"
),
Self::Timeout => write!(f, "bounded wait expired"),
Self::Unavailable(s) => write!(f, "substrate unavailable: {s}"),
}
}
}
impl std::error::Error for GpuPeerError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Self::Io(e) => Some(e),
_ => None,
}
}
}
#[cfg(feature = "persisted-calibration")]
fn calibrate_or_reuse(
region: &PeerRegion,
stream: &Arc<CudaStream>,
kernels: &calibration::CalibKernels,
ordinal: usize,
) -> Result<PeerCalibration, GpuPeerError> {
use crate::sched::calibration_store::{
ACCEL_DOORBELL_OK, ACCEL_SYS_ATOMICS_OK, ACCEL_TIMED_LOCK_OK, AccelCalibration, AccelKind,
CalibrationStore, HostStamp, StoreError, calibration_dir,
};
let Some(dir) = calibration_dir() else {
return calibration::calibrate(region, stream, kernels);
};
let stamp = HostStamp::detect();
let store = match CalibrationStore::open_or_create(&dir, &stamp) {
Ok(s) => s,
Err(e) => {
notice!(
"flynnel gpu peer: the calibration table under {} is unusable ({e:?}); \
measuring this device",
dir.display()
);
return calibration::calibrate(region, stream, kernels);
}
};
let stored = store.read().and_then(|(_cpu, accel)| {
accel
.into_iter()
.find(|a| a.ordinal as usize == ordinal && a.is_trustworthy())
});
let cal = match stored {
Some(prior) => calibration::calibrate_with_prior(
region,
stream,
kernels,
PeerCalibration {
rtt_min_ns: prior.rtt_min_ns,
rtt_median_ns: prior.rtt_median_ns,
rtt_p99_ns: prior.rtt_p99_ns,
one_way_ns: prior.one_way_ns,
clock_err_ns: prior.clock_err_ns,
delta_ns: prior.delta_ns,
launch_ns: prior.launch_ns,
doorbell_ok: true,
timed_lock_ok: false,
sys_atomics_ok: false,
lock_cpu_contended: 0,
lock_gpu_contended: 0,
},
)?,
None => calibration::calibrate(region, stream, kernels)?,
};
if !cal.doorbell_ok {
return Ok(cal);
}
let caps = device_capabilities(ordinal);
let mut flags = ACCEL_DOORBELL_OK;
if cal.timed_lock_ok {
flags |= ACCEL_TIMED_LOCK_OK;
}
if cal.sys_atomics_ok {
flags |= ACCEL_SYS_ATOMICS_OK;
}
let record = AccelCalibration::new(
AccelKind::GpuPeer,
ordinal as u32,
caps.capability,
caps.multiprocessors,
caps.clock_khz,
caps.memory_mib,
flags,
cal.rtt_min_ns,
cal.rtt_median_ns,
cal.rtt_p99_ns,
cal.one_way_ns,
cal.clock_err_ns,
cal.delta_ns,
cal.launch_ns,
);
match store.try_acquire_writer() {
Ok(writer) => {
writer.beat();
let (cpu, mut slots) = match store.read() {
Some((cpu, accel)) => (cpu, accel),
None => (Default::default(), Vec::new()),
};
match slots.iter().position(|a| a.ordinal as usize == ordinal) {
Some(i) => {
slots[i] = match slots[i].wave() {
Some(wave) => record.with_wave(wave),
None => record,
};
}
None => slots.push(record),
}
writer.publish(&cpu, &slots);
}
Err(StoreError::WriterActive) => {}
Err(e) => notice!(
"flynnel gpu peer: calibrated device {ordinal} but could not publish it ({e:?})"
),
}
Ok(cal)
}
#[cfg(not(feature = "persisted-calibration"))]
fn calibrate_or_reuse(
region: &PeerRegion,
stream: &Arc<CudaStream>,
kernels: &calibration::CalibKernels,
_ordinal: usize,
) -> Result<PeerCalibration, GpuPeerError> {
calibration::calibrate(region, stream, kernels)
}
#[cfg(feature = "persisted-calibration")]
struct DeviceCapabilities {
capability: u32,
multiprocessors: u32,
clock_khz: u32,
memory_mib: u32,
}
#[cfg(feature = "persisted-calibration")]
fn device_capabilities(ordinal: usize) -> DeviceCapabilities {
let mut dev: cu::CUdevice = 0;
let got = unsafe { cu::cuDeviceGet(&mut dev, ordinal as i32) };
if got != cu::CUresult::CUDA_SUCCESS {
return DeviceCapabilities {
capability: 0,
multiprocessors: 0,
clock_khz: 0,
memory_mib: 0,
};
}
let attr = |a: cu::CUdevice_attribute| -> u32 {
let mut v: i32 = 0;
let r = unsafe { cu::cuDeviceGetAttribute(&mut v, a, dev) };
if r == cu::CUresult::CUDA_SUCCESS && v > 0 { v as u32 } else { 0 }
};
let mut bytes: usize = 0;
let mem = unsafe { cu::cuDeviceTotalMem_v2(&mut bytes, dev) };
let memory_mib = if mem == cu::CUresult::CUDA_SUCCESS {
(bytes / (1024 * 1024)) as u32
} else {
0
};
DeviceCapabilities {
capability: attr(cu::CUdevice_attribute::CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR) * 10
+ attr(cu::CUdevice_attribute::CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR),
multiprocessors: attr(cu::CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT),
clock_khz: attr(cu::CUdevice_attribute::CU_DEVICE_ATTRIBUTE_CLOCK_RATE),
memory_mib,
}
}
pub fn device_memory_in_use() -> Option<(u64, u64)> {
let mut free: usize = 0;
let mut total: usize = 0;
let r = unsafe { cu::cuMemGetInfo_v2(&mut free, &mut total) };
if r != cu::CUresult::CUDA_SUCCESS {
return None;
}
Some((total.saturating_sub(free) as u64, total as u64))
}
#[derive(Debug, Clone)]
pub struct GpuPeerConfig {
pub region_path: Option<PathBuf>,
pub lanes: u32,
pub slot_bytes: u32,
pub slots_per_lane: u32,
pub quantum_ns: u64,
pub barrier_deadline_ns: u64,
pub idle_exit_ns: u64,
pub device_ordinal: usize,
pub vram_block_bytes: u32,
pub vram_blocks: u32,
pub blocks_per_lane: u32,
pub lane_teams: Vec<u32>,
pub user_ops_cuda: Option<String>,
pub user_ops_nvrtc_options: Vec<String>,
}
impl Default for GpuPeerConfig {
fn default() -> Self {
Self {
region_path: None,
lanes: 4,
slot_bytes: 4096,
slots_per_lane: 64,
quantum_ns: 250_000_000,
barrier_deadline_ns: 5_000_000,
idle_exit_ns: 2_000_000,
device_ordinal: 0,
vram_block_bytes: 65_536,
vram_blocks: 1024,
blocks_per_lane: 1,
lane_teams: Vec::new(),
user_ops_cuda: None,
user_ops_nvrtc_options: Vec::new(),
}
}
}
pub struct WideKernel {
_module: Arc<CudaModule>,
func: CudaFunction,
}
impl WideKernel {
pub(crate) fn new(module: Arc<CudaModule>, func: CudaFunction) -> Self {
Self { _module: module, func }
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ResidentHandle {
block: u32,
lane: u32,
bytes: u32,
}
impl ResidentHandle {
#[inline]
pub fn len(&self) -> usize {
self.bytes as usize
}
#[inline]
pub fn is_empty(&self) -> bool {
self.bytes == 0
}
#[inline]
pub fn lane(&self) -> u32 {
self.lane
}
}
pub struct GpuPeer {
region: PeerRegion,
lane_set: LaneSet,
poller: poller::Poller,
pool: Option<VramPool>,
calibration: PeerCalibration,
lane_teams: Vec<u32>,
quantum_ns: u64,
barrier_deadline_ns: u64,
device_ordinal: usize,
user_ops: bool,
wave_costs: Option<wave::WaveCosts>,
_module: Arc<CudaModule>,
_stream: Arc<CudaStream>,
wide_stream: Arc<CudaStream>,
_ctx: Arc<CudaContext>,
displaced_foreign_context: bool,
}
fn current_context() -> Option<cu::CUcontext> {
let mut ctx: cu::CUcontext = core::ptr::null_mut();
let rc: cu::CUresult = unsafe { cu::cuCtxGetCurrent(&mut ctx) };
if rc == cu::CUresult::CUDA_SUCCESS && !ctx.is_null() { Some(ctx) } else { None }
}
fn clamp_team_size(requested: u32, sm_count: Option<u32>) -> u32 {
sm_count.map_or(requested, |sm| requested.min(sm))
}
fn warn_on_foreign_context(prior: Option<cu::CUcontext>) -> bool {
let Some(prior) = prior else { return false };
match current_context() {
Some(ours) if ours != prior => {
notice!(
"flynnel gpu_peer: a different CUDA context was current on this \
thread ({prior:?}) and the peer's primary context ({ours:?}) has \
replaced it. The peer operates on the device primary context; \
work the caller enqueues after this point runs on the primary \
context, not the one it built."
);
true
}
_ => false,
}
}
struct ComposedModule {
key: (String, Vec<String>),
ptx: Ptx,
}
const COMPOSED_SLOTS: usize = 256;
const COMPOSED_MASK: usize = COMPOSED_SLOTS - 1;
static COMPOSED_PTX: [AtomicPtr<ComposedModule>; COMPOSED_SLOTS] =
[const { AtomicPtr::new(core::ptr::null_mut()) }; COMPOSED_SLOTS];
static COMPOSED_COMPILES: AtomicU64 = AtomicU64::new(0);
fn composed_key_hash(key: &(String, Vec<String>)) -> u64 {
use core::hash::{Hash, Hasher};
let mut h = std::hash::DefaultHasher::new();
key.hash(&mut h);
h.finish()
}
fn composed_lookup(key: &(String, Vec<String>)) -> Option<Ptx> {
let mut idx = (composed_key_hash(key) as usize) & COMPOSED_MASK;
for _ in 0..COMPOSED_SLOTS {
let published = COMPOSED_PTX[idx].load(Ordering::Acquire);
if published.is_null() {
return None;
}
let entry = unsafe { &*published };
if entry.key == *key {
return Some(entry.ptx.clone());
}
idx = (idx + 1) & COMPOSED_MASK;
}
None
}
fn composed_ptx(src: String, options: &[String]) -> Result<Ptx, GpuPeerError> {
let key = (src, options.to_vec());
if let Some(ptx) = composed_lookup(&key) {
return Ok(ptx);
}
let opts = cudarc::nvrtc::CompileOptions { options: key.1.clone(), ..Default::default() };
let ptx = cudarc::nvrtc::compile_ptx_with_opts(&key.0, opts)
.map_err(|e| GpuPeerError::Driver(format!("user-ops NVRTC compile: {e:?}")))?;
COMPOSED_COMPILES.fetch_add(1, Ordering::Relaxed);
Ok(composed_publish(key, ptx))
}
fn composed_publish(key: (String, Vec<String>), ptx: Ptx) -> Ptx {
let hash = composed_key_hash(&key);
let mut prepared: Option<&'static ComposedModule> = None;
let mut idx = (hash as usize) & COMPOSED_MASK;
for _ in 0..COMPOSED_SLOTS {
let published = COMPOSED_PTX[idx].load(Ordering::Acquire);
if published.is_null() {
let entry = *prepared.get_or_insert_with(|| {
&*Box::leak(Box::new(ComposedModule { key: key.clone(), ptx: ptx.clone() }))
});
let fresh = entry as *const ComposedModule as *mut ComposedModule;
match COMPOSED_PTX[idx].compare_exchange(
core::ptr::null_mut(),
fresh,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => return entry.ptx.clone(),
Err(taken) => {
let other = unsafe { &*taken };
if other.key == key {
return other.ptx.clone();
}
}
}
} else {
let other = unsafe { &*published };
if other.key == key {
return other.ptx.clone();
}
}
idx = (idx + 1) & COMPOSED_MASK;
}
ptx
}
impl GpuPeer {
pub fn init(config: GpuPeerConfig) -> Result<Self, GpuPeerError> {
if !crate::backend::detect::cuda_available() {
return Err(GpuPeerError::NoDevice(
"the CUDA driver is not loadable on this host".to_string(),
));
}
if !unsafe { cudarc::driver::sys::is_culib_present() } {
return Err(GpuPeerError::NoDevice(
"the CUDA driver is not loadable on this host".to_string(),
));
}
let prior = current_context();
let ctx = CudaContext::new(config.device_ordinal)
.map_err(|e| GpuPeerError::NoDevice(format!("{e:?}")))?;
let displaced = warn_on_foreign_context(prior);
let stream = ctx.default_stream();
let module = match &config.user_ops_cuda {
None => match ctx.load_module(Ptx::from_src(PEER_PTX)) {
Ok(m) => m,
Err(ptx_err) => {
notice!(
"flynnel gpu_peer: checked-in PTX rejected ({ptx_err:?}); \
compiling the peer kernels with NVRTC instead"
);
let ptx = cudarc::nvrtc::compile_ptx(PEER_CU).map_err(|e| {
GpuPeerError::Driver(format!(
"PTX load: {ptx_err:?}; NVRTC fallback compile: {e:?}"
))
})?;
ctx.load_module(ptx).map_err(|e| {
GpuPeerError::Driver(format!("NVRTC-fallback PTX load: {e:?}"))
})?
}
},
Some(user_src) => {
let src = format!(
"#define FLYNNEL_USER_OPS 1\n{PEER_CU}\n{}\n{user_src}\n",
wave::WAVE_CU
);
let ptx = composed_ptx(src, &config.user_ops_nvrtc_options)?;
ctx.load_module(ptx)
.map_err(|e| GpuPeerError::Driver(format!("user-ops PTX load: {e:?}")))?
}
};
let load = |name: &str| {
module
.load_function(name)
.map_err(|e| GpuPeerError::Driver(format!("kernel `{name}`: {e:?}")))
};
let f_poller = load("flynnel_peer_poller")?;
let kernels = calibration::CalibKernels {
calib_pong: load("flynnel_peer_calib_pong")?,
fischer: load("flynnel_peer_fischer")?,
cas_probe: load("flynnel_peer_cas_probe")?,
};
if config.lanes as usize > layout::MAX_POLLER_LANES {
return Err(GpuPeerError::Unavailable(
"lane count exceeds the per-lane poller words in the region header",
));
}
let geometry = Geometry {
lanes: config.lanes.max(1),
slot_bytes: config.slot_bytes.max(64),
slots_per_lane: config.slots_per_lane.max(2),
};
let (path, remove_on_drop) = match &config.region_path {
Some(p) => (p.clone(), false),
None => {
static INSTANCE: core::sync::atomic::AtomicU32 =
core::sync::atomic::AtomicU32::new(0);
let n = INSTANCE.fetch_add(1, core::sync::atomic::Ordering::Relaxed);
(
std::env::temp_dir().join(format!(
"flynnel_gpu_peer_{}_{n}.bin",
std::process::id()
)),
true,
)
}
};
let region = PeerRegion::create(&ctx, &path, geometry, remove_on_drop)?;
let calibration =
calibrate_or_reuse(®ion, &stream, &kernels, config.device_ordinal)?;
if !calibration.doorbell_ok {
return Err(GpuPeerError::Unavailable(
"doorbell handshake failed during calibration",
));
}
let pool = if config.vram_blocks > 0 {
Some(VramPool::new(
&ctx,
&stream,
config.vram_block_bytes.max(64),
config.vram_blocks,
)?)
} else {
None
};
let (vbase, vbytes, vblocks) = match &pool {
Some(p) => (p.base(), p.block_bytes(), p.blocks()),
None => (0, 0, 0),
};
let lane_set = LaneSet::new(geometry);
let mut lane_streams = Vec::with_capacity(geometry.lanes as usize);
for lane in 0..geometry.lanes {
lane_streams.push(
ctx.new_stream()
.map_err(|e| GpuPeerError::Driver(format!("lane {lane} stream: {e:?}")))?,
);
}
let sm_count = crate::backend::detect::cuda_sm_count(config.device_ordinal);
let lane_count = geometry.lanes.max(1) as usize;
let requested: Vec<u32> = if config.lane_teams.is_empty() {
vec![config.blocks_per_lane.max(1); lane_count]
} else if config.lane_teams.len() == lane_count {
config.lane_teams.iter().map(|t| (*t).max(1)).collect()
} else {
return Err(GpuPeerError::Unavailable(
"lane_teams must name a team for every lane, or be empty",
));
};
let lane_teams: Vec<u32> =
requested.iter().map(|&asked| clamp_team_size(asked, sm_count)).collect();
for (lane, (&asked, &got)) in requested.iter().zip(&lane_teams).enumerate() {
if got != asked {
notice!(
"flynnel gpu_peer: lane {lane} asked for {asked} blocks, which exceeds \
device {}'s {got} streaming multiprocessors; it runs a team of {got}",
config.device_ordinal
);
}
}
let lane0_team = *lane_teams.first().expect("a region has at least one lane");
let poller = poller::Poller::new(
lane_streams,
f_poller,
config.quantum_ns,
config.idle_exit_ns,
vbase,
vbytes,
vblocks,
lane_teams.clone(),
config.barrier_deadline_ns,
);
let wide_stream = ctx
.new_stream()
.map_err(|e| GpuPeerError::Driver(format!("wide stream: {e:?}")))?;
Ok(Self {
region,
lane_set,
poller,
pool,
calibration,
lane_teams,
quantum_ns: config.quantum_ns,
barrier_deadline_ns: config.barrier_deadline_ns.max(1),
device_ordinal: config.device_ordinal,
user_ops: config.user_ops_cuda.is_some(),
wave_costs: wave::stored_wave_costs(config.device_ordinal, lane0_team),
_module: module,
_stream: stream,
wide_stream,
_ctx: ctx,
displaced_foreign_context: displaced,
})
}
pub fn displaced_foreign_context(&self) -> bool {
self.displaced_foreign_context
}
#[inline]
pub fn calibration(&self) -> PeerCalibration {
self.calibration
}
#[inline]
pub fn geometry(&self) -> Geometry {
self.region.geometry()
}
#[inline]
pub fn team_size(&self) -> u32 {
*self.lane_teams.first().expect("a region has at least one lane")
}
#[inline]
pub fn lane_team_size(&self, lane: u32) -> Option<u32> {
self.lane_teams.get(lane as usize).copied()
}
pub(crate) fn lanes_share_a_team(&self) -> bool {
self.lane_teams.windows(2).all(|pair| pair[0] == pair[1])
}
pub fn user_ops_compiles() -> u64 {
COMPOSED_COMPILES.load(Ordering::Relaxed)
}
#[inline]
pub fn region(&self) -> &PeerRegion {
&self.region
}
pub fn submit(&mut self, op: u32, payload: &[u8]) -> Result<Ticket, GpuPeerError> {
let t0 = Instant::now();
loop {
if let Some(t) = self.lane_set.try_submit(&self.region, op, payload)? {
self.poller.ensure_running(&self.region, t.lane)?;
return Ok(t);
}
self.poller.ensure_running_all(&self.region)?;
if t0.elapsed() > Duration::from_secs(10) {
return Err(GpuPeerError::Timeout);
}
std::thread::yield_now();
}
}
#[inline]
pub fn is_done(&self, ticket: Ticket) -> bool {
self.lane_set.is_done(&self.region, ticket)
}
pub fn wait(&mut self, ticket: Ticket, timeout: Duration) -> Result<u32, GpuPeerError> {
let status = self.wait_status(ticket, timeout)?;
if status != STATUS_DONE {
return Err(GpuPeerError::Unavailable("slot completed with a failed status"));
}
Ok(status)
}
pub fn wait_status(&mut self, ticket: Ticket, timeout: Duration) -> Result<u32, GpuPeerError> {
let t0 = Instant::now();
let mut spins = 0u32;
while !self.lane_set.is_done(&self.region, ticket) {
spins = spins.wrapping_add(1);
if spins.is_multiple_of(4096) {
self.poller.ensure_running(&self.region, ticket.lane)?;
if t0.elapsed() > timeout {
return Err(GpuPeerError::Timeout);
}
}
core::hint::spin_loop();
}
Ok(self.lane_set.status(&self.region, ticket))
}
pub fn read_result(&self, ticket: Ticket, dst: &mut [u8]) -> Result<(), GpuPeerError> {
self.lane_set.read_result(&self.region, ticket, dst)
}
pub fn reap(&mut self, ticket: Ticket) -> Result<(), GpuPeerError> {
self.lane_set.reap(ticket)
}
pub fn barrier_stalls(&self) -> (u32, u32) {
(
self.region.load_u32(layout::HDR_STALL_COUNT_OFF),
self.region.load_u32(layout::HDR_STALL_MAX_DEPTH_OFF),
)
}
pub fn barrier_wait_max_ns(&self) -> u32 {
self.region.load_u32(layout::HDR_BARRIER_WAIT_MAX_OFF)
}
fn submit_on_lane(
&mut self,
lane: u32,
op: u32,
payload: &[u8],
) -> Result<Ticket, GpuPeerError> {
let t0 = Instant::now();
loop {
if let Some(t) = self.lane_set.try_submit_on(&self.region, lane, op, payload)? {
self.poller.ensure_running(&self.region, t.lane)?;
return Ok(t);
}
self.poller.ensure_running(&self.region, lane)?;
if t0.elapsed() > Duration::from_secs(10) {
return Err(GpuPeerError::Timeout);
}
std::thread::yield_now();
}
}
pub fn pin(&mut self, data: &[u8]) -> Result<ResidentHandle, GpuPeerError> {
let pool = self
.pool
.as_mut()
.ok_or(GpuPeerError::Unavailable("resident pool disabled"))?;
let block_bytes = pool.block_bytes() as usize;
if data.len() > block_bytes
|| data.len() + RESIDENT_PARAMS_BYTES > self.region.geometry().payload_max()
{
return Err(GpuPeerError::PayloadTooLarge {
len: data.len(),
max: block_bytes.min(
self.region.geometry().payload_max() - RESIDENT_PARAMS_BYTES,
),
});
}
let block = pool
.alloc()
.ok_or(GpuPeerError::Unavailable("resident pool exhausted"))?;
let lane = block % self.region.geometry().lanes;
let mut payload = Vec::with_capacity(RESIDENT_PARAMS_BYTES + data.len());
payload.extend_from_slice(&block.to_le_bytes());
payload.extend_from_slice(&(data.len() as u32).to_le_bytes());
payload.extend_from_slice(data);
let t = self.submit_on_lane(lane, OP_H2V, &payload)?;
let status = self.wait(t, Duration::from_secs(10))?;
self.reap(t)?;
if status != STATUS_DONE {
if let Some(p) = self.pool.as_mut() {
p.release(block);
}
return Err(GpuPeerError::Unavailable("resident upload rejected"));
}
Ok(ResidentHandle { block, lane, bytes: data.len() as u32 })
}
pub fn write_resident(
&mut self,
handle: &ResidentHandle,
data: &[u8],
) -> Result<(), GpuPeerError> {
if data.len() > handle.len() {
return Err(GpuPeerError::PayloadTooLarge { len: data.len(), max: handle.len() });
}
let mut payload = Vec::with_capacity(RESIDENT_PARAMS_BYTES + data.len());
payload.extend_from_slice(&handle.block.to_le_bytes());
payload.extend_from_slice(&(data.len() as u32).to_le_bytes());
payload.extend_from_slice(data);
let t = self.submit_on_lane(handle.lane, layout::OP_H2V, &payload)?;
let status = self.wait(t, Duration::from_secs(10))?;
self.reap(t)?;
if status != STATUS_DONE {
return Err(GpuPeerError::Unavailable("resident re-upload rejected"));
}
Ok(())
}
pub fn pin_bulk(&mut self, data: &[u8]) -> Result<ResidentHandle, GpuPeerError> {
self.pin_bulk_to(data, None)
}
pub fn pin_bulk_on_lane(
&mut self,
data: &[u8],
lane: u32,
) -> Result<ResidentHandle, GpuPeerError> {
self.pin_bulk_to(data, Some(lane))
}
fn pin_bulk_to(
&mut self,
data: &[u8],
lane: Option<u32>,
) -> Result<ResidentHandle, GpuPeerError> {
let pool = self
.pool
.as_mut()
.ok_or(GpuPeerError::Unavailable("resident pool disabled"))?;
let block_bytes = pool.block_bytes() as usize;
let need = data.len().div_ceil(block_bytes.max(1)).max(1);
if need > pool.free_blocks() {
return Err(GpuPeerError::Unavailable("resident pool exhausted"));
}
let first = pool
.alloc_span(need as u32)
.ok_or(GpuPeerError::Unavailable("no contiguous resident span"))?;
let dst = pool.block_ptr(first);
unsafe {
cudarc::driver::result::memcpy_htod_async(
dst,
data,
self.wide_stream.cu_stream() as _,
)
.map_err(|e| GpuPeerError::Driver(format!("pin_bulk htod: {e:?}")))?;
}
self.wide_stream
.synchronize()
.map_err(|e| GpuPeerError::Driver(format!("pin_bulk sync: {e:?}")))?;
let lanes = self.region.geometry().lanes.max(1);
Ok(ResidentHandle {
block: first,
lane: lane.map_or(first % lanes, |chosen| chosen % lanes),
bytes: data.len() as u32,
})
}
pub fn write_resident_bulk(
&mut self,
handle: &ResidentHandle,
data: &[u8],
) -> Result<(), GpuPeerError> {
let (dst, _) = self.resident_ptr(handle)?;
unsafe {
cudarc::driver::result::memcpy_htod_async(
dst,
data,
self.wide_stream.cu_stream() as _,
)
.map_err(|e| GpuPeerError::Driver(format!("write_resident_bulk: {e:?}")))?;
}
self.wide_stream
.synchronize()
.map_err(|e| GpuPeerError::Driver(format!("write_resident_bulk sync: {e:?}")))
}
pub fn fetch_bulk(
&mut self,
handle: &ResidentHandle,
out: &mut [u8],
) -> Result<(), GpuPeerError> {
let (src, _) = self.resident_ptr(handle)?;
unsafe {
cudarc::driver::result::memcpy_dtoh_async(
out,
src,
self.wide_stream.cu_stream() as _,
)
.map_err(|e| GpuPeerError::Driver(format!("fetch_bulk: {e:?}")))?;
}
self.wide_stream
.synchronize()
.map_err(|e| GpuPeerError::Driver(format!("fetch_bulk sync: {e:?}")))
}
pub fn fetch_bulk_at(
&mut self,
handle: &ResidentHandle,
offset: usize,
out: &mut [u8],
) -> Result<(), GpuPeerError> {
let (base, len) = self.resident_ptr(handle)?;
if out.len() > len || offset > len - out.len() {
return Err(GpuPeerError::Unavailable("the read runs past the resident span"));
}
unsafe {
cudarc::driver::result::memcpy_dtoh_async(
out,
base + offset as u64,
self.wide_stream.cu_stream() as _,
)
.map_err(|e| GpuPeerError::Driver(format!("fetch_bulk_at: {e:?}")))?;
}
self.wide_stream
.synchronize()
.map_err(|e| GpuPeerError::Driver(format!("fetch_bulk_at sync: {e:?}")))
}
pub fn write_resident_bulk_at(
&mut self,
handle: &ResidentHandle,
offset: usize,
data: &[u8],
) -> Result<(), GpuPeerError> {
let (base, len) = self.resident_ptr(handle)?;
if data.len() > len || offset > len - data.len() {
return Err(GpuPeerError::Unavailable("the write runs past the resident span"));
}
unsafe {
cudarc::driver::result::memcpy_htod_async(
base + offset as u64,
data,
self.wide_stream.cu_stream() as _,
)
.map_err(|e| GpuPeerError::Driver(format!("write_resident_bulk_at: {e:?}")))?;
}
self.wide_stream
.synchronize()
.map_err(|e| GpuPeerError::Driver(format!("write_resident_bulk_at sync: {e:?}")))
}
pub fn pin_prefetch(
&mut self,
data: &[u8],
) -> Result<(ResidentHandle, Ticket), GpuPeerError> {
let pool = self
.pool
.as_mut()
.ok_or(GpuPeerError::Unavailable("resident pool disabled"))?;
let block_bytes = pool.block_bytes() as usize;
if data.len() > block_bytes
|| data.len() + RESIDENT_PARAMS_BYTES > self.region.geometry().payload_max()
{
return Err(GpuPeerError::PayloadTooLarge {
len: data.len(),
max: block_bytes
.min(self.region.geometry().payload_max() - RESIDENT_PARAMS_BYTES),
});
}
let block = pool
.alloc()
.ok_or(GpuPeerError::Unavailable("resident pool exhausted"))?;
let lane = block % self.region.geometry().lanes;
let mut payload = Vec::with_capacity(RESIDENT_PARAMS_BYTES + data.len());
payload.extend_from_slice(&block.to_le_bytes());
payload.extend_from_slice(&(data.len() as u32).to_le_bytes());
payload.extend_from_slice(data);
let t = self.submit_on_lane(lane, layout::OP_H2V, &payload)?;
Ok((ResidentHandle { block, lane, bytes: data.len() as u32 }, t))
}
pub fn submit_user(
&mut self,
op: u32,
handle: Option<&ResidentHandle>,
args: &[u8],
) -> Result<Ticket, GpuPeerError> {
if op < layout::OP_USER_BASE {
return Err(GpuPeerError::Unavailable("op below OP_USER_BASE"));
}
let (block, count, lane) = match handle {
Some(h) => (h.block, h.bytes, Some(h.lane)),
None => (layout::NO_BLOCK, 0u32, None),
};
let mut payload = Vec::with_capacity(RESIDENT_PARAMS_BYTES + args.len());
payload.extend_from_slice(&block.to_le_bytes());
payload.extend_from_slice(&count.to_le_bytes());
payload.extend_from_slice(args);
match lane {
Some(l) => self.submit_on_lane(l, op, &payload),
None => self.submit(op, &payload),
}
}
pub fn submit_user_on_lane(
&mut self,
op: u32,
handle: Option<&ResidentHandle>,
args: &[u8],
lane: u32,
) -> Result<Ticket, GpuPeerError> {
if op < layout::OP_USER_BASE {
return Err(GpuPeerError::Unavailable("op below OP_USER_BASE"));
}
let (block, count) = match handle {
Some(h) => (h.block, h.bytes),
None => (layout::NO_BLOCK, 0u32),
};
let mut payload = Vec::with_capacity(RESIDENT_PARAMS_BYTES + args.len());
payload.extend_from_slice(&block.to_le_bytes());
payload.extend_from_slice(&count.to_le_bytes());
payload.extend_from_slice(args);
let lanes = self.region.geometry().lanes.max(1);
self.submit_on_lane(lane % lanes, op, &payload)
}
pub fn submit_resident(
&mut self,
op: u32,
handle: &ResidentHandle,
) -> Result<Ticket, GpuPeerError> {
let mut params = [0u8; RESIDENT_PARAMS_BYTES];
params[..4].copy_from_slice(&handle.block.to_le_bytes());
params[4..].copy_from_slice(&handle.bytes.to_le_bytes());
self.submit_on_lane(handle.lane, op, ¶ms)
}
pub fn fetch(
&mut self,
handle: &ResidentHandle,
out: &mut [u8],
) -> Result<(), GpuPeerError> {
let mut params = [0u8; RESIDENT_PARAMS_BYTES];
params[..4].copy_from_slice(&handle.block.to_le_bytes());
params[4..].copy_from_slice(&handle.bytes.to_le_bytes());
let t = self.submit_on_lane(handle.lane, OP_V2H, ¶ms)?;
let status = self.wait(t, Duration::from_secs(10))?;
if status != STATUS_DONE {
self.reap(t)?;
return Err(GpuPeerError::Unavailable("resident download rejected"));
}
let n = out.len().min(handle.len());
let mut buf = vec![0u8; RESIDENT_PARAMS_BYTES + n];
let read = self.read_result(t, &mut buf);
self.reap(t)?;
read?;
out[..n].copy_from_slice(&buf[RESIDENT_PARAMS_BYTES..]);
Ok(())
}
pub fn unpin(&mut self, handle: ResidentHandle) -> Result<(), GpuPeerError> {
let pool = self
.pool
.as_mut()
.ok_or(GpuPeerError::Unavailable("resident pool disabled"))?;
let span = (handle.bytes as usize)
.div_ceil(pool.block_bytes().max(1) as usize)
.max(1) as u32;
for b in handle.block..handle.block + span {
pool.release(b);
}
Ok(())
}
pub fn pool_stats(&self) -> (usize, u32) {
match &self.pool {
Some(p) => (p.free_blocks(), p.blocks()),
None => (0, 0),
}
}
pub fn resident_ptr(&self, handle: &ResidentHandle) -> Result<(u64, usize), GpuPeerError> {
let pool = self
.pool
.as_ref()
.ok_or(GpuPeerError::Unavailable("resident pool disabled"))?;
Ok((pool.block_ptr(handle.block), handle.len()))
}
pub fn compile_wide_kernel(&self, src: &str, entry: &str) -> Result<WideKernel, GpuPeerError> {
let ptx = cudarc::nvrtc::compile_ptx(src)
.map_err(|e| GpuPeerError::Driver(format!("wide-kernel NVRTC compile: {e:?}")))?;
self.load_wide_kernel(ptx, entry)
}
pub fn load_wide_kernel_ptx(&self, ptx: &str, entry: &str) -> Result<WideKernel, GpuPeerError> {
self.load_wide_kernel(Ptx::from_src(ptx), entry)
}
fn load_wide_kernel(&self, ptx: Ptx, entry: &str) -> Result<WideKernel, GpuPeerError> {
let module = self
._ctx
.load_module(ptx)
.map_err(|e| GpuPeerError::Driver(format!("wide-kernel PTX load: {e:?}")))?;
let func = module
.load_function(entry)
.map_err(|e| GpuPeerError::Driver(format!("wide-kernel entry `{entry}`: {e:?}")))?;
Ok(WideKernel { _module: module, func })
}
pub fn context(&self) -> &Arc<CudaContext> {
&self._ctx
}
pub fn wide_stream(&self) -> &Arc<CudaStream> {
&self.wide_stream
}
fn launch_wide_inner(
&self,
kernel: &WideKernel,
grid_blocks: u32,
block_threads: u32,
ptrs: &[u64],
scalars: &[u32],
) -> Result<(), GpuPeerError> {
let block = block_threads.clamp(1, 1024);
let grid = if grid_blocks == 0 {
scalars.first().copied().unwrap_or(1).div_ceil(block).max(1)
} else {
grid_blocks
};
let mut b = self.wide_stream.launch_builder(&kernel.func);
for p in ptrs {
b.arg(p);
}
for s in scalars {
b.arg(s);
}
unsafe {
b.launch(LaunchConfig {
grid_dim: (grid, 1, 1),
block_dim: (block, 1, 1),
shared_mem_bytes: 0,
})
}
.map_err(|e| GpuPeerError::Driver(format!("wide launch: {e:?}")))?;
Ok(())
}
pub fn launch_wide(
&self,
kernel: &WideKernel,
grid_blocks: u32,
block_threads: u32,
ptrs: &[u64],
scalars: &[u32],
) -> Result<(), GpuPeerError> {
self.launch_wide_inner(kernel, grid_blocks, block_threads, ptrs, scalars)?;
self.sync_wide()
}
pub fn launch_wide_async(
&self,
kernel: &WideKernel,
grid_blocks: u32,
block_threads: u32,
ptrs: &[u64],
scalars: &[u32],
) -> Result<(), GpuPeerError> {
self.launch_wide_inner(kernel, grid_blocks, block_threads, ptrs, scalars)
}
pub fn sync_wide(&self) -> Result<(), GpuPeerError> {
self.wide_stream
.synchronize()
.map_err(|e| GpuPeerError::Driver(format!("wide sync: {e:?}")))
}
pub fn pause_poller(&mut self) -> Result<(), GpuPeerError> {
self.poller.pause(&self.region)
}
pub fn resume_poller(&mut self) {
self.poller.resume();
}
#[inline]
pub fn poller_paused(&self) -> bool {
self.poller.is_paused()
}
pub fn l2_capability(&self) -> Result<L2Capability, GpuPeerError> {
L2Capability::query(self._ctx_ref())
}
pub fn l2_benchmark(
&self,
hot_bytes: usize,
pol_bytes: usize,
iters: u32,
runs: u32,
) -> Result<L2BenchReport, GpuPeerError> {
l2_persist::benchmark(self._ctx_ref(), hot_bytes, pol_bytes, iters, runs)
}
#[inline]
fn _ctx_ref(&self) -> &Arc<CudaContext> {
&self._ctx
}
pub fn timed_lock_acquire(&self, timeout: Duration) -> Result<(), GpuPeerError> {
if !self.calibration.timed_lock_ok {
return Err(GpuPeerError::Unavailable("timed lock not validated on this host"));
}
if timed_lock::acquire(
&self.region,
Duration::from_nanos(self.calibration.delta_ns),
timeout,
) {
Ok(())
} else {
Err(GpuPeerError::Timeout)
}
}
pub fn timed_lock_release(&self) {
timed_lock::release(&self.region);
}
}
impl Drop for GpuPeer {
fn drop(&mut self) {
self.poller.shutdown(&self.region);
}
}
#[cfg(test)]
mod tests {
use super::*;
fn device_present(test_name: &str) -> bool {
if crate::backend::detect::cuda_available() {
return true;
}
println!("{test_name}: no CUDA driver on this host, so the test body did not run");
false
}
#[test]
fn init_reports_displacing_a_foreign_context() {
if !device_present("init_reports_displacing_a_foreign_context") {
return;
}
let foreign = CudaContext::new_non_primary(0, 0)
.expect("a CUDA device is required for this test");
foreign.bind_to_thread().expect("make the foreign context current");
let before = current_context().expect("the foreign context is current");
let peer = GpuPeer::init(GpuPeerConfig::default()).expect("peer init");
assert!(
peer.displaced_foreign_context(),
"init replaced a context the caller had current and must say so"
);
let after = current_context().expect("a context is current after init");
assert_ne!(
after, before,
"the report is only meaningful if the context actually changed"
);
}
#[test]
fn a_pointer_from_a_displaced_context_is_refused_not_misread() {
if !device_present("a_pointer_from_a_displaced_context_is_refused_not_misread") {
return;
}
const N: usize = 256;
const PATTERN: u8 = 0xAB;
let foreign = CudaContext::new_non_primary(0, 0)
.expect("a CUDA device is required for this test");
foreign.bind_to_thread().expect("make the foreign context current");
let mut dptr: cu::CUdeviceptr = 0;
let alloc = unsafe { cu::cuMemAlloc_v2(&mut dptr, N) };
assert_eq!(alloc, cu::CUresult::CUDA_SUCCESS, "allocate under the foreign context");
let set = unsafe { cu::cuMemsetD8_v2(dptr, PATTERN, N) };
assert_eq!(set, cu::CUresult::CUDA_SUCCESS);
let sync = unsafe { cu::cuCtxSynchronize() };
assert_eq!(sync, cu::CUresult::CUDA_SUCCESS);
let peer = GpuPeer::init(GpuPeerConfig::default()).expect("peer init");
assert!(
peer.displaced_foreign_context(),
"the peer must have replaced the context this pointer belongs to, \
or the test is not exercising a displacement at all"
);
let mut back = [0u8; N];
let copy = unsafe {
cu::cuMemcpyDtoH_v2(back.as_mut_ptr().cast::<core::ffi::c_void>(), dptr, N)
};
if copy == cu::CUresult::CUDA_SUCCESS {
let intact = back.iter().all(|&b| b == PATTERN);
println!(
"displaced-context pointer: driver ACCEPTED the read, bytes \
{} what was written",
if intact { "match" } else { "DO NOT match" }
);
assert!(
intact,
"the driver accepted a pointer from the displaced context and \
returned bytes that are not the ones written to it. That is a \
silent cross-context read, and the displacement report is then \
the only warning a consumer will ever get"
);
} else {
println!("displaced-context pointer: driver REFUSED the read with {copy:?}");
}
let _free: cu::CUresult = unsafe { cu::cuMemFree_v2(dptr) };
}
#[test]
fn init_reports_nothing_when_it_displaces_nothing() {
if !device_present("init_reports_nothing_when_it_displaces_nothing") {
return;
}
let peer = GpuPeer::init(GpuPeerConfig::default())
.expect("a CUDA device is required for this test");
assert!(
!peer.displaced_foreign_context(),
"the peer's own primary context is not a foreign one"
);
}
}