pub mod calibration;
pub mod group;
pub mod hybrid;
pub mod l2_persist;
pub mod lanes;
pub mod linalg;
pub mod ozaki;
pub mod layout;
pub mod region;
pub mod timed_lock;
pub mod vram;
mod poller;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant};
use cudarc::driver::{CudaContext, CudaFunction, CudaModule, CudaStream, LaunchConfig, PushKernelArg};
use cudarc::nvrtc::Ptx;
pub use calibration::PeerCalibration;
pub use lanes::{LaneSet, RegionWords, Ticket};
pub use layout::{
Geometry, OP_ADD1_F32, OP_ADD1_F32_V, OP_H2V, OP_NOP, OP_SUM_U32, OP_SUM_U32_V, OP_V2H,
RESIDENT_PARAMS_BYTES, STATUS_DONE, STATUS_ERR,
};
pub use group::{GroupHandle, PeerGroup};
pub use l2_persist::{L2BenchReport, L2Capability, L2Persist};
pub use region::PeerRegion;
pub use vram::VramPool;
pub(crate) const PEER_PTX: &str = include_str!("../../kernels/gpu_peer.ptx");
const PEER_CU: &str = include_str!("../../kernels/gpu_peer.cu");
#[derive(Debug)]
pub enum GpuPeerError {
NoDevice(String),
Driver(String),
Io(std::io::Error),
PayloadTooLarge {
len: usize,
max: usize,
},
ReapOutOfOrder {
lane: u32,
expected: u32,
got: u32,
},
Timeout,
Unavailable(&'static str),
}
impl core::fmt::Display for GpuPeerError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self {
Self::NoDevice(s) => write!(f, "no CUDA device: {s}"),
Self::Driver(s) => write!(f, "CUDA driver error: {s}"),
Self::Io(e) => write!(f, "region I/O error: {e}"),
Self::PayloadTooLarge { len, max } => {
write!(f, "payload {len} bytes exceeds slot capacity {max}")
}
Self::ReapOutOfOrder { lane, expected, got } => write!(
f,
"lane {lane} reap out of order: expected seq {expected}, got {got}"
),
Self::Timeout => write!(f, "bounded wait expired"),
Self::Unavailable(s) => write!(f, "substrate unavailable: {s}"),
}
}
}
impl std::error::Error for GpuPeerError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Self::Io(e) => Some(e),
_ => None,
}
}
}
#[derive(Debug, Clone)]
pub struct GpuPeerConfig {
pub region_path: Option<PathBuf>,
pub lanes: u32,
pub slot_bytes: u32,
pub slots_per_lane: u32,
pub quantum_ns: u64,
pub idle_exit_ns: u64,
pub device_ordinal: usize,
pub vram_block_bytes: u32,
pub vram_blocks: u32,
pub blocks_per_lane: u32,
pub user_ops_cuda: Option<String>,
}
impl Default for GpuPeerConfig {
fn default() -> Self {
Self {
region_path: None,
lanes: 4,
slot_bytes: 4096,
slots_per_lane: 64,
quantum_ns: 250_000_000,
idle_exit_ns: 2_000_000,
device_ordinal: 0,
vram_block_bytes: 65_536,
vram_blocks: 1024,
blocks_per_lane: 1,
user_ops_cuda: None,
}
}
}
pub struct WideKernel {
_module: Arc<CudaModule>,
func: CudaFunction,
}
impl WideKernel {
pub(crate) fn new(module: Arc<CudaModule>, func: CudaFunction) -> Self {
Self { _module: module, func }
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ResidentHandle {
block: u32,
lane: u32,
bytes: u32,
}
impl ResidentHandle {
#[inline]
pub fn len(&self) -> usize {
self.bytes as usize
}
#[inline]
pub fn is_empty(&self) -> bool {
self.bytes == 0
}
#[inline]
pub fn lane(&self) -> u32 {
self.lane
}
}
pub struct GpuPeer {
region: PeerRegion,
lane_set: LaneSet,
poller: poller::Poller,
pool: Option<VramPool>,
calibration: PeerCalibration,
_module: Arc<CudaModule>,
_stream: Arc<CudaStream>,
wide_stream: Arc<CudaStream>,
_ctx: Arc<CudaContext>,
}
impl GpuPeer {
pub fn init(config: GpuPeerConfig) -> Result<Self, GpuPeerError> {
let ctx = CudaContext::new(config.device_ordinal)
.map_err(|e| GpuPeerError::NoDevice(format!("{e:?}")))?;
let stream = ctx.default_stream();
let module = match &config.user_ops_cuda {
None => match ctx.load_module(Ptx::from_src(PEER_PTX)) {
Ok(m) => m,
Err(ptx_err) => {
eprintln!(
"flynnel gpu_peer: checked-in PTX rejected ({ptx_err:?}); \
compiling the peer kernels with NVRTC instead"
);
let ptx = cudarc::nvrtc::compile_ptx(PEER_CU).map_err(|e| {
GpuPeerError::Driver(format!(
"PTX load: {ptx_err:?}; NVRTC fallback compile: {e:?}"
))
})?;
ctx.load_module(ptx).map_err(|e| {
GpuPeerError::Driver(format!("NVRTC-fallback PTX load: {e:?}"))
})?
}
},
Some(user_src) => {
let src = format!("#define FLYNNEL_USER_OPS 1\n{PEER_CU}\n{user_src}\n");
let ptx = cudarc::nvrtc::compile_ptx(src).map_err(|e| {
GpuPeerError::Driver(format!("user-ops NVRTC compile: {e:?}"))
})?;
ctx.load_module(ptx)
.map_err(|e| GpuPeerError::Driver(format!("user-ops PTX load: {e:?}")))?
}
};
let load = |name: &str| {
module
.load_function(name)
.map_err(|e| GpuPeerError::Driver(format!("kernel `{name}`: {e:?}")))
};
let f_poller = load("flynnel_peer_poller")?;
let kernels = calibration::CalibKernels {
calib_pong: load("flynnel_peer_calib_pong")?,
fischer: load("flynnel_peer_fischer")?,
cas_probe: load("flynnel_peer_cas_probe")?,
};
let geometry = Geometry {
lanes: config.lanes.max(1),
slot_bytes: config.slot_bytes.max(64),
slots_per_lane: config.slots_per_lane.max(2),
};
let (path, remove_on_drop) = match &config.region_path {
Some(p) => (p.clone(), false),
None => {
static INSTANCE: core::sync::atomic::AtomicU32 =
core::sync::atomic::AtomicU32::new(0);
let n = INSTANCE.fetch_add(1, core::sync::atomic::Ordering::Relaxed);
(
std::env::temp_dir().join(format!(
"flynnel_gpu_peer_{}_{n}.bin",
std::process::id()
)),
true,
)
}
};
let region = PeerRegion::create(&ctx, &path, geometry, remove_on_drop)?;
let calibration = calibration::calibrate(®ion, &stream, &kernels)?;
if !calibration.doorbell_ok {
return Err(GpuPeerError::Unavailable(
"doorbell handshake failed during calibration",
));
}
let pool = if config.vram_blocks > 0 {
Some(VramPool::new(
&ctx,
&stream,
config.vram_block_bytes.max(64),
config.vram_blocks,
)?)
} else {
None
};
let (vbase, vbytes, vblocks) = match &pool {
Some(p) => (p.base(), p.block_bytes(), p.blocks()),
None => (0, 0, 0),
};
let lane_set = LaneSet::new(geometry);
let poller = poller::Poller::new(
Arc::clone(&stream),
f_poller,
config.quantum_ns,
config.idle_exit_ns,
vbase,
vbytes,
vblocks,
config.blocks_per_lane,
);
let wide_stream = ctx
.new_stream()
.map_err(|e| GpuPeerError::Driver(format!("wide stream: {e:?}")))?;
Ok(Self {
region,
lane_set,
poller,
pool,
calibration,
_module: module,
_stream: stream,
wide_stream,
_ctx: ctx,
})
}
#[inline]
pub fn calibration(&self) -> PeerCalibration {
self.calibration
}
#[inline]
pub fn geometry(&self) -> Geometry {
self.region.geometry()
}
#[inline]
pub fn region(&self) -> &PeerRegion {
&self.region
}
pub fn submit(&mut self, op: u32, payload: &[u8]) -> Result<Ticket, GpuPeerError> {
let t0 = Instant::now();
loop {
if let Some(t) = self.lane_set.try_submit(&self.region, op, payload)? {
self.poller.ensure_running(&self.region)?;
return Ok(t);
}
self.poller.ensure_running(&self.region)?;
if t0.elapsed() > Duration::from_secs(10) {
return Err(GpuPeerError::Timeout);
}
std::thread::yield_now();
}
}
#[inline]
pub fn is_done(&self, ticket: Ticket) -> bool {
self.lane_set.is_done(&self.region, ticket)
}
pub fn wait(&mut self, ticket: Ticket, timeout: Duration) -> Result<u32, GpuPeerError> {
let t0 = Instant::now();
let mut spins = 0u32;
while !self.lane_set.is_done(&self.region, ticket) {
spins = spins.wrapping_add(1);
if spins.is_multiple_of(4096) {
self.poller.ensure_running(&self.region)?;
if t0.elapsed() > timeout {
return Err(GpuPeerError::Timeout);
}
}
core::hint::spin_loop();
}
Ok(self.lane_set.status(&self.region, ticket))
}
pub fn read_result(&self, ticket: Ticket, dst: &mut [u8]) {
self.lane_set.read_result(&self.region, ticket, dst);
}
pub fn reap(&mut self, ticket: Ticket) -> Result<(), GpuPeerError> {
self.lane_set.reap(ticket)
}
fn submit_on_lane(
&mut self,
lane: u32,
op: u32,
payload: &[u8],
) -> Result<Ticket, GpuPeerError> {
let t0 = Instant::now();
loop {
if let Some(t) = self.lane_set.try_submit_on(&self.region, lane, op, payload)? {
self.poller.ensure_running(&self.region)?;
return Ok(t);
}
self.poller.ensure_running(&self.region)?;
if t0.elapsed() > Duration::from_secs(10) {
return Err(GpuPeerError::Timeout);
}
std::thread::yield_now();
}
}
pub fn pin(&mut self, data: &[u8]) -> Result<ResidentHandle, GpuPeerError> {
let pool = self
.pool
.as_mut()
.ok_or(GpuPeerError::Unavailable("resident pool disabled"))?;
let block_bytes = pool.block_bytes() as usize;
if data.len() > block_bytes
|| data.len() + RESIDENT_PARAMS_BYTES > self.region.geometry().payload_max()
{
return Err(GpuPeerError::PayloadTooLarge {
len: data.len(),
max: block_bytes.min(
self.region.geometry().payload_max() - RESIDENT_PARAMS_BYTES,
),
});
}
let block = pool
.alloc()
.ok_or(GpuPeerError::Unavailable("resident pool exhausted"))?;
let lane = block % self.region.geometry().lanes;
let mut payload = Vec::with_capacity(RESIDENT_PARAMS_BYTES + data.len());
payload.extend_from_slice(&block.to_le_bytes());
payload.extend_from_slice(&(data.len() as u32).to_le_bytes());
payload.extend_from_slice(data);
let t = self.submit_on_lane(lane, OP_H2V, &payload)?;
let status = self.wait(t, Duration::from_secs(10))?;
self.reap(t)?;
if status != STATUS_DONE {
if let Some(p) = self.pool.as_mut() {
p.release(block);
}
return Err(GpuPeerError::Unavailable("resident upload rejected"));
}
Ok(ResidentHandle { block, lane, bytes: data.len() as u32 })
}
pub fn write_resident(
&mut self,
handle: &ResidentHandle,
data: &[u8],
) -> Result<(), GpuPeerError> {
if data.len() > handle.len() {
return Err(GpuPeerError::PayloadTooLarge { len: data.len(), max: handle.len() });
}
let mut payload = Vec::with_capacity(RESIDENT_PARAMS_BYTES + data.len());
payload.extend_from_slice(&handle.block.to_le_bytes());
payload.extend_from_slice(&(data.len() as u32).to_le_bytes());
payload.extend_from_slice(data);
let t = self.submit_on_lane(handle.lane, layout::OP_H2V, &payload)?;
let status = self.wait(t, Duration::from_secs(10))?;
self.reap(t)?;
if status != STATUS_DONE {
return Err(GpuPeerError::Unavailable("resident re-upload rejected"));
}
Ok(())
}
pub fn pin_bulk(&mut self, data: &[u8]) -> Result<ResidentHandle, GpuPeerError> {
let pool = self
.pool
.as_mut()
.ok_or(GpuPeerError::Unavailable("resident pool disabled"))?;
let block_bytes = pool.block_bytes() as usize;
let need = data.len().div_ceil(block_bytes.max(1)).max(1);
if need > pool.free_blocks() {
return Err(GpuPeerError::Unavailable("resident pool exhausted"));
}
let first = pool
.alloc_span(need as u32)
.ok_or(GpuPeerError::Unavailable("no contiguous resident span"))?;
let dst = pool.block_ptr(first);
unsafe {
cudarc::driver::result::memcpy_htod_async(
dst,
data,
self.wide_stream.cu_stream() as _,
)
.map_err(|e| GpuPeerError::Driver(format!("pin_bulk htod: {e:?}")))?;
}
self.wide_stream
.synchronize()
.map_err(|e| GpuPeerError::Driver(format!("pin_bulk sync: {e:?}")))?;
Ok(ResidentHandle {
block: first,
lane: first % self.region.geometry().lanes,
bytes: data.len() as u32,
})
}
pub fn write_resident_bulk(
&mut self,
handle: &ResidentHandle,
data: &[u8],
) -> Result<(), GpuPeerError> {
let (dst, _) = self.resident_ptr(handle)?;
unsafe {
cudarc::driver::result::memcpy_htod_async(
dst,
data,
self.wide_stream.cu_stream() as _,
)
.map_err(|e| GpuPeerError::Driver(format!("write_resident_bulk: {e:?}")))?;
}
self.wide_stream
.synchronize()
.map_err(|e| GpuPeerError::Driver(format!("write_resident_bulk sync: {e:?}")))
}
pub fn fetch_bulk(
&mut self,
handle: &ResidentHandle,
out: &mut [u8],
) -> Result<(), GpuPeerError> {
let (src, _) = self.resident_ptr(handle)?;
unsafe {
cudarc::driver::result::memcpy_dtoh_async(
out,
src,
self.wide_stream.cu_stream() as _,
)
.map_err(|e| GpuPeerError::Driver(format!("fetch_bulk: {e:?}")))?;
}
self.wide_stream
.synchronize()
.map_err(|e| GpuPeerError::Driver(format!("fetch_bulk sync: {e:?}")))
}
pub fn pin_prefetch(
&mut self,
data: &[u8],
) -> Result<(ResidentHandle, Ticket), GpuPeerError> {
let pool = self
.pool
.as_mut()
.ok_or(GpuPeerError::Unavailable("resident pool disabled"))?;
let block_bytes = pool.block_bytes() as usize;
if data.len() > block_bytes
|| data.len() + RESIDENT_PARAMS_BYTES > self.region.geometry().payload_max()
{
return Err(GpuPeerError::PayloadTooLarge {
len: data.len(),
max: block_bytes
.min(self.region.geometry().payload_max() - RESIDENT_PARAMS_BYTES),
});
}
let block = pool
.alloc()
.ok_or(GpuPeerError::Unavailable("resident pool exhausted"))?;
let lane = block % self.region.geometry().lanes;
let mut payload = Vec::with_capacity(RESIDENT_PARAMS_BYTES + data.len());
payload.extend_from_slice(&block.to_le_bytes());
payload.extend_from_slice(&(data.len() as u32).to_le_bytes());
payload.extend_from_slice(data);
let t = self.submit_on_lane(lane, layout::OP_H2V, &payload)?;
Ok((ResidentHandle { block, lane, bytes: data.len() as u32 }, t))
}
pub fn submit_user(
&mut self,
op: u32,
handle: Option<&ResidentHandle>,
args: &[u8],
) -> Result<Ticket, GpuPeerError> {
if op < layout::OP_USER_BASE {
return Err(GpuPeerError::Unavailable("op below OP_USER_BASE"));
}
let (block, count, lane) = match handle {
Some(h) => (h.block, h.bytes, Some(h.lane)),
None => (layout::NO_BLOCK, 0u32, None),
};
let mut payload = Vec::with_capacity(RESIDENT_PARAMS_BYTES + args.len());
payload.extend_from_slice(&block.to_le_bytes());
payload.extend_from_slice(&count.to_le_bytes());
payload.extend_from_slice(args);
match lane {
Some(l) => self.submit_on_lane(l, op, &payload),
None => self.submit(op, &payload),
}
}
pub fn submit_resident(
&mut self,
op: u32,
handle: &ResidentHandle,
) -> Result<Ticket, GpuPeerError> {
let mut params = [0u8; RESIDENT_PARAMS_BYTES];
params[..4].copy_from_slice(&handle.block.to_le_bytes());
params[4..].copy_from_slice(&handle.bytes.to_le_bytes());
self.submit_on_lane(handle.lane, op, ¶ms)
}
pub fn fetch(
&mut self,
handle: &ResidentHandle,
out: &mut [u8],
) -> Result<(), GpuPeerError> {
let mut params = [0u8; RESIDENT_PARAMS_BYTES];
params[..4].copy_from_slice(&handle.block.to_le_bytes());
params[4..].copy_from_slice(&handle.bytes.to_le_bytes());
let t = self.submit_on_lane(handle.lane, OP_V2H, ¶ms)?;
let status = self.wait(t, Duration::from_secs(10))?;
if status != STATUS_DONE {
self.reap(t)?;
return Err(GpuPeerError::Unavailable("resident download rejected"));
}
let n = out.len().min(handle.len());
let mut buf = vec![0u8; RESIDENT_PARAMS_BYTES + n];
self.read_result(t, &mut buf);
out[..n].copy_from_slice(&buf[RESIDENT_PARAMS_BYTES..]);
self.reap(t)?;
Ok(())
}
pub fn unpin(&mut self, handle: ResidentHandle) -> Result<(), GpuPeerError> {
let pool = self
.pool
.as_mut()
.ok_or(GpuPeerError::Unavailable("resident pool disabled"))?;
let span = (handle.bytes as usize)
.div_ceil(pool.block_bytes().max(1) as usize)
.max(1) as u32;
for b in handle.block..handle.block + span {
pool.release(b);
}
Ok(())
}
pub fn pool_stats(&self) -> (usize, u32) {
match &self.pool {
Some(p) => (p.free_blocks(), p.blocks()),
None => (0, 0),
}
}
pub fn resident_ptr(&self, handle: &ResidentHandle) -> Result<(u64, usize), GpuPeerError> {
let pool = self
.pool
.as_ref()
.ok_or(GpuPeerError::Unavailable("resident pool disabled"))?;
Ok((pool.block_ptr(handle.block), handle.len()))
}
pub fn compile_wide_kernel(&self, src: &str, entry: &str) -> Result<WideKernel, GpuPeerError> {
let ptx = cudarc::nvrtc::compile_ptx(src)
.map_err(|e| GpuPeerError::Driver(format!("wide-kernel NVRTC compile: {e:?}")))?;
self.load_wide_kernel(ptx, entry)
}
pub fn load_wide_kernel_ptx(&self, ptx: &str, entry: &str) -> Result<WideKernel, GpuPeerError> {
self.load_wide_kernel(Ptx::from_src(ptx), entry)
}
fn load_wide_kernel(&self, ptx: Ptx, entry: &str) -> Result<WideKernel, GpuPeerError> {
let module = self
._ctx
.load_module(ptx)
.map_err(|e| GpuPeerError::Driver(format!("wide-kernel PTX load: {e:?}")))?;
let func = module
.load_function(entry)
.map_err(|e| GpuPeerError::Driver(format!("wide-kernel entry `{entry}`: {e:?}")))?;
Ok(WideKernel { _module: module, func })
}
pub fn context(&self) -> &Arc<CudaContext> {
&self._ctx
}
pub fn wide_stream(&self) -> &Arc<CudaStream> {
&self.wide_stream
}
fn launch_wide_inner(
&self,
kernel: &WideKernel,
grid_blocks: u32,
block_threads: u32,
ptrs: &[u64],
scalars: &[u32],
) -> Result<(), GpuPeerError> {
let block = block_threads.clamp(1, 1024);
let grid = if grid_blocks == 0 {
scalars.first().copied().unwrap_or(1).div_ceil(block).max(1)
} else {
grid_blocks
};
let mut b = self.wide_stream.launch_builder(&kernel.func);
for p in ptrs {
b.arg(p);
}
for s in scalars {
b.arg(s);
}
unsafe {
b.launch(LaunchConfig {
grid_dim: (grid, 1, 1),
block_dim: (block, 1, 1),
shared_mem_bytes: 0,
})
}
.map_err(|e| GpuPeerError::Driver(format!("wide launch: {e:?}")))?;
Ok(())
}
pub fn launch_wide(
&self,
kernel: &WideKernel,
grid_blocks: u32,
block_threads: u32,
ptrs: &[u64],
scalars: &[u32],
) -> Result<(), GpuPeerError> {
self.launch_wide_inner(kernel, grid_blocks, block_threads, ptrs, scalars)?;
self.sync_wide()
}
pub fn launch_wide_async(
&self,
kernel: &WideKernel,
grid_blocks: u32,
block_threads: u32,
ptrs: &[u64],
scalars: &[u32],
) -> Result<(), GpuPeerError> {
self.launch_wide_inner(kernel, grid_blocks, block_threads, ptrs, scalars)
}
pub fn sync_wide(&self) -> Result<(), GpuPeerError> {
self.wide_stream
.synchronize()
.map_err(|e| GpuPeerError::Driver(format!("wide sync: {e:?}")))
}
pub fn pause_poller(&mut self) -> Result<(), GpuPeerError> {
self.poller.pause(&self.region)
}
pub fn resume_poller(&mut self) {
self.poller.resume();
}
#[inline]
pub fn poller_paused(&self) -> bool {
self.poller.is_paused()
}
pub fn l2_capability(&self) -> Result<L2Capability, GpuPeerError> {
L2Capability::query(self._ctx_ref())
}
pub fn l2_benchmark(
&self,
hot_bytes: usize,
pol_bytes: usize,
iters: u32,
runs: u32,
) -> Result<L2BenchReport, GpuPeerError> {
l2_persist::benchmark(self._ctx_ref(), hot_bytes, pol_bytes, iters, runs)
}
#[inline]
fn _ctx_ref(&self) -> &Arc<CudaContext> {
&self._ctx
}
pub fn timed_lock_acquire(&self, timeout: Duration) -> Result<(), GpuPeerError> {
if !self.calibration.timed_lock_ok {
return Err(GpuPeerError::Unavailable("timed lock not validated on this host"));
}
if timed_lock::acquire(
&self.region,
Duration::from_nanos(self.calibration.delta_ns),
timeout,
) {
Ok(())
} else {
Err(GpuPeerError::Timeout)
}
}
pub fn timed_lock_release(&self) {
timed_lock::release(&self.region);
}
}
impl Drop for GpuPeer {
fn drop(&mut self) {
self.poller.shutdown(&self.region);
}
}