use std::sync::Arc;
use cudarc::driver::CudaContext;
use cudarc::driver::sys as cu;
use super::GpuPeerError;
const LIMIT_PERSISTING_L2: cu::CUlimit = cu::CUlimit::CU_LIMIT_PERSISTING_L2_CACHE_SIZE;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct L2Capability {
pub max_persisting_l2: usize,
pub max_access_window: usize,
}
impl L2Capability {
#[inline]
pub fn supported(&self) -> bool {
self.max_persisting_l2 > 0 && self.max_access_window > 0
}
pub fn query(ctx: &Arc<CudaContext>) -> Result<Self, GpuPeerError> {
ctx.bind_to_thread()
.map_err(|e| GpuPeerError::Driver(format!("bind_to_thread: {e:?}")))?;
let mut dev: cu::CUdevice = 0;
let r = unsafe { cu::cuCtxGetDevice(&mut dev) };
if r != cu::CUresult::CUDA_SUCCESS {
return Err(GpuPeerError::Driver(format!("cuCtxGetDevice: {r:?}")));
}
let attr = |a: cu::CUdevice_attribute| -> usize {
let mut v: i32 = 0;
let r = unsafe { cu::cuDeviceGetAttribute(&mut v, a, dev) };
if r == cu::CUresult::CUDA_SUCCESS && v > 0 { v as usize } else { 0 }
};
Ok(Self {
max_persisting_l2: attr(
cu::CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MAX_PERSISTING_L2_CACHE_SIZE,
),
max_access_window: attr(
cu::CUdevice_attribute::CU_DEVICE_ATTRIBUTE_MAX_ACCESS_POLICY_WINDOW_SIZE,
),
})
}
}
pub struct L2Persist {
ctx: Arc<CudaContext>,
cap: L2Capability,
reserved: usize,
}
impl L2Persist {
pub fn reserve(ctx: &Arc<CudaContext>, bytes: usize) -> Result<Self, GpuPeerError> {
let cap = L2Capability::query(ctx)?;
if !cap.supported() {
return Err(GpuPeerError::Unavailable("L2 persistence unsupported on this device"));
}
let want = bytes.min(cap.max_persisting_l2);
let r = unsafe { cu::cuCtxSetLimit(LIMIT_PERSISTING_L2, want) };
if r != cu::CUresult::CUDA_SUCCESS {
return Err(GpuPeerError::Driver(format!("cuCtxSetLimit(persisting L2): {r:?}")));
}
let mut got: usize = 0;
let _rc: cu::CUresult = unsafe { cu::cuCtxGetLimit(&mut got, LIMIT_PERSISTING_L2) };
Ok(Self { ctx: Arc::clone(ctx), cap, reserved: got })
}
#[inline]
pub fn capability(&self) -> L2Capability {
self.cap
}
#[inline]
pub fn reserved(&self) -> usize {
self.reserved
}
pub fn pin_window(
&self,
stream: &cudarc::driver::CudaStream,
dev_ptr: u64,
num_bytes: usize,
hit_ratio: f32,
) -> Result<(), GpuPeerError> {
let n = num_bytes.min(self.cap.max_access_window);
let window = cu::CUaccessPolicyWindow_st {
base_ptr: dev_ptr as *mut core::ffi::c_void,
num_bytes: n,
hitRatio: hit_ratio.clamp(0.0, 1.0),
hitProp: cu::CUaccessProperty::CU_ACCESS_PROPERTY_PERSISTING,
missProp: cu::CUaccessProperty::CU_ACCESS_PROPERTY_STREAMING,
};
let mut value: cu::CUstreamAttrValue = unsafe { core::mem::zeroed() };
value.accessPolicyWindow = window;
let r = unsafe {
cu::cuStreamSetAttribute(
stream.cu_stream(),
cu::CUstreamAttrID::CU_LAUNCH_ATTRIBUTE_ACCESS_POLICY_WINDOW,
&value,
)
};
if r != cu::CUresult::CUDA_SUCCESS {
return Err(GpuPeerError::Driver(format!("cuStreamSetAttribute: {r:?}")));
}
Ok(())
}
pub fn clear_window(&self, stream: &cudarc::driver::CudaStream) -> Result<(), GpuPeerError> {
let mut value: cu::CUstreamAttrValue = unsafe { core::mem::zeroed() };
value.accessPolicyWindow = cu::CUaccessPolicyWindow_st {
base_ptr: core::ptr::null_mut(),
num_bytes: 0,
hitRatio: 0.0,
hitProp: cu::CUaccessProperty::CU_ACCESS_PROPERTY_STREAMING,
missProp: cu::CUaccessProperty::CU_ACCESS_PROPERTY_STREAMING,
};
let r = unsafe {
cu::cuStreamSetAttribute(
stream.cu_stream(),
cu::CUstreamAttrID::CU_LAUNCH_ATTRIBUTE_ACCESS_POLICY_WINDOW,
&value,
)
};
if r != cu::CUresult::CUDA_SUCCESS {
return Err(GpuPeerError::Driver(format!("cuStreamSetAttribute(clear): {r:?}")));
}
self.ctx
.bind_to_thread()
.map_err(|e| GpuPeerError::Driver(format!("bind_to_thread: {e:?}")))?;
let _rc: cu::CUresult = unsafe { cu::cuCtxResetPersistingL2Cache() };
Ok(())
}
}
impl Drop for L2Persist {
fn drop(&mut self) {
if self.ctx.bind_to_thread().is_ok() {
let _rc: cu::CUresult = unsafe { cu::cuCtxSetLimit(LIMIT_PERSISTING_L2, 0) };
let _rc2: cu::CUresult = unsafe { cu::cuCtxResetPersistingL2Cache() };
}
}
}
#[derive(Debug, Clone, Copy)]
pub struct L2BenchReport {
pub reserved: usize,
pub hot_bytes: usize,
pub pol_bytes: usize,
pub persist_us: f64,
pub stream_us: f64,
pub speedup: f64,
}
pub fn benchmark(
ctx: &Arc<CudaContext>,
hot_bytes: usize,
pol_bytes: usize,
iters: u32,
runs: u32,
) -> Result<L2BenchReport, GpuPeerError> {
use std::time::Instant;
let l2 = L2Persist::reserve(ctx, hot_bytes)?;
let hot_bytes = (hot_bytes.min(l2.reserved()).max(4096)) & !3usize;
let hot_n = (hot_bytes / 4) as u32;
let pol_n = (pol_bytes / 4) as u32;
ctx.bind_to_thread()
.map_err(|e| GpuPeerError::Driver(format!("bind_to_thread: {e:?}")))?;
let ptx = std::ffi::CString::new(super::PEER_PTX)
.map_err(|e| GpuPeerError::Driver(format!("PTX CString: {e}")))?;
let mut module: cu::CUmodule = core::ptr::null_mut();
let r = unsafe { cu::cuModuleLoadData(&mut module, ptx.as_ptr() as *const core::ffi::c_void) };
if r != cu::CUresult::CUDA_SUCCESS {
return Err(GpuPeerError::Driver(format!("cuModuleLoadData: {r:?}")));
}
let fname = std::ffi::CString::new("flynnel_l2_hammer").expect("no NUL");
let mut func: cu::CUfunction = core::ptr::null_mut();
let r = unsafe { cu::cuModuleGetFunction(&mut func, module, fname.as_ptr()) };
if r != cu::CUresult::CUDA_SUCCESS {
let _u: cu::CUresult = unsafe { cu::cuModuleUnload(module) };
return Err(GpuPeerError::Driver(format!("cuModuleGetFunction: {r:?}")));
}
let mut hot: cu::CUdeviceptr = 0;
let mut pol: cu::CUdeviceptr = 0;
let mut out: cu::CUdeviceptr = 0;
unsafe {
let a = cu::cuMemAlloc_v2(&mut hot, hot_bytes);
let b = cu::cuMemAlloc_v2(&mut pol, pol_bytes);
let c = cu::cuMemAlloc_v2(&mut out, 8);
if a != cu::CUresult::CUDA_SUCCESS
|| b != cu::CUresult::CUDA_SUCCESS
|| c != cu::CUresult::CUDA_SUCCESS
{
let _u: cu::CUresult = cu::cuModuleUnload(module);
return Err(GpuPeerError::Driver("cuMemAlloc failed".into()));
}
let _s1: cu::CUresult = cu::cuMemsetD32_v2(hot, 1, hot_n as usize);
let _s2: cu::CUresult = cu::cuMemsetD32_v2(pol, 1, pol_n as usize);
}
let stream = ctx.default_stream();
let s = stream.cu_stream();
let launch = |func: cu::CUfunction| -> cu::CUresult {
let mut d_hot = hot;
let mut d_pol = pol;
let mut d_out = out;
let mut hn = hot_n;
let mut pn = pol_n;
let mut it = iters;
let mut params: [*mut core::ffi::c_void; 6] = [
(&mut d_hot) as *mut _ as *mut core::ffi::c_void,
(&mut hn) as *mut _ as *mut core::ffi::c_void,
(&mut d_pol) as *mut _ as *mut core::ffi::c_void,
(&mut pn) as *mut _ as *mut core::ffi::c_void,
(&mut it) as *mut _ as *mut core::ffi::c_void,
(&mut d_out) as *mut _ as *mut core::ffi::c_void,
];
unsafe {
cu::cuLaunchKernel(
func, 256, 1, 1, 256, 1, 1, 0, s, params.as_mut_ptr(), core::ptr::null_mut(),
)
}
};
let measure = |pin: bool| -> Result<f64, GpuPeerError> {
if pin {
l2.pin_window(&stream, hot, hot_bytes, 1.0)?;
} else {
l2.clear_window(&stream)?;
}
let _warm: cu::CUresult = launch(func);
let _sy: cu::CUresult = unsafe { cu::cuStreamSynchronize(s) };
let mut best = f64::INFINITY;
for _ in 0..runs.max(1) {
let t0 = Instant::now();
let lr = launch(func);
let sy = unsafe { cu::cuStreamSynchronize(s) };
if lr != cu::CUresult::CUDA_SUCCESS || sy != cu::CUresult::CUDA_SUCCESS {
return Err(GpuPeerError::Driver(format!("l2 launch: {lr:?} / sync {sy:?}")));
}
best = best.min(t0.elapsed().as_secs_f64() * 1e6);
}
Ok(best)
};
let stream_us = measure(false)?;
let persist_us = measure(true)?;
let mut host_out: u64 = 0;
let _cp: cu::CUresult = unsafe {
cu::cuMemcpyDtoH_v2((&mut host_out) as *mut u64 as *mut core::ffi::c_void, out, 8)
};
let _w: Result<(), GpuPeerError> = l2.clear_window(&stream);
unsafe {
let _f1: cu::CUresult = cu::cuMemFree_v2(hot);
let _f2: cu::CUresult = cu::cuMemFree_v2(pol);
let _f3: cu::CUresult = cu::cuMemFree_v2(out);
let _u: cu::CUresult = cu::cuModuleUnload(module);
}
if host_out == 0 {
return Err(GpuPeerError::Driver("l2 hammer produced zero (reads elided?)".into()));
}
Ok(L2BenchReport {
reserved: l2.reserved(),
hot_bytes,
pol_bytes,
persist_us,
stream_us,
speedup: stream_us / persist_us.max(f64::MIN_POSITIVE),
})
}