use std::sync::Arc;
use std::time::{Duration, Instant};
use cudarc::driver::sys as cu;
use cudarc::driver::{CudaFunction, CudaStream, LaunchConfig, PushKernelArg};
use super::GpuPeerError;
use super::layout::{
self, FLAG_CALIBRATED, FLAG_DOORBELL_OK, FLAG_SYS_ATOMICS_OK, FLAG_TIMED_LOCK_OK, GTS_SLOTS,
HDR_CALIB_PING_OFF, HDR_CALIB_PONG_OFF, HDR_FISCHER_ACQS_OFF, HDR_FISCHER_CS_OFF,
HDR_FISCHER_VIOL_OFF, HDR_FISCHER_X_OFF, HDR_GTS_OFF,
};
use super::region::PeerRegion;
use super::timed_lock;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PeerCalibration {
pub rtt_min_ns: u64,
pub rtt_median_ns: u64,
pub rtt_p99_ns: u64,
pub one_way_ns: u64,
pub clock_err_ns: u64,
pub delta_ns: u64,
pub launch_ns: u64,
pub doorbell_ok: bool,
pub timed_lock_ok: bool,
pub sys_atomics_ok: bool,
pub lock_cpu_contended: u32,
pub lock_gpu_contended: u32,
}
impl PeerCalibration {
pub fn store(&self, r: &PeerRegion) {
r.store_u64(layout::HDR_RTT_MIN_OFF, self.rtt_min_ns);
r.store_u64(layout::HDR_RTT_MED_OFF, self.rtt_median_ns);
r.store_u64(layout::HDR_RTT_P99_OFF, self.rtt_p99_ns);
r.store_u64(layout::HDR_ONE_WAY_OFF, self.one_way_ns);
r.store_u64(layout::HDR_CLOCK_ERR_OFF, self.clock_err_ns);
r.store_u64(layout::HDR_DELTA_OFF, self.delta_ns);
r.store_u64(layout::HDR_LAUNCH_OFF, self.launch_ns);
let mut flags = FLAG_CALIBRATED;
if self.doorbell_ok {
flags |= FLAG_DOORBELL_OK;
}
if self.timed_lock_ok {
flags |= FLAG_TIMED_LOCK_OK;
}
if self.sys_atomics_ok {
flags |= FLAG_SYS_ATOMICS_OK;
}
r.store_u64(layout::HDR_FLAGS_OFF, flags);
}
}
pub fn derive_delta_ns(one_way_ns: u64) -> u64 {
(one_way_ns.saturating_mul(10)).clamp(5_000, 100_000)
}
pub struct CalibKernels {
pub calib_pong: CudaFunction,
pub fischer: CudaFunction,
pub cas_probe: CudaFunction,
}
fn one_one() -> LaunchConfig {
LaunchConfig { grid_dim: (1, 1, 1), block_dim: (1, 1, 1), shared_mem_bytes: 0 }
}
fn flush_stream(stream: &Arc<CudaStream>) {
let _rc: cu::CUresult = unsafe { cu::cuStreamQuery(stream.cu_stream()) };
}
fn now_ns(t0: Instant) -> u64 {
t0.elapsed().as_nanos() as u64
}
pub fn calibrate(
region: &PeerRegion,
stream: &Arc<CudaStream>,
kernels: &CalibKernels,
) -> Result<PeerCalibration, GpuPeerError> {
let dev_base = region.dev_base();
let mut launch_samples = Vec::with_capacity(32);
for _ in 0..32 {
let zero_rounds: u32 = 0;
let t0 = Instant::now();
let mut b = stream.launch_builder(&kernels.calib_pong);
b.arg(&dev_base);
b.arg(&zero_rounds);
unsafe { b.launch(one_one()) }
.map_err(|e| GpuPeerError::Driver(format!("calib launch: {e:?}")))?;
stream
.synchronize()
.map_err(|e| GpuPeerError::Driver(format!("calib sync: {e:?}")))?;
launch_samples.push(now_ns(t0));
}
launch_samples.sort_unstable();
let launch_ns = launch_samples[launch_samples.len() / 2];
let rounds: u32 = (GTS_SLOTS - 8) as u32;
region.store_u32(HDR_CALIB_PING_OFF, 0);
region.store_u32(HDR_CALIB_PONG_OFF, 0);
let mut b = stream.launch_builder(&kernels.calib_pong);
b.arg(&dev_base);
b.arg(&rounds);
unsafe { b.launch(one_one()) }
.map_err(|e| GpuPeerError::Driver(format!("pong launch: {e:?}")))?;
flush_stream(stream);
let wall = Instant::now();
let mut rtt = Vec::with_capacity(rounds as usize);
let mut pairs: Vec<(u64, u64, u64)> = Vec::with_capacity(rounds as usize); let mut doorbell_ok = true;
for i in 1..=rounds {
let t0 = now_ns(wall);
region.store_u32(HDR_CALIB_PING_OFF, i);
region.release_fence();
loop {
if region.load_u32(HDR_CALIB_PONG_OFF) == i {
break;
}
if now_ns(wall).saturating_sub(t0) > 2_000_000_000 {
doorbell_ok = false;
break;
}
core::hint::spin_loop();
}
if !doorbell_ok {
break;
}
let t1 = now_ns(wall);
let gts = region.load_u64(HDR_GTS_OFF + (i as usize) * 8);
rtt.push(t1 - t0);
pairs.push((t1 - t0, gts, t0 + (t1 - t0) / 2));
}
stream
.synchronize()
.map_err(|e| GpuPeerError::Driver(format!("pong drain: {e:?}")))?;
if !doorbell_ok || rtt.len() < 16 {
let cal = PeerCalibration {
rtt_min_ns: 0,
rtt_median_ns: 0,
rtt_p99_ns: 0,
one_way_ns: 0,
clock_err_ns: 0,
delta_ns: 0,
launch_ns,
doorbell_ok: false,
timed_lock_ok: false,
sys_atomics_ok: false,
lock_cpu_contended: 0,
lock_gpu_contended: 0,
};
cal.store(region);
return Ok(cal);
}
rtt.sort_unstable();
let rtt_min_ns = rtt[0];
let rtt_median_ns = rtt[rtt.len() / 2];
let rtt_p99_ns = rtt[(rtt.len() * 99) / 100];
pairs.sort_unstable_by_key(|p| p.0);
let q = (pairs.len() / 4).clamp(4, pairs.len());
let (_, g0, m0) = pairs[0];
let mut mn = i64::MAX;
let mut mx = i64::MIN;
for &(_, g, m) in pairs.iter().take(q) {
let off = (g.wrapping_sub(g0) as i64) - (m.wrapping_sub(m0) as i64);
mn = mn.min(off);
mx = mx.max(off);
}
let clock_err_ns = ((mx - mn) / 2).max(1) as u64;
let one_way_ns = rtt_p99_ns / 2 + clock_err_ns;
let delta_ns = derive_delta_ns(one_way_ns);
let validated = validate(region, stream, kernels, delta_ns)?;
let cal = PeerCalibration {
rtt_min_ns,
rtt_median_ns,
rtt_p99_ns,
one_way_ns,
clock_err_ns,
delta_ns: validated.delta_ns,
launch_ns,
doorbell_ok: true,
timed_lock_ok: validated.timed_lock_ok,
sys_atomics_ok: validated.sys_atomics_ok,
lock_cpu_contended: validated.lock_cpu_contended,
lock_gpu_contended: validated.lock_gpu_contended,
};
cal.store(region);
Ok(cal)
}
struct Validated {
delta_ns: u64,
timed_lock_ok: bool,
sys_atomics_ok: bool,
lock_cpu_contended: u32,
lock_gpu_contended: u32,
}
fn validate(
region: &PeerRegion,
stream: &Arc<CudaStream>,
kernels: &CalibKernels,
mut delta_ns: u64,
) -> Result<Validated, GpuPeerError> {
let mut timed_lock_ok = false;
let mut passes = 0u32;
let mut lock_cpu_contended = 0u32;
let mut lock_gpu_contended = 0u32;
for _attempt in 0..6 {
match fischer_self_test(region, stream, kernels, delta_ns)? {
SelfTest::Pass { cpu_contended, gpu_contended } => {
passes += 1;
lock_cpu_contended = cpu_contended;
lock_gpu_contended = gpu_contended;
if passes >= 2 {
timed_lock_ok = true;
break;
}
}
SelfTest::Violated => {
passes = 0;
delta_ns = (delta_ns * 2).min(100_000);
}
SelfTest::Inconclusive => {
passes = 0;
}
}
}
let sys_atomics_ok = cas_conservation_probe(region, stream, kernels)?
&& cas_conservation_probe(region, stream, kernels)?;
Ok(Validated {
delta_ns,
timed_lock_ok,
sys_atomics_ok,
lock_cpu_contended,
lock_gpu_contended,
})
}
pub fn calibrate_with_prior(
region: &PeerRegion,
stream: &Arc<CudaStream>,
kernels: &CalibKernels,
prior: PeerCalibration,
) -> Result<PeerCalibration, GpuPeerError> {
let validated = validate(region, stream, kernels, derive_delta_ns(prior.one_way_ns))?;
let cal = PeerCalibration {
delta_ns: validated.delta_ns,
timed_lock_ok: validated.timed_lock_ok,
sys_atomics_ok: validated.sys_atomics_ok,
lock_cpu_contended: validated.lock_cpu_contended,
lock_gpu_contended: validated.lock_gpu_contended,
..prior
};
cal.store(region);
Ok(cal)
}
enum SelfTest {
Pass { cpu_contended: u32, gpu_contended: u32 },
Violated,
Inconclusive,
}
fn fischer_self_test(
region: &PeerRegion,
stream: &Arc<CudaStream>,
kernels: &CalibKernels,
delta_ns: u64,
) -> Result<SelfTest, GpuPeerError> {
const ACQS: u32 = 150;
const CS_NS: u64 = 2_000;
const CONTENTION_FLOOR: u32 = ACQS / 10;
region.store_u32(HDR_FISCHER_X_OFF, 0);
region.store_i32(HDR_FISCHER_CS_OFF, 0);
region.store_u32(HDR_FISCHER_VIOL_OFF, 0);
region.store_u32(HDR_FISCHER_ACQS_OFF, 0);
region.store_u32(layout::HDR_FISCHER_STARTED_OFF, 0);
region.store_u32(layout::HDR_FISCHER_GPU_CONT_OFF, 0);
region.release_fence();
let dev_base = region.dev_base();
let acqs = ACQS;
let cs = CS_NS;
let mut b = stream.launch_builder(&kernels.fischer);
b.arg(&dev_base);
b.arg(&acqs);
b.arg(&delta_ns);
b.arg(&cs);
unsafe { b.launch(one_one()) }
.map_err(|e| GpuPeerError::Driver(format!("fischer launch: {e:?}")))?;
flush_stream(stream);
let t0 = Instant::now();
while region.load_u32(layout::HDR_FISCHER_STARTED_OFF) == 0 {
if t0.elapsed() > Duration::from_secs(3) {
stream.synchronize().ok();
return Ok(SelfTest::Inconclusive);
}
core::hint::spin_loop();
}
let delta = Duration::from_nanos(delta_ns);
let cs_hold = Duration::from_nanos(CS_NS);
let deadline = Duration::from_secs(6);
let mut cpu_done = 0u32;
let mut cpu_contended = 0u32;
for _ in 0..ACQS {
match timed_lock::acquire_with_contention(region, delta, deadline) {
Some(contended) => {
if contended {
cpu_contended += 1;
}
timed_lock::critical_section_checked(region, cs_hold);
timed_lock::release(region);
cpu_done += 1;
}
None => break,
}
}
let t0 = Instant::now();
while region.load_u32(HDR_FISCHER_ACQS_OFF) < ACQS {
if t0.elapsed() > Duration::from_secs(6) {
break;
}
std::thread::yield_now();
}
stream
.synchronize()
.map_err(|e| GpuPeerError::Driver(format!("fischer drain: {e:?}")))?;
let gpu_done = region.load_u32(HDR_FISCHER_ACQS_OFF);
let gpu_contended = region.load_u32(layout::HDR_FISCHER_GPU_CONT_OFF);
let violations = region.load_u32(HDR_FISCHER_VIOL_OFF);
if violations > 0 {
return Ok(SelfTest::Violated);
}
if cpu_done == ACQS
&& gpu_done == ACQS
&& cpu_contended >= CONTENTION_FLOOR
&& gpu_contended >= CONTENTION_FLOOR
{
Ok(SelfTest::Pass { cpu_contended, gpu_contended })
} else {
Ok(SelfTest::Inconclusive)
}
}
fn cas_conservation_probe(
region: &PeerRegion,
stream: &Arc<CudaStream>,
kernels: &CalibKernels,
) -> Result<bool, GpuPeerError> {
let g = region.geometry();
let slab = g.slab_off();
let slab_bytes = g.region_bytes() - slab;
let n = ((slab_bytes / 4).saturating_sub(32)).min(60_000) as u32;
if n < 4_096 {
return Ok(false);
}
for i in 0..n as usize {
region.store_u32(slab + i * 4, 0);
}
let gpu_won_off = slab + n as usize * 4;
let started_off = gpu_won_off + 4;
region.store_u32(gpu_won_off, 0);
region.store_u32(started_off, 0);
region.release_fence();
let dev_slots = region.dev_base() + slab as u64;
let dev_won = region.dev_base() + gpu_won_off as u64;
let dev_started = region.dev_base() + started_off as u64;
let mut b = stream.launch_builder(&kernels.cas_probe);
b.arg(&dev_slots);
b.arg(&n);
b.arg(&dev_won);
b.arg(&dev_started);
unsafe {
b.launch(LaunchConfig {
grid_dim: (1, 1, 1),
block_dim: (64, 1, 1),
shared_mem_bytes: 0,
})
}
.map_err(|e| GpuPeerError::Driver(format!("cas probe launch: {e:?}")))?;
flush_stream(stream);
let t0 = Instant::now();
while region.load_u32(started_off) == 0 {
if t0.elapsed() > Duration::from_secs(3) {
stream.synchronize().ok();
return Ok(false); }
core::hint::spin_loop();
}
let base = region.base_addr() as usize;
let mut cpu_won = 0u64;
for i in 0..n as usize {
let a = unsafe { core::sync::atomic::AtomicU32::from_ptr((base + slab + i * 4) as *mut u32) };
if a
.compare_exchange(
0,
1,
core::sync::atomic::Ordering::SeqCst,
core::sync::atomic::Ordering::SeqCst,
)
.is_ok()
{
cpu_won += 1;
}
}
stream
.synchronize()
.map_err(|e| GpuPeerError::Driver(format!("cas probe drain: {e:?}")))?;
let gpu_won = region.load_u32(gpu_won_off) as u64;
let mut unclaimed = 0u64;
for i in 0..n as usize {
if region.load_u32(slab + i * 4) == 0 {
unclaimed += 1;
}
}
let conserved = unclaimed == 0 && cpu_won + gpu_won == n as u64;
let floor = n as u64 / 20;
let contended = cpu_won >= floor && gpu_won >= floor;
Ok(conserved && contended)
}
#[cfg(test)]
mod tests {
use super::derive_delta_ns;
#[test]
fn delta_derivation_clamps() {
assert_eq!(derive_delta_ns(0), 5_000);
assert_eq!(derive_delta_ns(100), 5_000);
assert_eq!(derive_delta_ns(1_600), 16_000);
assert_eq!(derive_delta_ns(2_500), 25_000);
assert_eq!(derive_delta_ns(50_000), 100_000);
}
}