use super::GpuPeerError;
use super::layout::{
Geometry, LANE_HEAD_OFF, LANE_TAIL_OFF, SLOT_LEN_OFF, SLOT_OP_OFF, SLOT_PAYLOAD_OFF,
SLOT_SEQ_OFF, SLOT_STATUS_OFF, STATUS_SUBMITTED,
};
pub trait RegionWords {
fn geometry(&self) -> Geometry;
fn load_u32(&self, off: usize) -> u32;
fn store_u32(&self, off: usize, v: u32);
fn write_bytes(&self, off: usize, src: &[u8]);
fn read_bytes(&self, off: usize, dst: &mut [u8]);
fn release_fence(&self);
}
impl RegionWords for super::region::PeerRegion {
#[inline]
fn geometry(&self) -> Geometry {
Self::geometry(self)
}
#[inline]
fn load_u32(&self, off: usize) -> u32 {
Self::load_u32(self, off)
}
#[inline]
fn store_u32(&self, off: usize, v: u32) {
Self::store_u32(self, off, v)
}
#[inline]
fn write_bytes(&self, off: usize, src: &[u8]) {
Self::write_bytes(self, off, src)
}
#[inline]
fn read_bytes(&self, off: usize, dst: &mut [u8]) {
Self::read_bytes(self, off, dst)
}
#[inline]
fn release_fence(&self) {
Self::release_fence(self)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Ticket {
pub lane: u32,
pub seq: u32,
}
pub struct LaneSet {
geometry: Geometry,
next_lane: u32,
head: Vec<u32>,
reaped: Vec<u32>,
}
impl LaneSet {
pub fn new(geometry: Geometry) -> Self {
Self {
geometry,
next_lane: 0,
head: vec![0; geometry.lanes as usize],
reaped: vec![0; geometry.lanes as usize],
}
}
#[inline]
pub fn in_flight(&self, lane: u32) -> u32 {
self.head[lane as usize].wrapping_sub(self.reaped[lane as usize])
}
pub fn in_flight_total(&self) -> u32 {
(0..self.geometry.lanes).map(|l| self.in_flight(l)).sum()
}
pub fn try_submit<R: RegionWords>(
&mut self,
r: &R,
op: u32,
payload: &[u8],
) -> Result<Option<Ticket>, GpuPeerError> {
if payload.len() > self.geometry.payload_max() {
return Err(GpuPeerError::PayloadTooLarge {
len: payload.len(),
max: self.geometry.payload_max(),
});
}
let lanes = self.geometry.lanes;
for probe in 0..lanes {
let lane = (self.next_lane + probe) % lanes;
if self.in_flight(lane) < self.geometry.slots_per_lane {
self.next_lane = (lane + 1) % lanes;
return Ok(Some(self.submit_on(r, lane, op, payload)));
}
}
Ok(None)
}
pub(crate) fn try_submit_on<R: RegionWords>(
&mut self,
r: &R,
lane: u32,
op: u32,
payload: &[u8],
) -> Result<Option<Ticket>, GpuPeerError> {
if payload.len() > self.geometry.payload_max() {
return Err(GpuPeerError::PayloadTooLarge {
len: payload.len(),
max: self.geometry.payload_max(),
});
}
if self.in_flight(lane) >= self.geometry.slots_per_lane {
return Ok(None);
}
Ok(Some(self.submit_on(r, lane, op, payload)))
}
fn submit_on<R: RegionWords>(&mut self, r: &R, lane: u32, op: u32, payload: &[u8]) -> Ticket {
let seq = self.head[lane as usize];
let slot = self.geometry.slot_off(lane, seq);
r.store_u32(slot + SLOT_OP_OFF, op);
r.store_u32(slot + SLOT_LEN_OFF, payload.len() as u32);
r.store_u32(slot + SLOT_SEQ_OFF, seq);
r.store_u32(slot + SLOT_STATUS_OFF, STATUS_SUBMITTED);
if !payload.is_empty() {
r.write_bytes(slot + SLOT_PAYLOAD_OFF, payload);
}
r.release_fence();
let new_head = seq.wrapping_add(1);
self.head[lane as usize] = new_head;
r.store_u32(self.geometry.lane_hdr_off(lane) + LANE_HEAD_OFF, new_head);
Ticket { lane, seq }
}
#[inline]
pub fn is_done<R: RegionWords>(&self, r: &R, t: Ticket) -> bool {
let tail = r.load_u32(self.geometry.lane_hdr_off(t.lane) + LANE_TAIL_OFF);
(tail.wrapping_sub(t.seq) as i32) > 0
}
#[inline]
pub fn status<R: RegionWords>(&self, r: &R, t: Ticket) -> u32 {
r.load_u32(self.geometry.slot_off(t.lane, t.seq) + SLOT_STATUS_OFF)
}
pub fn read_result<R: RegionWords>(
&self,
r: &R,
t: Ticket,
dst: &mut [u8],
) -> Result<(), GpuPeerError> {
let max = self.geometry.payload_max();
if dst.len() > max {
return Err(GpuPeerError::PayloadTooLarge { len: dst.len(), max });
}
let slot = self.geometry.slot_off(t.lane, t.seq);
r.read_bytes(slot + SLOT_PAYLOAD_OFF, dst);
Ok(())
}
pub fn reap(&mut self, t: Ticket) -> Result<(), GpuPeerError> {
if self.reaped[t.lane as usize] != t.seq {
return Err(GpuPeerError::ReapOutOfOrder {
lane: t.lane,
expected: self.reaped[t.lane as usize],
got: t.seq,
});
}
self.reaped[t.lane as usize] = t.seq.wrapping_add(1);
Ok(())
}
}
#[cfg(test)]
pub(crate) mod tests {
use super::*;
use crate::gpu_peer::layout::{OP_ADD1_F32, STATUS_DONE};
use std::cell::UnsafeCell;
pub(crate) struct MemRegion {
buf: UnsafeCell<Vec<u8>>,
g: Geometry,
}
unsafe impl Sync for MemRegion {}
impl MemRegion {
pub(crate) fn new(g: Geometry) -> Self {
Self { buf: UnsafeCell::new(vec![0u8; g.region_bytes()]), g }
}
fn ptr(&self, off: usize) -> *mut u8 {
unsafe { (*self.buf.get()).as_mut_ptr().add(off) }
}
}
impl RegionWords for MemRegion {
fn geometry(&self) -> Geometry {
self.g
}
fn load_u32(&self, off: usize) -> u32 {
unsafe { (self.ptr(off) as *const u32).read_volatile() }
}
fn store_u32(&self, off: usize, v: u32) {
unsafe { (self.ptr(off) as *mut u32).write_volatile(v) }
}
fn write_bytes(&self, off: usize, src: &[u8]) {
unsafe { core::ptr::copy_nonoverlapping(src.as_ptr(), self.ptr(off), src.len()) }
}
fn read_bytes(&self, off: usize, dst: &mut [u8]) {
unsafe { core::ptr::copy_nonoverlapping(self.ptr(off), dst.as_mut_ptr(), dst.len()) }
}
fn release_fence(&self) {
core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst);
}
}
pub(crate) fn fake_consume_all(r: &MemRegion) {
let g = r.geometry();
for lane in 0..g.lanes {
let hdr = g.lane_hdr_off(lane);
let head = r.load_u32(hdr + LANE_HEAD_OFF);
let mut tail = r.load_u32(hdr + LANE_TAIL_OFF);
while tail != head {
let slot = g.slot_off(lane, tail);
r.store_u32(slot + SLOT_STATUS_OFF, STATUS_DONE);
r.release_fence();
tail = tail.wrapping_add(1);
r.store_u32(hdr + LANE_TAIL_OFF, tail);
}
}
}
fn g4() -> Geometry {
Geometry { lanes: 2, slot_bytes: 256, slots_per_lane: 4 }
}
#[test]
fn submit_publishes_descriptor_and_head() {
let r = MemRegion::new(g4());
let mut ls = LaneSet::new(g4());
let t = ls
.try_submit(&r, OP_ADD1_F32, &[1, 2, 3, 4])
.expect("payload fits")
.expect("lane free");
assert_eq!(t, Ticket { lane: 0, seq: 0 });
let slot = g4().slot_off(0, 0);
assert_eq!(r.load_u32(slot + SLOT_OP_OFF), OP_ADD1_F32);
assert_eq!(r.load_u32(slot + SLOT_LEN_OFF), 4);
assert_eq!(r.load_u32(g4().lane_hdr_off(0) + LANE_HEAD_OFF), 1);
assert!(!ls.is_done(&r, t), "not done until the consumer moves tail");
}
#[test]
fn backpressure_gates_on_reap_not_on_consume() {
let r = MemRegion::new(g4());
let mut ls = LaneSet::new(g4());
let mut tickets = Vec::new();
for _ in 0..8 {
tickets.push(
ls.try_submit(&r, OP_ADD1_F32, &[0u8; 8]).expect("fits").expect("free"),
);
}
assert!(ls.try_submit(&r, OP_ADD1_F32, &[0u8; 8]).expect("fits").is_none());
fake_consume_all(&r);
assert!(ls.try_submit(&r, OP_ADD1_F32, &[0u8; 8]).expect("fits").is_none());
assert!(ls.is_done(&r, tickets[0]));
assert_eq!(ls.status(&r, tickets[0]), STATUS_DONE);
ls.reap(tickets[0]).expect("in order");
assert!(ls.try_submit(&r, OP_ADD1_F32, &[0u8; 8]).expect("fits").is_some());
}
#[test]
fn reap_enforces_lane_order() {
let r = MemRegion::new(g4());
let mut ls = LaneSet::new(g4());
let t0 = ls.try_submit(&r, OP_ADD1_F32, &[]).expect("fits").expect("free");
let t1 = ls.try_submit(&r, OP_ADD1_F32, &[]).expect("fits").expect("free");
let t2 = ls.try_submit(&r, OP_ADD1_F32, &[]).expect("fits").expect("free");
assert_eq!(t2.lane, t0.lane);
fake_consume_all(&r);
assert!(ls.reap(t2).is_err(), "seq 1 before seq 0 must be rejected");
ls.reap(t0).expect("oldest first");
ls.reap(t2).expect("now in order");
ls.reap(t1).expect("other lane independent");
}
#[test]
fn oversized_payload_is_rejected() {
let r = MemRegion::new(g4());
let mut ls = LaneSet::new(g4());
let big = vec![0u8; g4().payload_max() + 1];
assert!(matches!(
ls.try_submit(&r, OP_ADD1_F32, &big),
Err(GpuPeerError::PayloadTooLarge { .. })
));
}
#[test]
fn a_read_past_the_slot_is_refused_rather_than_served_from_the_next() {
let g = g4();
let max = g.payload_max();
let r = MemRegion::new(g);
let mut ls = LaneSet::new(g);
let mine = ls
.try_submit(&r, OP_ADD1_F32, &vec![0xAAu8; max])
.expect("capacity fits")
.expect("lane free");
let neighbor = ls
.try_submit_on(&r, mine.lane, OP_ADD1_F32, &vec![0xBBu8; max])
.expect("capacity fits")
.expect("lane free");
assert_eq!(neighbor.seq, mine.seq + 1, "the next slot on the same lane");
fake_consume_all(&r);
let mut exact = vec![0u8; max];
ls.read_result(&r, mine, &mut exact).expect("capacity is readable");
assert!(
exact.iter().all(|&b| b == 0xAA),
"a read of the full capacity fills the whole buffer"
);
match ls.read_result(&r, mine, &mut vec![0u8; max + 1]) {
Err(GpuPeerError::PayloadTooLarge { len, max: reported }) => {
assert_eq!(len, max + 1, "the refusal names the length it refused");
assert_eq!(reported, max, "and the capacity it measured against");
}
other => panic!("a read past the slot must be refused, got {other:?}"),
}
let mut theirs = vec![0u8; max];
ls.read_result(&r, neighbor, &mut theirs).expect("capacity is readable");
assert!(
theirs.iter().all(|&b| b == 0xBB),
"the bytes past the boundary belong to the next submission"
);
}
#[test]
fn sequence_wraparound_keeps_flowing() {
let r = MemRegion::new(g4());
let mut ls = LaneSet::new(g4());
let mut last = None;
for _ in 0..1000 {
let t = ls.try_submit(&r, OP_ADD1_F32, &[0u8; 4]).expect("fits").expect("free");
fake_consume_all(&r);
assert!(ls.is_done(&r, t));
ls.reap(t).expect("in order");
last = Some(t);
}
let last = last.expect("looped");
assert_eq!(ls.in_flight_total(), 0);
assert_eq!(last.seq, 499);
}
}