pub const MAGIC: u64 = 0x5250_5550_4759_4C46;
pub const VERSION: u32 = 1;
pub const HDR_MAGIC_OFF: usize = 0x000;
pub const HDR_VERSION_OFF: usize = 0x008;
pub const HDR_LANES_OFF: usize = 0x00C;
pub const HDR_SLOT_BYTES_OFF: usize = 0x010;
pub const HDR_SLOTS_PER_OFF: usize = 0x014;
pub const HDR_FLAGS_OFF: usize = 0x018;
pub const HDR_RTT_MIN_OFF: usize = 0x020;
pub const HDR_RTT_MED_OFF: usize = 0x028;
pub const HDR_RTT_P99_OFF: usize = 0x030;
pub const HDR_ONE_WAY_OFF: usize = 0x038;
pub const HDR_CLOCK_ERR_OFF: usize = 0x040;
pub const HDR_DELTA_OFF: usize = 0x048;
pub const HDR_LAUNCH_OFF: usize = 0x050;
pub const HDR_STOP_OFF: usize = 0x058;
pub const HDR_EXITS_OFF: usize = 0x05C;
pub const HDR_ACTIVE_GEN_OFF: usize = 0x060;
pub const HDR_CALIB_PING_OFF: usize = 0x064;
pub const HDR_CALIB_PONG_OFF: usize = 0x068;
pub const HDR_FISCHER_X_OFF: usize = 0x080;
pub const HDR_FISCHER_CS_OFF: usize = 0x0C0;
pub const HDR_FISCHER_VIOL_OFF: usize = 0x100;
pub const HDR_FISCHER_ACQS_OFF: usize = 0x140;
pub const HDR_FISCHER_STARTED_OFF: usize = 0x148;
pub const HDR_FISCHER_GPU_CONT_OFF: usize = 0x14C;
pub const HDR_STALL_COUNT_OFF: usize = 0x150;
pub const HDR_STALL_MAX_DEPTH_OFF: usize = 0x154;
pub const HDR_BARRIER_WAIT_MAX_OFF: usize = 0x158;
pub const HDR_GTS_OFF: usize = 0x180;
pub const GTS_SLOTS: usize = 400;
pub const HDR_LANE_EXITS_OFF: usize = 0xE00;
pub const HDR_LANE_GEN_OFF: usize = 0xF00;
pub const MAX_POLLER_LANES: usize = 64;
pub const HDR_BYTES: usize = 0x1000;
#[inline]
pub const fn lane_exits_off(lane: u32) -> usize {
HDR_LANE_EXITS_OFF + (lane as usize) * 4
}
#[inline]
pub const fn lane_gen_off(lane: u32) -> usize {
HDR_LANE_GEN_OFF + (lane as usize) * 4
}
pub const LANE_STRIDE: usize = 0x100;
pub const LANE_HEAD_OFF: usize = 0x00;
pub const LANE_TAIL_OFF: usize = 0x40;
pub const SLOT_OP_OFF: usize = 0x00;
pub const SLOT_LEN_OFF: usize = 0x04;
pub const SLOT_SEQ_OFF: usize = 0x08;
pub const SLOT_STATUS_OFF: usize = 0x0C;
pub const SLOT_PAYLOAD_OFF: usize = 0x10;
pub const OP_NOP: u32 = 0;
pub const OP_ADD1_F32: u32 = 1;
pub const OP_SUM_U32: u32 = 2;
pub const OP_H2V: u32 = 3;
pub const OP_V2H: u32 = 4;
pub const OP_ADD1_F32_V: u32 = 5;
pub const OP_SUM_U32_V: u32 = 6;
pub const RESIDENT_PARAMS_BYTES: usize = 8;
pub const OP_USER_BASE: u32 = 100;
pub const NO_BLOCK: u32 = u32::MAX;
pub const USER_OP_YIELD: u32 = 0x5949_4C44;
pub const OP_WAVE_CALIBRATE: u32 = 0xFFFF_FF00;
pub const STATUS_SUBMITTED: u32 = 0;
pub const STATUS_DONE: u32 = 1;
pub const STATUS_ERR: u32 = 2;
pub const STATUS_TEAM_INCOMPLETE: u32 = 3;
pub const FLAG_DOORBELL_OK: u64 = 1 << 0;
pub const FLAG_TIMED_LOCK_OK: u64 = 1 << 1;
pub const FLAG_SYS_ATOMICS_OK: u64 = 1 << 2;
pub const FLAG_CALIBRATED: u64 = 1 << 3;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Geometry {
pub lanes: u32,
pub slot_bytes: u32,
pub slots_per_lane: u32,
}
impl Geometry {
#[inline]
pub fn lane_hdr_off(&self, lane: u32) -> usize {
HDR_BYTES + lane as usize * LANE_STRIDE
}
#[inline]
pub fn slab_off(&self) -> usize {
HDR_BYTES + self.lanes as usize * LANE_STRIDE
}
#[inline]
pub fn slot_off(&self, lane: u32, seq: u32) -> usize {
self.slab_off()
+ (lane as usize * self.slots_per_lane as usize
+ (seq % self.slots_per_lane) as usize)
* self.slot_bytes as usize
}
#[inline]
pub fn payload_max(&self) -> usize {
self.slot_bytes as usize - SLOT_PAYLOAD_OFF
}
#[inline]
pub fn region_bytes(&self) -> usize {
self.slab_off()
+ self.lanes as usize * self.slots_per_lane as usize * self.slot_bytes as usize
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn status_words_match_the_kernel_defines() {
assert_eq!(STATUS_SUBMITTED, 0);
assert_eq!(STATUS_DONE, 1);
assert_eq!(STATUS_ERR, 2);
assert_eq!(STATUS_TEAM_INCOMPLETE, 3);
assert_eq!(USER_OP_YIELD, 0x5949_4C44);
assert_eq!(OP_WAVE_CALIBRATE, 0xFFFF_FF00);
#[expect(clippy::assertions_on_constants, reason = "guard over const opcodes")]
{
assert!(OP_WAVE_CALIBRATE >= OP_USER_BASE && OP_WAVE_CALIBRATE != u32::MAX);
}
let all = [STATUS_SUBMITTED, STATUS_DONE, STATUS_ERR, STATUS_TEAM_INCOMPLETE];
for (i, a) in all.iter().enumerate() {
for b in &all[i + 1..] {
assert_ne!(a, b, "two status words collide");
}
}
}
#[test]
fn offsets_match_the_kernel_defines() {
assert_eq!(HDR_STOP_OFF, 0x058);
assert_eq!(HDR_EXITS_OFF, 0x05C);
assert_eq!(HDR_ACTIVE_GEN_OFF, 0x060);
assert_eq!(HDR_CALIB_PING_OFF, 0x064);
assert_eq!(HDR_CALIB_PONG_OFF, 0x068);
assert_eq!(HDR_FISCHER_X_OFF, 0x080);
assert_eq!(HDR_FISCHER_CS_OFF, 0x0C0);
assert_eq!(HDR_FISCHER_VIOL_OFF, 0x100);
assert_eq!(HDR_FISCHER_ACQS_OFF, 0x140);
assert_eq!(HDR_FISCHER_STARTED_OFF, 0x148);
assert_eq!(HDR_FISCHER_GPU_CONT_OFF, 0x14C);
assert_eq!(HDR_GTS_OFF, 0x180);
assert_eq!(HDR_LANE_EXITS_OFF, 0xE00);
assert_eq!(HDR_LANE_GEN_OFF, 0xF00);
assert_eq!(HDR_BYTES, 0x1000);
assert_eq!(LANE_STRIDE, 0x100);
assert_eq!(LANE_TAIL_OFF, 0x40);
assert_eq!(SLOT_PAYLOAD_OFF, 0x10);
#[expect(clippy::assertions_on_constants, reason = "layout guard over const offsets")]
{
assert!(HDR_GTS_OFF + GTS_SLOTS * 8 <= HDR_LANE_EXITS_OFF);
assert!(HDR_LANE_EXITS_OFF + MAX_POLLER_LANES * 4 <= HDR_LANE_GEN_OFF);
assert!(HDR_LANE_GEN_OFF + MAX_POLLER_LANES * 4 <= HDR_BYTES);
}
}
#[test]
fn per_lane_poller_words_are_distinct_and_in_range() {
assert_eq!(lane_exits_off(0), HDR_LANE_EXITS_OFF);
assert_eq!(lane_gen_off(0), HDR_LANE_GEN_OFF);
assert_eq!(lane_exits_off(3), HDR_LANE_EXITS_OFF + 12);
assert_eq!(lane_gen_off(3), HDR_LANE_GEN_OFF + 12);
let last = (MAX_POLLER_LANES - 1) as u32;
assert!(lane_exits_off(last) + 4 <= HDR_LANE_GEN_OFF);
assert!(lane_gen_off(last) + 4 <= HDR_BYTES);
}
#[test]
fn geometry_math_is_consistent() {
let g = Geometry { lanes: 4, slot_bytes: 4096, slots_per_lane: 64 };
assert_eq!(g.lane_hdr_off(0), HDR_BYTES);
assert_eq!(g.lane_hdr_off(3), HDR_BYTES + 3 * LANE_STRIDE);
assert_eq!(g.slab_off(), HDR_BYTES + 4 * LANE_STRIDE);
assert_eq!(g.slot_off(1, 64), g.slot_off(1, 0));
assert_ne!(g.slot_off(1, 1), g.slot_off(1, 0));
assert_eq!(g.payload_max(), 4096 - 16);
assert_eq!(
g.region_bytes(),
g.slab_off() + 4usize * 64 * 4096
);
}
}