use crate::error::{LevelZeroError, LevelZeroResult};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct GroupCount {
pub x: u32,
pub y: u32,
pub z: u32,
}
impl GroupCount {
#[must_use]
pub fn total(&self) -> u64 {
u64::from(self.x) * u64::from(self.y) * u64::from(self.z)
}
}
#[derive(Debug, Clone, Copy)]
pub struct GroupCountPlanner {
local_x: u32,
local_y: u32,
local_z: u32,
}
impl GroupCountPlanner {
pub fn new_1d(local_x: u32) -> LevelZeroResult<Self> {
Self::new_3d(local_x, 1, 1)
}
pub fn new_3d(local_x: u32, local_y: u32, local_z: u32) -> LevelZeroResult<Self> {
if local_x == 0 || local_y == 0 || local_z == 0 {
return Err(LevelZeroError::InvalidArgument(
"workgroup size must be non-zero in every dimension".into(),
));
}
Ok(Self {
local_x,
local_y,
local_z,
})
}
#[must_use]
pub fn local_size(&self) -> (u32, u32, u32) {
(self.local_x, self.local_y, self.local_z)
}
pub fn plan_1d(&self, global_x: u64) -> LevelZeroResult<GroupCount> {
self.plan_3d(global_x, 1, 1)
}
pub fn plan_3d(
&self,
global_x: u64,
global_y: u64,
global_z: u64,
) -> LevelZeroResult<GroupCount> {
let x = ceil_div_u32(global_x, self.local_x)?;
let y = ceil_div_u32(global_y, self.local_y)?;
let z = ceil_div_u32(global_z, self.local_z)?;
Ok(GroupCount { x, y, z })
}
}
fn ceil_div_u32(num: u64, den: u32) -> LevelZeroResult<u32> {
if num == 0 {
return Ok(0);
}
let groups = num.div_ceil(u64::from(den));
u32::try_from(groups).map_err(|_| {
LevelZeroError::InvalidArgument(format!("group count {groups} exceeds u32 dispatch limit"))
})
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CommandQueueGroupInfo {
pub ordinal: u32,
pub compute: bool,
pub copy: bool,
pub num_queues: u32,
}
#[derive(Debug, Clone, Default)]
pub struct CommandQueueGroupTable {
groups: Vec<CommandQueueGroupInfo>,
}
impl CommandQueueGroupTable {
#[must_use]
pub fn new(groups: Vec<CommandQueueGroupInfo>) -> Self {
Self { groups }
}
#[must_use]
pub fn groups(&self) -> &[CommandQueueGroupInfo] {
&self.groups
}
pub fn select_compute_ordinal(&self) -> LevelZeroResult<u32> {
self.groups
.iter()
.filter(|g| g.compute)
.min_by_key(|g| g.ordinal)
.map(|g| g.ordinal)
.ok_or_else(|| {
LevelZeroError::Unsupported("device exposes no compute queue group".into())
})
}
pub fn select_copy_ordinal(&self) -> LevelZeroResult<u32> {
if let Some(g) = self
.groups
.iter()
.filter(|g| g.copy && !g.compute)
.min_by_key(|g| g.ordinal)
{
return Ok(g.ordinal);
}
self.groups
.iter()
.filter(|g| g.copy)
.min_by_key(|g| g.ordinal)
.map(|g| g.ordinal)
.ok_or_else(|| LevelZeroError::Unsupported("device exposes no copy queue group".into()))
}
#[must_use]
pub fn has_dedicated_copy_engine(&self) -> bool {
self.groups.iter().any(|g| g.copy && !g.compute)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EventScope {
Device,
HostVisible,
KernelTimestamp,
}
impl EventScope {
#[must_use]
pub fn ze_flags(self) -> u32 {
match self {
EventScope::Device => 0,
EventScope::HostVisible => 0x1,
EventScope::KernelTimestamp => 0x4,
}
}
}
#[derive(Debug, Clone)]
pub struct EventPoolDesc {
pub capacity: u32,
pub scope: EventScope,
next_index: u32,
}
impl EventPoolDesc {
pub fn new(capacity: u32, scope: EventScope) -> LevelZeroResult<Self> {
if capacity == 0 {
return Err(LevelZeroError::InvalidArgument(
"event pool capacity must be > 0".into(),
));
}
Ok(Self {
capacity,
scope,
next_index: 0,
})
}
pub fn alloc_event(&mut self) -> LevelZeroResult<u32> {
if self.next_index >= self.capacity {
return Err(LevelZeroError::OutOfMemory);
}
let idx = self.next_index;
self.next_index += 1;
Ok(idx)
}
#[must_use]
pub fn remaining(&self) -> u32 {
self.capacity - self.next_index
}
#[must_use]
pub fn is_host_visible(&self) -> bool {
matches!(self.scope, EventScope::HostVisible)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Barrier {
pub signal_event: Option<u32>,
pub wait_events: Vec<u32>,
}
impl Barrier {
#[must_use]
pub fn global() -> Self {
Self {
signal_event: None,
wait_events: Vec::new(),
}
}
#[must_use]
pub fn signalling(event: u32) -> Self {
Self {
signal_event: Some(event),
wait_events: Vec::new(),
}
}
#[must_use]
pub fn waiting(events: Vec<u32>) -> Self {
Self {
signal_event: None,
wait_events: events,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RecordedCommand {
MemoryCopy {
dst: u64,
src: u64,
bytes: u64,
},
MemoryFill {
dst: u64,
bytes: u64,
pattern_len: u32,
},
LaunchKernel {
kernel: String,
groups: GroupCount,
signal_event: Option<u32>,
},
Barrier(Barrier),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CommandListState {
Recording,
Closed,
}
#[derive(Debug, Clone)]
pub struct CommandListRecorder {
ordinal: u32,
commands: Vec<RecordedCommand>,
state: CommandListState,
}
impl CommandListRecorder {
#[must_use]
pub fn new(ordinal: u32) -> Self {
Self {
ordinal,
commands: Vec::new(),
state: CommandListState::Recording,
}
}
#[must_use]
pub fn ordinal(&self) -> u32 {
self.ordinal
}
#[must_use]
pub fn state(&self) -> CommandListState {
self.state
}
#[must_use]
pub fn len(&self) -> usize {
self.commands.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.commands.is_empty()
}
#[must_use]
pub fn commands(&self) -> &[RecordedCommand] {
&self.commands
}
fn ensure_recording(&self) -> LevelZeroResult<()> {
if self.state != CommandListState::Recording {
return Err(LevelZeroError::CommandListError(
"command list is closed; reset before appending".into(),
));
}
Ok(())
}
pub fn append_memory_copy(&mut self, dst: u64, src: u64, bytes: u64) -> LevelZeroResult<()> {
self.ensure_recording()?;
if bytes == 0 {
return Err(LevelZeroError::InvalidArgument(
"memory copy size must be > 0".into(),
));
}
self.commands
.push(RecordedCommand::MemoryCopy { dst, src, bytes });
Ok(())
}
pub fn append_memory_fill(
&mut self,
dst: u64,
bytes: u64,
pattern_len: u32,
) -> LevelZeroResult<()> {
self.ensure_recording()?;
if pattern_len == 0 {
return Err(LevelZeroError::InvalidArgument(
"fill pattern length must be > 0".into(),
));
}
if bytes % u64::from(pattern_len) != 0 {
return Err(LevelZeroError::InvalidArgument(
"fill size must be a multiple of the pattern length".into(),
));
}
self.commands.push(RecordedCommand::MemoryFill {
dst,
bytes,
pattern_len,
});
Ok(())
}
pub fn append_launch_kernel(
&mut self,
kernel: impl Into<String>,
groups: GroupCount,
signal_event: Option<u32>,
) -> LevelZeroResult<()> {
self.ensure_recording()?;
if groups.total() == 0 {
return Err(LevelZeroError::InvalidArgument(
"kernel launch grid must contain at least one workgroup".into(),
));
}
self.commands.push(RecordedCommand::LaunchKernel {
kernel: kernel.into(),
groups,
signal_event,
});
Ok(())
}
pub fn append_barrier(&mut self, barrier: Barrier) -> LevelZeroResult<()> {
self.ensure_recording()?;
self.commands.push(RecordedCommand::Barrier(barrier));
Ok(())
}
pub fn close(&mut self) -> LevelZeroResult<()> {
self.ensure_recording()?;
self.state = CommandListState::Closed;
Ok(())
}
pub fn reset(&mut self) {
self.commands.clear();
self.state = CommandListState::Recording;
}
#[must_use]
pub fn launch_count(&self) -> usize {
self.commands
.iter()
.filter(|c| matches!(c, RecordedCommand::LaunchKernel { .. }))
.count()
}
}
#[derive(Debug, Clone)]
pub struct ReusableCommandList {
inner: CommandListRecorder,
reuse_count: u64,
}
impl ReusableCommandList {
#[must_use]
pub fn new(ordinal: u32) -> Self {
Self {
inner: CommandListRecorder::new(ordinal),
reuse_count: 0,
}
}
pub fn recorder_mut(&mut self) -> &mut CommandListRecorder {
&mut self.inner
}
#[must_use]
pub fn recorder(&self) -> &CommandListRecorder {
&self.inner
}
#[must_use]
pub fn reuse_count(&self) -> u64 {
self.reuse_count
}
pub fn recycle(&mut self) -> LevelZeroResult<()> {
if self.inner.state() != CommandListState::Closed {
return Err(LevelZeroError::CommandListError(
"close the command list before recycling it".into(),
));
}
self.inner.reset();
self.reuse_count += 1;
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn group_count_total() {
let g = GroupCount { x: 4, y: 2, z: 3 };
assert_eq!(g.total(), 24);
}
#[test]
fn planner_1d_ceil_div() {
let p = GroupCountPlanner::new_1d(256).unwrap();
assert_eq!(p.local_size(), (256, 1, 1));
assert_eq!(p.plan_1d(1000).unwrap(), GroupCount { x: 4, y: 1, z: 1 });
assert_eq!(p.plan_1d(512).unwrap(), GroupCount { x: 2, y: 1, z: 1 });
assert_eq!(p.plan_1d(0).unwrap(), GroupCount { x: 0, y: 1, z: 1 });
}
#[test]
fn planner_3d_per_dimension() {
let p = GroupCountPlanner::new_3d(16, 16, 1).unwrap();
let g = p.plan_3d(100, 50, 8).unwrap();
assert_eq!(g.x, 7); assert_eq!(g.y, 4); assert_eq!(g.z, 8); }
#[test]
fn planner_rejects_zero_local_and_overflow() {
assert!(GroupCountPlanner::new_3d(0, 1, 1).is_err());
let p = GroupCountPlanner::new_1d(1).unwrap();
assert!(matches!(
p.plan_1d(1u64 << 33),
Err(LevelZeroError::InvalidArgument(_))
));
}
#[test]
fn queue_group_selection() {
let table = CommandQueueGroupTable::new(vec![
CommandQueueGroupInfo {
ordinal: 0,
compute: true,
copy: true,
num_queues: 4,
},
CommandQueueGroupInfo {
ordinal: 1,
compute: false,
copy: true,
num_queues: 1,
},
]);
assert_eq!(table.select_compute_ordinal().unwrap(), 0);
assert_eq!(table.select_copy_ordinal().unwrap(), 1);
assert!(table.has_dedicated_copy_engine());
assert_eq!(table.groups().len(), 2);
}
#[test]
fn queue_group_fallback_to_shared_copy() {
let table = CommandQueueGroupTable::new(vec![CommandQueueGroupInfo {
ordinal: 0,
compute: true,
copy: true,
num_queues: 1,
}]);
assert_eq!(table.select_copy_ordinal().unwrap(), 0);
assert!(!table.has_dedicated_copy_engine());
}
#[test]
fn queue_group_errors_when_missing_capability() {
let copy_only = CommandQueueGroupTable::new(vec![CommandQueueGroupInfo {
ordinal: 0,
compute: false,
copy: true,
num_queues: 1,
}]);
assert!(matches!(
copy_only.select_compute_ordinal(),
Err(LevelZeroError::Unsupported(_))
));
let compute_only = CommandQueueGroupTable::new(vec![CommandQueueGroupInfo {
ordinal: 0,
compute: true,
copy: false,
num_queues: 1,
}]);
assert!(matches!(
compute_only.select_copy_ordinal(),
Err(LevelZeroError::Unsupported(_))
));
}
#[test]
fn event_pool_allocation() {
let mut pool = EventPoolDesc::new(2, EventScope::HostVisible).unwrap();
assert!(pool.is_host_visible());
assert_eq!(pool.scope.ze_flags(), 0x1);
assert_eq!(pool.remaining(), 2);
assert_eq!(pool.alloc_event().unwrap(), 0);
assert_eq!(pool.alloc_event().unwrap(), 1);
assert_eq!(pool.remaining(), 0);
assert!(matches!(
pool.alloc_event(),
Err(LevelZeroError::OutOfMemory)
));
}
#[test]
fn event_pool_rejects_zero_capacity() {
assert!(EventPoolDesc::new(0, EventScope::Device).is_err());
assert_eq!(EventScope::KernelTimestamp.ze_flags(), 0x4);
assert_eq!(EventScope::HostVisible.ze_flags(), 0x1);
assert_eq!(EventScope::Device.ze_flags(), 0);
}
#[test]
fn barrier_constructors() {
let g = Barrier::global();
assert!(g.signal_event.is_none());
assert!(g.wait_events.is_empty());
let s = Barrier::signalling(3);
assert_eq!(s.signal_event, Some(3));
let w = Barrier::waiting(vec![1, 2]);
assert_eq!(w.wait_events, vec![1, 2]);
assert!(w.signal_event.is_none());
}
#[test]
fn recorder_records_in_order() {
let mut rec = CommandListRecorder::new(0);
assert_eq!(rec.ordinal(), 0);
assert!(rec.is_empty());
assert_eq!(rec.state(), CommandListState::Recording);
rec.append_memory_copy(2, 1, 256).unwrap();
rec.append_launch_kernel("gemm_f32", GroupCount { x: 4, y: 1, z: 1 }, Some(0))
.unwrap();
rec.append_barrier(Barrier::global()).unwrap();
assert_eq!(rec.len(), 3);
assert_eq!(rec.launch_count(), 1);
match &rec.commands()[0] {
RecordedCommand::MemoryCopy { dst, src, bytes } => {
assert_eq!((*dst, *src, *bytes), (2, 1, 256));
}
other => panic!("expected MemoryCopy, got {other:?}"),
}
}
#[test]
fn recorder_validates_args() {
let mut rec = CommandListRecorder::new(0);
assert!(rec.append_memory_copy(1, 0, 0).is_err());
assert!(rec.append_memory_fill(1, 10, 0).is_err());
assert!(
rec.append_memory_fill(1, 10, 4).is_err(),
"10 not a multiple of 4"
);
rec.append_memory_fill(1, 12, 4).unwrap();
assert!(
rec.append_launch_kernel("k", GroupCount { x: 0, y: 1, z: 1 }, None)
.is_err()
);
}
#[test]
fn recorder_close_blocks_append_until_reset() {
let mut rec = CommandListRecorder::new(0);
rec.append_memory_copy(2, 1, 128).unwrap();
rec.close().unwrap();
assert_eq!(rec.state(), CommandListState::Closed);
assert!(matches!(
rec.append_memory_copy(3, 1, 64),
Err(LevelZeroError::CommandListError(_))
));
assert!(rec.close().is_err());
rec.reset();
assert_eq!(rec.state(), CommandListState::Recording);
assert!(rec.is_empty());
rec.append_memory_copy(3, 1, 64).unwrap();
}
#[test]
fn reusable_command_list_recycle_loop() {
let mut list = ReusableCommandList::new(0);
assert_eq!(list.reuse_count(), 0);
for expected in 0..3u64 {
assert_eq!(list.reuse_count(), expected);
let rec = list.recorder_mut();
rec.append_launch_kernel("k", GroupCount { x: 1, y: 1, z: 1 }, None)
.unwrap();
rec.close().unwrap();
list.recycle().unwrap();
}
assert_eq!(list.reuse_count(), 3);
assert!(list.recorder().is_empty());
assert_eq!(list.recorder().state(), CommandListState::Recording);
}
#[test]
fn reusable_rejects_recycle_while_open() {
let mut list = ReusableCommandList::new(0);
list.recorder_mut()
.append_launch_kernel("k", GroupCount { x: 1, y: 1, z: 1 }, None)
.unwrap();
assert!(matches!(
list.recycle(),
Err(LevelZeroError::CommandListError(_))
));
assert_eq!(list.reuse_count(), 0);
}
}