1use core::ffi::c_void;
2
3use log;
8
9use super::{
10 Access, AnyBuffer, AnyBufferRange, BufferRange, DefaultExecutor, Error, Executor, WorkItem,
11};
12use oneliner_iree_abi::{
13 iree_hal_executable_library_v0_t, IREE_HAL_EXECUTABLE_LIBRARY_VERSION_LATEST,
14};
15use portable_atomic::{AtomicI32, Ordering};
16
17pub use oneliner_iree_abi::{
18 iree_hal_executable_dispatch_state_v0_t, iree_hal_executable_environment_v0_t,
19 iree_hal_executable_import_thunk_v0_t, iree_hal_executable_import_v0_t,
20 iree_hal_executable_library_header_t, iree_hal_executable_library_query_fn_t,
21 iree_hal_executable_workgroup_state_v0_t, iree_hal_processor_v0_t, DispatchFn,
22};
23
24pub const MAX_BINDINGS: usize = 32;
25
26pub unsafe fn dispatch_fn_from_library(
33 query: iree_hal_executable_library_query_fn_t,
34 ordinal: usize,
35) -> Result<DispatchFn, Error> {
36 let environment = empty_environment();
37 let library = unsafe { query(IREE_HAL_EXECUTABLE_LIBRARY_VERSION_LATEST, &environment) }
38 as *const iree_hal_executable_library_v0_t;
39 if library.is_null() {
40 return Err(Error::MissingDispatchFunction { ordinal });
41 }
42
43 let exports = unsafe { &(*library).exports };
44 if ordinal >= exports.count as usize || exports.ptrs.is_null() {
45 return Err(Error::MissingDispatchFunction { ordinal });
46 }
47
48 let function = unsafe { *exports.ptrs.add(ordinal) };
49 log::trace!(
50 "Resolved dispatch function for ordinal {} at address {:#x}",
51 ordinal,
52 function as usize
53 );
54 Ok(function)
55}
56
57pub unsafe fn dispatch(
64 function: DispatchFn,
65 params: &[u32],
66 workload: &[u32],
67 ranges: &[AnyBufferRange],
68) {
69 unsafe { try_dispatch(function, params, workload, ranges) }.expect("backend dispatch failed");
70}
71
72pub unsafe fn try_dispatch(
79 function: DispatchFn,
80 params: &[u32],
81 workload: &[u32],
82 ranges: &[AnyBufferRange],
83) -> Result<(), Error> {
84 let mut executor = DefaultExecutor::default();
85 unsafe { try_dispatch_with_executor(&mut executor, function, params, workload, ranges) }
86}
87
88pub unsafe fn try_dispatch_with_executor<E>(
96 executor: &mut E,
97 function: DispatchFn,
98 params: &[u32],
99 workload: &[u32],
100 ranges: &[AnyBufferRange],
101) -> Result<(), Error>
102where
103 E: Executor,
104{
105 if ranges.len() > MAX_BINDINGS {
106 return Err(Error::TooManyBindings {
107 provided: ranges.len(),
108 capacity: MAX_BINDINGS,
109 });
110 }
111 let constant_count = u16::try_from(params.len()).map_err(|_| Error::TooManyConstants {
112 provided: params.len(),
113 capacity: u16::MAX as usize,
114 })?;
115 let workload_z = u16::try_from(workload.get(2).copied().unwrap_or(1)).map_err(|_| {
116 Error::WorkgroupCountTooLarge {
117 dimension: 'z',
118 value: workload[2],
119 }
120 })?;
121
122 let mut binding_ptrs = [core::ptr::null_mut(); MAX_BINDINGS];
123 let mut binding_lengths = [0usize; MAX_BINDINGS];
124 for (index, range) in ranges.iter().enumerate() {
125 match range.buffer {
126 AnyBuffer::Ref(buffer) => {
127 binding_ptrs[index] = unsafe { buffer.ptr.add(range.offset) as *mut c_void };
128 binding_lengths[index] = range.length;
129 }
130 AnyBuffer::Mut(buffer) => {
131 binding_ptrs[index] = unsafe { buffer.ptr.add(range.offset) as *mut c_void };
132 binding_lengths[index] = range.length;
133 }
134 }
135 log::trace!(
136 "Binding {}: ptr = {:#x}, length = {}, align16 = {}",
137 index,
138 binding_ptrs[index] as usize,
139 binding_lengths[index],
140 (binding_ptrs[index] as usize) % 16
141 );
142 }
143
144 let environment = empty_environment();
145 let workload_x = workload.first().copied().unwrap_or(1);
146 let workload_y = workload.get(1).copied().unwrap_or(1);
147 let dispatch_state = iree_hal_executable_dispatch_state_v0_t {
148 workgroup_size_x: 1,
149 workgroup_size_y: 1,
150 workgroup_size_z: 1,
151 constant_count,
152 workgroup_count_x: workload_x,
153 workgroup_count_y: workload_y,
154 workgroup_count_z: workload_z,
155 max_concurrency: 1,
156 binding_count: ranges.len() as u8,
157 constants: params.as_ptr(),
158 binding_ptrs: binding_ptrs.as_ptr(),
159 binding_lengths: binding_lengths.as_ptr(),
160 };
161 let dispatch_status = AtomicI32::new(0);
162
163 for z in 0..workload_z {
164 for y in 0..workload_y {
165 for x in 0..workload_x {
166 log::trace!("Dispatching workgroup (x={}, y={}, z={})", x, y, z);
167 let work_item = unsafe {
168 WorkItem::iree(
169 function,
170 core::ptr::addr_of!(environment)
171 as *mut iree_hal_executable_environment_v0_t,
172 core::ptr::addr_of!(dispatch_state)
173 as *mut iree_hal_executable_dispatch_state_v0_t,
174 iree_hal_executable_workgroup_state_v0_t {
175 workgroup_id_x: x,
176 workgroup_id_y: y,
177 workgroup_id_z: z,
178 reserved: 0,
179 processor_id: 0,
180 local_memory: core::ptr::null_mut(),
181 local_memory_size: 0,
182 },
183 core::ptr::addr_of!(dispatch_status),
184 )
185 };
186 executor.schedule(work_item);
187 }
188 }
189 }
190 executor.wait_job_completion();
191 let status = dispatch_status.load(Ordering::Acquire);
192 if status != 0 {
193 return Err(Error::DispatchFailed { status });
194 }
195 Ok(())
196}
197
198fn empty_environment() -> iree_hal_executable_environment_v0_t {
199 iree_hal_executable_environment_v0_t {
200 constants: core::ptr::null(),
201 import_thunk: None,
202 import_funcs: core::ptr::null(),
203 import_contexts: core::ptr::null(),
204 processor: iree_hal_processor_v0_t { data: [0; 8] },
205 }
206}