Skip to main content

oneliner_runtime/
iree.rs

1use core::ffi::c_void;
2
3// TODO: logging switching.
4// #[cfg(feature = "ariel-os")]
5// use ariel_os::log;
6// #[cfg(not(feature = "ariel-os"))]
7use log;
8
9use super::{
10    Access, AnyBuffer, AnyBufferRange, BufferRange, DefaultExecutor, Error, Executor, WorkItem,
11};
12use oneliner_iree_abi::{
13    iree_hal_executable_library_v0_t, IREE_HAL_EXECUTABLE_LIBRARY_VERSION_LATEST,
14};
15use portable_atomic::{AtomicI32, Ordering};
16
17pub use oneliner_iree_abi::{
18    iree_hal_executable_dispatch_state_v0_t, iree_hal_executable_environment_v0_t,
19    iree_hal_executable_import_thunk_v0_t, iree_hal_executable_import_v0_t,
20    iree_hal_executable_library_header_t, iree_hal_executable_library_query_fn_t,
21    iree_hal_executable_workgroup_state_v0_t, iree_hal_processor_v0_t, DispatchFn,
22};
23
24pub const MAX_BINDINGS: usize = 32;
25
26/// Resolves an export ordinal from an IREE static library.
27///
28/// # Safety
29///
30/// `query` must return a valid IREE v0 executable library for the duration of
31/// the call, including a valid export pointer table.
32pub unsafe fn dispatch_fn_from_library(
33    query: iree_hal_executable_library_query_fn_t,
34    ordinal: usize,
35) -> Result<DispatchFn, Error> {
36    let environment = empty_environment();
37    let library = unsafe { query(IREE_HAL_EXECUTABLE_LIBRARY_VERSION_LATEST, &environment) }
38        as *const iree_hal_executable_library_v0_t;
39    if library.is_null() {
40        return Err(Error::MissingDispatchFunction { ordinal });
41    }
42
43    let exports = unsafe { &(*library).exports };
44    if ordinal >= exports.count as usize || exports.ptrs.is_null() {
45        return Err(Error::MissingDispatchFunction { ordinal });
46    }
47
48    let function = unsafe { *exports.ptrs.add(ordinal) };
49    log::trace!(
50        "Resolved dispatch function for ordinal {} at address {:#x}",
51        ordinal,
52        function as usize
53    );
54    Ok(function)
55}
56
57/// Dispatches an IREE workload and panics on failure.
58///
59/// # Safety
60///
61/// `function` and every buffer range must satisfy the same requirements as
62/// [`try_dispatch`].
63pub unsafe fn dispatch(
64    function: DispatchFn,
65    params: &[u32],
66    workload: &[u32],
67    ranges: &[AnyBufferRange],
68) {
69    unsafe { try_dispatch(function, params, workload, ranges) }.expect("backend dispatch failed");
70}
71
72/// Dispatches an IREE workload with the default executor.
73///
74/// # Safety
75///
76/// `function` must be a valid IREE dispatch function. Every buffer pointer must
77/// remain valid for its declared range until all scheduled work completes.
78pub unsafe fn try_dispatch(
79    function: DispatchFn,
80    params: &[u32],
81    workload: &[u32],
82    ranges: &[AnyBufferRange],
83) -> Result<(), Error> {
84    let mut executor = DefaultExecutor::default();
85    unsafe { try_dispatch_with_executor(&mut executor, function, params, workload, ranges) }
86}
87
88/// Dispatches an IREE workload through a caller-provided executor.
89///
90/// # Safety
91///
92/// `function` must be a valid IREE dispatch function. Every buffer pointer must
93/// remain valid for its declared range until the executor has completed all
94/// submitted work.
95pub unsafe fn try_dispatch_with_executor<E>(
96    executor: &mut E,
97    function: DispatchFn,
98    params: &[u32],
99    workload: &[u32],
100    ranges: &[AnyBufferRange],
101) -> Result<(), Error>
102where
103    E: Executor,
104{
105    if ranges.len() > MAX_BINDINGS {
106        return Err(Error::TooManyBindings {
107            provided: ranges.len(),
108            capacity: MAX_BINDINGS,
109        });
110    }
111    let constant_count = u16::try_from(params.len()).map_err(|_| Error::TooManyConstants {
112        provided: params.len(),
113        capacity: u16::MAX as usize,
114    })?;
115    let workload_z = u16::try_from(workload.get(2).copied().unwrap_or(1)).map_err(|_| {
116        Error::WorkgroupCountTooLarge {
117            dimension: 'z',
118            value: workload[2],
119        }
120    })?;
121
122    let mut binding_ptrs = [core::ptr::null_mut(); MAX_BINDINGS];
123    let mut binding_lengths = [0usize; MAX_BINDINGS];
124    for (index, range) in ranges.iter().enumerate() {
125        match range.buffer {
126            AnyBuffer::Ref(buffer) => {
127                binding_ptrs[index] = unsafe { buffer.ptr.add(range.offset) as *mut c_void };
128                binding_lengths[index] = range.length;
129            }
130            AnyBuffer::Mut(buffer) => {
131                binding_ptrs[index] = unsafe { buffer.ptr.add(range.offset) as *mut c_void };
132                binding_lengths[index] = range.length;
133            }
134        }
135        log::trace!(
136            "Binding {}: ptr = {:#x}, length = {}, align16 = {}",
137            index,
138            binding_ptrs[index] as usize,
139            binding_lengths[index],
140            (binding_ptrs[index] as usize) % 16
141        );
142    }
143
144    let environment = empty_environment();
145    let workload_x = workload.first().copied().unwrap_or(1);
146    let workload_y = workload.get(1).copied().unwrap_or(1);
147    let dispatch_state = iree_hal_executable_dispatch_state_v0_t {
148        workgroup_size_x: 1,
149        workgroup_size_y: 1,
150        workgroup_size_z: 1,
151        constant_count,
152        workgroup_count_x: workload_x,
153        workgroup_count_y: workload_y,
154        workgroup_count_z: workload_z,
155        max_concurrency: 1,
156        binding_count: ranges.len() as u8,
157        constants: params.as_ptr(),
158        binding_ptrs: binding_ptrs.as_ptr(),
159        binding_lengths: binding_lengths.as_ptr(),
160    };
161    let dispatch_status = AtomicI32::new(0);
162
163    for z in 0..workload_z {
164        for y in 0..workload_y {
165            for x in 0..workload_x {
166                log::trace!("Dispatching workgroup (x={}, y={}, z={})", x, y, z);
167                let work_item = unsafe {
168                    WorkItem::iree(
169                        function,
170                        core::ptr::addr_of!(environment)
171                            as *mut iree_hal_executable_environment_v0_t,
172                        core::ptr::addr_of!(dispatch_state)
173                            as *mut iree_hal_executable_dispatch_state_v0_t,
174                        iree_hal_executable_workgroup_state_v0_t {
175                            workgroup_id_x: x,
176                            workgroup_id_y: y,
177                            workgroup_id_z: z,
178                            reserved: 0,
179                            processor_id: 0,
180                            local_memory: core::ptr::null_mut(),
181                            local_memory_size: 0,
182                        },
183                        core::ptr::addr_of!(dispatch_status),
184                    )
185                };
186                executor.schedule(work_item);
187            }
188        }
189    }
190    executor.wait_job_completion();
191    let status = dispatch_status.load(Ordering::Acquire);
192    if status != 0 {
193        return Err(Error::DispatchFailed { status });
194    }
195    Ok(())
196}
197
198fn empty_environment() -> iree_hal_executable_environment_v0_t {
199    iree_hal_executable_environment_v0_t {
200        constants: core::ptr::null(),
201        import_thunk: None,
202        import_funcs: core::ptr::null(),
203        import_contexts: core::ptr::null(),
204        processor: iree_hal_processor_v0_t { data: [0; 8] },
205    }
206}