1#![cfg_attr(target_os = "none", no_std)]
2#![doc = include_str!("../README.md")]
3
4extern crate alloc;
5
6use core::{num::NonZeroUsize, ptr::NonNull};
7
8mod op;
9
10mod array;
11mod common;
12mod dbox;
13mod def;
14mod pool;
15mod streaming;
16
17pub use array::*;
18pub use dbox::*;
19pub use def::*;
20pub use op::DmaOp;
21pub use pool::*;
22pub use streaming::*;
23
24#[derive(Clone)]
25pub struct DeviceDma {
26 op: &'static dyn DmaOp,
27 constraints: DmaConstraints,
28}
29
30impl DeviceDma {
31 pub fn new(dma_mask: u64, op: &'static dyn DmaOp) -> Self {
32 Self {
33 constraints: DmaConstraints::new(dma_mask),
34 op,
35 }
36 }
37
38 pub fn with_constraints(&self, constraints: DmaConstraints) -> Self {
39 Self {
40 op: self.op,
41 constraints,
42 }
43 }
44
45 pub fn constraints(&self) -> DmaConstraints {
46 self.constraints
47 }
48
49 pub fn dma_mask(&self) -> u64 {
50 self.constraints.addr_mask
51 }
52
53 pub fn flush(&self, addr: NonNull<u8>, size: usize) {
54 self.op.flush(addr, size)
55 }
56
57 pub fn invalidate(&self, addr: NonNull<u8>, size: usize) {
58 self.op.invalidate(addr, size)
59 }
60
61 pub fn flush_invalidate(&self, addr: NonNull<u8>, size: usize) {
62 self.op.flush_invalidate(addr, size)
63 }
64
65 pub fn page_size(&self) -> usize {
66 self.op.page_size()
67 }
68
69 pub(crate) unsafe fn alloc_contiguous(
70 &self,
71 layout: core::alloc::Layout,
72 ) -> Result<DmaAllocHandle, DmaError> {
73 let constraints = self.constraints.with_align(layout.align());
74 let res =
75 unsafe { self.op.alloc_contiguous(constraints, layout) }.ok_or(DmaError::NoMemory)?;
76 match self.check_alloc_handle(&res, constraints) {
77 Ok(()) => Ok(res),
78 Err(e) => {
79 unsafe { self.op.dealloc_contiguous(res) };
80 Err(e)
81 }
82 }
83 }
84
85 pub(crate) unsafe fn dealloc_contiguous(&self, handle: DmaAllocHandle) {
86 unsafe { self.op.dealloc_contiguous(handle) }
87 }
88
89 pub(crate) unsafe fn alloc_coherent(
90 &self,
91 layout: core::alloc::Layout,
92 ) -> Result<DmaAllocHandle, DmaError> {
93 let constraints = self.constraints.with_align(layout.align());
94 let res =
95 unsafe { self.op.alloc_coherent(constraints, layout) }.ok_or(DmaError::NoMemory)?;
96 match self.check_alloc_handle(&res, constraints) {
97 Ok(()) => Ok(res),
98 Err(e) => {
99 unsafe { self.op.dealloc_coherent(res) };
100 Err(e)
101 }
102 }
103 }
104
105 pub(crate) unsafe fn dealloc_coherent(&self, handle: DmaAllocHandle) {
106 unsafe { self.op.dealloc_coherent(handle) }
107 }
108
109 pub(crate) unsafe fn map_streaming(
110 &self,
111 addr: NonNull<u8>,
112 size: NonZeroUsize,
113 align: usize,
114 direction: DmaDirection,
115 ) -> Result<DmaMapHandle, DmaError> {
116 let constraints = self.constraints.with_align(align);
117 let res = unsafe { self.op.map_streaming(constraints, addr, size, direction) }?;
118 match self.check_map_handle(&res, constraints) {
119 Ok(()) => Ok(res),
120 Err(e) => {
121 unsafe { self.op.unmap_streaming(res) };
122 Err(e)
123 }
124 }
125 }
126
127 pub(crate) unsafe fn unmap_streaming(&self, handle: DmaMapHandle) {
128 unsafe { self.op.unmap_streaming(handle) }
129 }
130
131 pub(crate) fn sync_alloc_for_device(
132 &self,
133 handle: &DmaAllocHandle,
134 offset: usize,
135 size: usize,
136 direction: DmaDirection,
137 ) {
138 self.op
139 .sync_alloc_for_device(handle, offset, size, direction);
140 }
141
142 pub(crate) fn sync_alloc_for_cpu(
143 &self,
144 handle: &DmaAllocHandle,
145 offset: usize,
146 size: usize,
147 direction: DmaDirection,
148 ) {
149 self.op.sync_alloc_for_cpu(handle, offset, size, direction);
150 }
151
152 pub(crate) fn sync_map_for_device(
153 &self,
154 handle: &DmaMapHandle,
155 offset: usize,
156 size: usize,
157 direction: DmaDirection,
158 ) {
159 self.op.sync_map_for_device(handle, offset, size, direction);
160 }
161
162 pub(crate) fn sync_map_for_cpu(
163 &self,
164 handle: &DmaMapHandle,
165 offset: usize,
166 size: usize,
167 direction: DmaDirection,
168 ) {
169 self.op.sync_map_for_cpu(handle, offset, size, direction);
170 }
171
172 pub fn coherent_array_zero<T: DmaPod>(&self, len: usize) -> Result<CoherentArray<T>, DmaError> {
173 CoherentArray::new_zero(self, len)
174 }
175
176 pub fn coherent_array_zero_with_align<T: DmaPod>(
177 &self,
178 len: usize,
179 align: usize,
180 ) -> Result<CoherentArray<T>, DmaError> {
181 CoherentArray::new_zero_with_align(self, len, align)
182 }
183
184 pub fn contiguous_array_zero<T: DmaPod>(
185 &self,
186 len: usize,
187 direction: DmaDirection,
188 ) -> Result<ContiguousArray<T>, DmaError> {
189 ContiguousArray::new_zero(self, len, direction)
190 }
191
192 pub fn contiguous_array_zero_with_align<T: DmaPod>(
193 &self,
194 len: usize,
195 align: usize,
196 direction: DmaDirection,
197 ) -> Result<ContiguousArray<T>, DmaError> {
198 ContiguousArray::new_zero_with_align(self, len, align, direction)
199 }
200
201 pub fn coherent_box_zero<T: DmaPod>(&self) -> Result<CoherentBox<T>, DmaError> {
202 CoherentBox::new_zero(self)
203 }
204
205 pub fn coherent_box_zero_with_align<T: DmaPod>(
206 &self,
207 align: usize,
208 ) -> Result<CoherentBox<T>, DmaError> {
209 CoherentBox::new_zero_with_align(self, align)
210 }
211
212 pub fn contiguous_box_zero<T: DmaPod>(
213 &self,
214 direction: DmaDirection,
215 ) -> Result<ContiguousBox<T>, DmaError> {
216 ContiguousBox::new_zero(self, direction)
217 }
218
219 pub fn contiguous_box_zero_with_align<T: DmaPod>(
220 &self,
221 align: usize,
222 direction: DmaDirection,
223 ) -> Result<ContiguousBox<T>, DmaError> {
224 ContiguousBox::new_zero_with_align(self, align, direction)
225 }
226
227 pub fn map_streaming_slice<T: DmaPod>(
228 &self,
229 buff: &mut [T],
230 align: usize,
231 direction: DmaDirection,
232 ) -> Result<StreamingMap<T>, DmaError> {
233 StreamingMap::map(self, buff, align, direction)
234 }
235
236 pub fn map_streaming_slice_for_device<T: DmaPod>(
237 &self,
238 buff: &mut [T],
239 align: usize,
240 direction: DmaDirection,
241 ) -> Result<StreamingMap<T>, DmaError> {
242 let map = self.map_streaming_slice(buff, align, direction)?;
243 map.prepare_for_device_all();
244 Ok(map)
245 }
246
247 pub fn contiguous_buffer_pool(
248 &self,
249 layout: core::alloc::Layout,
250 direction: DmaDirection,
251 cap: usize,
252 ) -> ContiguousBufferPool {
253 let config = ContiguousBufferConfig {
254 size: layout.size(),
255 align: layout.align(),
256 direction,
257 };
258 ContiguousBufferPool::with_capacity(self.clone(), config, cap)
259 }
260
261 fn check_alloc_handle(
262 &self,
263 handle: &DmaAllocHandle,
264 constraints: DmaConstraints,
265 ) -> Result<(), DmaError> {
266 check_dma_range(handle.dma_addr(), handle.size(), constraints)?;
267 check_dma_align(handle.dma_addr(), handle.align().max(constraints.align))?;
268 Ok(())
269 }
270
271 fn check_map_handle(
272 &self,
273 handle: &DmaMapHandle,
274 constraints: DmaConstraints,
275 ) -> Result<(), DmaError> {
276 check_dma_range(handle.dma_addr(), handle.size(), constraints)?;
277 check_dma_align(handle.dma_addr(), handle.align().max(constraints.align))?;
278 Ok(())
279 }
280}
281
282fn check_dma_range(
283 addr: DmaAddr,
284 size: usize,
285 constraints: DmaConstraints,
286) -> Result<(), DmaError> {
287 let start = addr.as_u64();
288 let in_mask = if size == 0 {
289 start <= constraints.addr_mask
290 } else {
291 start
292 .checked_add(size.saturating_sub(1) as u64)
293 .map(|end| end <= constraints.addr_mask)
294 .unwrap_or(false)
295 };
296
297 if !in_mask {
298 return Err(DmaError::DmaMaskNotMatch {
299 addr,
300 mask: constraints.addr_mask,
301 });
302 }
303
304 if let Some(max) = constraints.max_segment_size
305 && size > max
306 {
307 return Err(DmaError::SegmentTooLarge { size, max });
308 }
309
310 if let Some(boundary) = constraints.boundary
311 && size > 0
312 {
313 let boundary = boundary as u64;
314 let end = start + size.saturating_sub(1) as u64;
315 if start / boundary != end / boundary {
316 return Err(DmaError::BoundaryCross {
317 addr,
318 size,
319 boundary: boundary as usize,
320 });
321 }
322 }
323
324 Ok(())
325}
326
327fn check_dma_align(addr: DmaAddr, align: usize) -> Result<(), DmaError> {
328 let align = align.max(1);
329 if !addr.as_u64().is_multiple_of(align as u64) {
330 return Err(DmaError::AlignMismatch {
331 required: align,
332 address: addr,
333 });
334 }
335 Ok(())
336}