1#![cfg_attr(target_os = "none", no_std)]
2#![doc = include_str!("../README.md")]
3
4extern crate alloc;
5
6use core::{num::NonZeroUsize, ptr::NonNull};
7
8mod op;
9
10mod array;
11mod common;
12mod dbox;
13mod def;
14mod owned;
15mod pool;
16mod streaming;
17
18pub use array::*;
19pub use dbox::*;
20pub use def::*;
21pub use op::DmaOp;
22pub use owned::*;
23pub use pool::*;
24pub use streaming::*;
25
26#[derive(Clone)]
27pub struct DeviceDma {
28 op: &'static dyn DmaOp,
29 constraints: DmaConstraints,
30 domain: DmaDomainId,
31}
32
33impl DeviceDma {
34 pub fn new(domain: DmaDomainId, dma_mask: u64, op: &'static dyn DmaOp) -> Self {
35 Self {
36 constraints: DmaConstraints::new(dma_mask),
37 domain,
38 op,
39 }
40 }
41
42 pub fn new_legacy(dma_mask: u64, op: &'static dyn DmaOp) -> Self {
43 Self::new(DmaDomainId::legacy_global(), dma_mask, op)
44 }
45
46 pub fn with_constraints(&self, constraints: DmaConstraints) -> Self {
47 Self {
48 op: self.op,
49 constraints,
50 domain: self.domain,
51 }
52 }
53
54 pub fn constraints(&self) -> DmaConstraints {
55 self.constraints
56 }
57
58 pub fn dma_mask(&self) -> u64 {
59 self.constraints.addr_mask
60 }
61
62 pub fn domain_id(&self) -> DmaDomainId {
63 self.domain
64 }
65
66 pub fn flush(&self, addr: NonNull<u8>, size: usize) {
67 self.op.flush(addr, size)
68 }
69
70 pub fn invalidate(&self, addr: NonNull<u8>, size: usize) {
71 self.op.invalidate(addr, size)
72 }
73
74 pub fn flush_invalidate(&self, addr: NonNull<u8>, size: usize) {
75 self.op.flush_invalidate(addr, size)
76 }
77
78 pub fn page_size(&self) -> usize {
79 self.op.page_size()
80 }
81
82 pub(crate) unsafe fn alloc_contiguous(
83 &self,
84 layout: core::alloc::Layout,
85 ) -> Result<DmaAllocHandle, DmaError> {
86 let constraints = self.constraints.with_align(layout.align());
87 let res =
88 unsafe { self.op.alloc_contiguous(constraints, layout) }.ok_or(DmaError::NoMemory)?;
89 match self.check_alloc_handle(&res, constraints) {
90 Ok(()) => Ok(res),
91 Err(e) => {
92 unsafe { self.op.dealloc_contiguous(res) };
93 Err(e)
94 }
95 }
96 }
97
98 pub(crate) unsafe fn dealloc_contiguous(&self, handle: DmaAllocHandle) {
99 unsafe { self.op.dealloc_contiguous(handle) }
100 }
101
102 pub(crate) unsafe fn alloc_coherent(
103 &self,
104 layout: core::alloc::Layout,
105 ) -> Result<DmaAllocHandle, DmaError> {
106 let constraints = self.constraints.with_align(layout.align());
107 let res =
108 unsafe { self.op.alloc_coherent(constraints, layout) }.ok_or(DmaError::NoMemory)?;
109 match self.check_alloc_handle(&res, constraints) {
110 Ok(()) => Ok(res),
111 Err(e) => {
112 unsafe { self.op.dealloc_coherent(res) };
113 Err(e)
114 }
115 }
116 }
117
118 pub(crate) unsafe fn dealloc_coherent(&self, handle: DmaAllocHandle) {
119 unsafe { self.op.dealloc_coherent(handle) }
120 }
121
122 pub(crate) unsafe fn map_streaming(
123 &self,
124 addr: NonNull<u8>,
125 size: NonZeroUsize,
126 align: usize,
127 direction: DmaDirection,
128 ) -> Result<DmaMapHandle, DmaError> {
129 let constraints = self.constraints.with_align(align);
130 let res = unsafe { self.op.map_streaming(constraints, addr, size, direction) }?;
131 match self.check_map_handle(&res, constraints) {
132 Ok(()) => Ok(res),
133 Err(e) => {
134 unsafe { self.op.unmap_streaming(res) };
135 Err(e)
136 }
137 }
138 }
139
140 pub(crate) unsafe fn unmap_streaming(&self, handle: DmaMapHandle) {
141 unsafe { self.op.unmap_streaming(handle) }
142 }
143
144 pub(crate) fn sync_alloc_for_device(
145 &self,
146 handle: &DmaAllocHandle,
147 offset: usize,
148 size: usize,
149 direction: DmaDirection,
150 ) {
151 self.op
152 .sync_alloc_for_device(handle, offset, size, direction);
153 }
154
155 pub(crate) fn sync_alloc_for_cpu(
156 &self,
157 handle: &DmaAllocHandle,
158 offset: usize,
159 size: usize,
160 direction: DmaDirection,
161 ) {
162 self.op.sync_alloc_for_cpu(handle, offset, size, direction);
163 }
164
165 pub(crate) fn sync_map_for_device(
166 &self,
167 handle: &DmaMapHandle,
168 offset: usize,
169 size: usize,
170 direction: DmaDirection,
171 ) {
172 self.op.sync_map_for_device(handle, offset, size, direction);
173 }
174
175 pub(crate) fn sync_map_for_cpu(
176 &self,
177 handle: &DmaMapHandle,
178 offset: usize,
179 size: usize,
180 direction: DmaDirection,
181 ) {
182 self.op.sync_map_for_cpu(handle, offset, size, direction);
183 }
184
185 pub fn coherent_array_zero<T: DmaPod>(&self, len: usize) -> Result<CoherentArray<T>, DmaError> {
186 CoherentArray::new_zero(self, len)
187 }
188
189 pub fn coherent_array_zero_with_align<T: DmaPod>(
190 &self,
191 len: usize,
192 align: usize,
193 ) -> Result<CoherentArray<T>, DmaError> {
194 CoherentArray::new_zero_with_align(self, len, align)
195 }
196
197 pub fn contiguous_array_zero<T: DmaPod>(
198 &self,
199 len: usize,
200 direction: DmaDirection,
201 ) -> Result<ContiguousArray<T>, DmaError> {
202 ContiguousArray::new_zero(self, len, direction)
203 }
204
205 pub fn contiguous_array_zero_with_align<T: DmaPod>(
206 &self,
207 len: usize,
208 align: usize,
209 direction: DmaDirection,
210 ) -> Result<ContiguousArray<T>, DmaError> {
211 ContiguousArray::new_zero_with_align(self, len, align, direction)
212 }
213
214 pub fn coherent_box_zero<T: DmaPod>(&self) -> Result<CoherentBox<T>, DmaError> {
215 CoherentBox::new_zero(self)
216 }
217
218 pub fn coherent_box_zero_with_align<T: DmaPod>(
219 &self,
220 align: usize,
221 ) -> Result<CoherentBox<T>, DmaError> {
222 CoherentBox::new_zero_with_align(self, align)
223 }
224
225 pub fn contiguous_box_zero<T: DmaPod>(
226 &self,
227 direction: DmaDirection,
228 ) -> Result<ContiguousBox<T>, DmaError> {
229 ContiguousBox::new_zero(self, direction)
230 }
231
232 pub fn contiguous_box_zero_with_align<T: DmaPod>(
233 &self,
234 align: usize,
235 direction: DmaDirection,
236 ) -> Result<ContiguousBox<T>, DmaError> {
237 ContiguousBox::new_zero_with_align(self, align, direction)
238 }
239
240 pub fn map_streaming_slice<T: DmaPod>(
241 &self,
242 buff: &mut [T],
243 align: usize,
244 direction: DmaDirection,
245 ) -> Result<StreamingMap<T>, DmaError> {
246 StreamingMap::map(self, buff, align, direction)
247 }
248
249 pub fn map_streaming_slice_for_device<T: DmaPod>(
250 &self,
251 buff: &mut [T],
252 align: usize,
253 direction: DmaDirection,
254 ) -> Result<StreamingMap<T>, DmaError> {
255 let map = self.map_streaming_slice(buff, align, direction)?;
256 map.prepare_for_device_all();
257 Ok(map)
258 }
259
260 pub fn contiguous_buffer_pool(
261 &self,
262 layout: core::alloc::Layout,
263 direction: DmaDirection,
264 cap: usize,
265 ) -> ContiguousBufferPool {
266 let config = ContiguousBufferConfig {
267 size: layout.size(),
268 align: layout.align(),
269 direction,
270 };
271 ContiguousBufferPool::with_capacity(self.clone(), config, cap)
272 }
273
274 fn check_alloc_handle(
275 &self,
276 handle: &DmaAllocHandle,
277 constraints: DmaConstraints,
278 ) -> Result<(), DmaError> {
279 check_dma_range(handle.dma_addr(), handle.size(), constraints)?;
280 check_dma_align(handle.dma_addr(), handle.align().max(constraints.align))?;
281 Ok(())
282 }
283
284 fn check_map_handle(
285 &self,
286 handle: &DmaMapHandle,
287 constraints: DmaConstraints,
288 ) -> Result<(), DmaError> {
289 check_dma_range(handle.dma_addr(), handle.size(), constraints)?;
290 check_dma_align(handle.dma_addr(), handle.align().max(constraints.align))?;
291 Ok(())
292 }
293}
294
295fn check_dma_range(
296 addr: DmaAddr,
297 size: usize,
298 constraints: DmaConstraints,
299) -> Result<(), DmaError> {
300 let start = addr.as_u64();
301 let in_mask = if size == 0 {
302 start <= constraints.addr_mask
303 } else {
304 start
305 .checked_add(size.saturating_sub(1) as u64)
306 .map(|end| end <= constraints.addr_mask)
307 .unwrap_or(false)
308 };
309
310 if !in_mask {
311 return Err(DmaError::DmaMaskNotMatch {
312 addr,
313 mask: constraints.addr_mask,
314 });
315 }
316
317 if let Some(max) = constraints.max_segment_size
318 && size > max
319 {
320 return Err(DmaError::SegmentTooLarge { size, max });
321 }
322
323 if let Some(boundary) = constraints.boundary
324 && size > 0
325 {
326 let boundary = boundary as u64;
327 let end = start + size.saturating_sub(1) as u64;
328 if start / boundary != end / boundary {
329 return Err(DmaError::BoundaryCross {
330 addr,
331 size,
332 boundary: boundary as usize,
333 });
334 }
335 }
336
337 Ok(())
338}
339
340fn check_dma_align(addr: DmaAddr, align: usize) -> Result<(), DmaError> {
341 let align = align.max(1);
342 if !addr.as_u64().is_multiple_of(align as u64) {
343 return Err(DmaError::AlignMismatch {
344 required: align,
345 address: addr,
346 });
347 }
348 Ok(())
349}