dma_api/
lib.rs

1#![cfg_attr(target_os = "none", no_std)]
2#![doc = include_str!("../README.md")]
3
4extern crate alloc;
5
6use core::{num::NonZeroUsize, ptr::NonNull};
7
8mod op;
9
10mod array;
11mod common;
12mod dbox;
13mod def;
14mod owned;
15mod pool;
16mod streaming;
17
18#[cfg(all(axtest, feature = "axtest"))]
19/// Coverage tests for DMA buffer and mapping APIs.
20pub mod axtest;
21
22pub use array::*;
23pub use dbox::*;
24pub use def::*;
25pub use op::DmaOp;
26pub use owned::*;
27pub use pool::*;
28pub use streaming::*;
29
30#[derive(Clone)]
31pub struct DeviceDma {
32    op: &'static dyn DmaOp,
33    constraints: DmaConstraints,
34    domain: DmaDomainId,
35}
36
37impl DeviceDma {
38    pub fn new(domain: DmaDomainId, dma_mask: u64, op: &'static dyn DmaOp) -> Self {
39        Self {
40            constraints: DmaConstraints::new(dma_mask),
41            domain,
42            op,
43        }
44    }
45
46    pub fn new_legacy(dma_mask: u64, op: &'static dyn DmaOp) -> Self {
47        Self::new(DmaDomainId::legacy_global(), dma_mask, op)
48    }
49
50    pub fn with_constraints(&self, constraints: DmaConstraints) -> Self {
51        Self {
52            op: self.op,
53            constraints,
54            domain: self.domain,
55        }
56    }
57
58    pub fn constraints(&self) -> DmaConstraints {
59        self.constraints
60    }
61
62    pub fn dma_mask(&self) -> u64 {
63        self.constraints.addr_mask
64    }
65
66    pub fn domain_id(&self) -> DmaDomainId {
67        self.domain
68    }
69
70    pub fn flush(&self, addr: NonNull<u8>, size: usize) {
71        self.op.flush(addr, size)
72    }
73
74    pub fn invalidate(&self, addr: NonNull<u8>, size: usize) {
75        self.op.invalidate(addr, size)
76    }
77
78    pub fn flush_invalidate(&self, addr: NonNull<u8>, size: usize) {
79        self.op.flush_invalidate(addr, size)
80    }
81
82    pub fn page_size(&self) -> usize {
83        self.op.page_size()
84    }
85
86    pub(crate) unsafe fn alloc_contiguous(
87        &self,
88        layout: core::alloc::Layout,
89    ) -> Result<DmaAllocHandle, DmaError> {
90        let constraints = self.constraints.with_align(layout.align());
91        let res =
92            unsafe { self.op.alloc_contiguous(constraints, layout) }.ok_or(DmaError::NoMemory)?;
93        match self.check_alloc_handle(&res, constraints) {
94            Ok(()) => Ok(res),
95            Err(e) => {
96                unsafe { self.op.dealloc_contiguous(res) };
97                Err(e)
98            }
99        }
100    }
101
102    pub(crate) unsafe fn dealloc_contiguous(&self, handle: DmaAllocHandle) {
103        unsafe { self.op.dealloc_contiguous(handle) }
104    }
105
106    pub(crate) unsafe fn alloc_coherent(
107        &self,
108        layout: core::alloc::Layout,
109    ) -> Result<DmaAllocHandle, DmaError> {
110        let constraints = self.constraints.with_align(layout.align());
111        let res =
112            unsafe { self.op.alloc_coherent(constraints, layout) }.ok_or(DmaError::NoMemory)?;
113        match self.check_alloc_handle(&res, constraints) {
114            Ok(()) => Ok(res),
115            Err(e) => {
116                if let Err(release_err) = unsafe { self.op.dealloc_coherent(res) } {
117                    log::error!(
118                        "failed to release invalid coherent DMA allocation; allocation \
119                         quarantined: {release_err}"
120                    );
121                }
122                Err(e)
123            }
124        }
125    }
126
127    pub(crate) unsafe fn dealloc_coherent(&self, handle: DmaAllocHandle) -> Result<(), DmaError> {
128        unsafe { self.op.dealloc_coherent(handle) }
129    }
130
131    pub(crate) unsafe fn map_streaming(
132        &self,
133        addr: NonNull<u8>,
134        size: NonZeroUsize,
135        align: usize,
136        direction: DmaDirection,
137    ) -> Result<DmaMapHandle, DmaError> {
138        let constraints = self.constraints.with_align(align);
139        let res = unsafe { self.op.map_streaming(constraints, addr, size, direction) }?;
140        match self.check_map_handle(&res, constraints) {
141            Ok(()) => Ok(res),
142            Err(e) => {
143                unsafe { self.op.unmap_streaming(res) };
144                Err(e)
145            }
146        }
147    }
148
149    pub(crate) unsafe fn unmap_streaming(&self, handle: DmaMapHandle) {
150        unsafe { self.op.unmap_streaming(handle) }
151    }
152
153    pub(crate) fn sync_alloc_for_device(
154        &self,
155        handle: &DmaAllocHandle,
156        offset: usize,
157        size: usize,
158        direction: DmaDirection,
159    ) {
160        self.op
161            .sync_alloc_for_device(handle, offset, size, direction);
162    }
163
164    pub(crate) fn sync_alloc_for_cpu(
165        &self,
166        handle: &DmaAllocHandle,
167        offset: usize,
168        size: usize,
169        direction: DmaDirection,
170    ) {
171        self.op.sync_alloc_for_cpu(handle, offset, size, direction);
172    }
173
174    pub(crate) fn sync_map_for_device(
175        &self,
176        handle: &DmaMapHandle,
177        offset: usize,
178        size: usize,
179        direction: DmaDirection,
180    ) {
181        self.op.sync_map_for_device(handle, offset, size, direction);
182    }
183
184    pub(crate) fn sync_map_for_cpu(
185        &self,
186        handle: &DmaMapHandle,
187        offset: usize,
188        size: usize,
189        direction: DmaDirection,
190    ) {
191        self.op.sync_map_for_cpu(handle, offset, size, direction);
192    }
193
194    pub fn coherent_array_zero<T: DmaPod>(&self, len: usize) -> Result<CoherentArray<T>, DmaError> {
195        CoherentArray::new_zero(self, len)
196    }
197
198    pub fn coherent_array_zero_with_align<T: DmaPod>(
199        &self,
200        len: usize,
201        align: usize,
202    ) -> Result<CoherentArray<T>, DmaError> {
203        CoherentArray::new_zero_with_align(self, len, align)
204    }
205
206    pub fn contiguous_array_zero<T: DmaPod>(
207        &self,
208        len: usize,
209        direction: DmaDirection,
210    ) -> Result<ContiguousArray<T>, DmaError> {
211        ContiguousArray::new_zero(self, len, direction)
212    }
213
214    pub fn contiguous_array_zero_with_align<T: DmaPod>(
215        &self,
216        len: usize,
217        align: usize,
218        direction: DmaDirection,
219    ) -> Result<ContiguousArray<T>, DmaError> {
220        ContiguousArray::new_zero_with_align(self, len, align, direction)
221    }
222
223    pub fn coherent_box_zero<T: DmaPod>(&self) -> Result<CoherentBox<T>, DmaError> {
224        CoherentBox::new_zero(self)
225    }
226
227    pub fn coherent_box_zero_with_align<T: DmaPod>(
228        &self,
229        align: usize,
230    ) -> Result<CoherentBox<T>, DmaError> {
231        CoherentBox::new_zero_with_align(self, align)
232    }
233
234    pub fn contiguous_box_zero<T: DmaPod>(
235        &self,
236        direction: DmaDirection,
237    ) -> Result<ContiguousBox<T>, DmaError> {
238        ContiguousBox::new_zero(self, direction)
239    }
240
241    pub fn contiguous_box_zero_with_align<T: DmaPod>(
242        &self,
243        align: usize,
244        direction: DmaDirection,
245    ) -> Result<ContiguousBox<T>, DmaError> {
246        ContiguousBox::new_zero_with_align(self, align, direction)
247    }
248
249    pub fn map_streaming_slice<T: DmaPod>(
250        &self,
251        buff: &mut [T],
252        align: usize,
253        direction: DmaDirection,
254    ) -> Result<StreamingMap<T>, DmaError> {
255        StreamingMap::map(self, buff, align, direction)
256    }
257
258    pub fn map_streaming_slice_for_device<T: DmaPod>(
259        &self,
260        buff: &mut [T],
261        align: usize,
262        direction: DmaDirection,
263    ) -> Result<StreamingMap<T>, DmaError> {
264        let map = self.map_streaming_slice(buff, align, direction)?;
265        map.prepare_for_device_all();
266        Ok(map)
267    }
268
269    pub fn contiguous_buffer_pool(
270        &self,
271        layout: core::alloc::Layout,
272        direction: DmaDirection,
273        cap: usize,
274    ) -> ContiguousBufferPool {
275        let config = ContiguousBufferConfig {
276            size: layout.size(),
277            align: layout.align(),
278            direction,
279        };
280        ContiguousBufferPool::with_capacity(self.clone(), config, cap)
281    }
282
283    fn check_alloc_handle(
284        &self,
285        handle: &DmaAllocHandle,
286        constraints: DmaConstraints,
287    ) -> Result<(), DmaError> {
288        check_dma_range(handle.dma_addr(), handle.size(), constraints)?;
289        check_dma_align(handle.dma_addr(), handle.align().max(constraints.align))?;
290        Ok(())
291    }
292
293    fn check_map_handle(
294        &self,
295        handle: &DmaMapHandle,
296        constraints: DmaConstraints,
297    ) -> Result<(), DmaError> {
298        check_dma_range(handle.dma_addr(), handle.size(), constraints)?;
299        check_dma_align(handle.dma_addr(), handle.align().max(constraints.align))?;
300        Ok(())
301    }
302}
303
304fn check_dma_range(
305    addr: DmaAddr,
306    size: usize,
307    constraints: DmaConstraints,
308) -> Result<(), DmaError> {
309    let start = addr.as_u64();
310    let in_mask = if size == 0 {
311        start <= constraints.addr_mask
312    } else {
313        start
314            .checked_add(size.saturating_sub(1) as u64)
315            .map(|end| end <= constraints.addr_mask)
316            .unwrap_or(false)
317    };
318
319    if !in_mask {
320        return Err(DmaError::DmaMaskNotMatch {
321            addr,
322            mask: constraints.addr_mask,
323        });
324    }
325
326    if let Some(max) = constraints.max_segment_size
327        && size > max
328    {
329        return Err(DmaError::SegmentTooLarge { size, max });
330    }
331
332    if let Some(boundary) = constraints.boundary
333        && size > 0
334    {
335        let boundary = boundary as u64;
336        let end = start + size.saturating_sub(1) as u64;
337        if start / boundary != end / boundary {
338            return Err(DmaError::BoundaryCross {
339                addr,
340                size,
341                boundary: boundary as usize,
342            });
343        }
344    }
345
346    Ok(())
347}
348
349fn check_dma_align(addr: DmaAddr, align: usize) -> Result<(), DmaError> {
350    let align = align.max(1);
351    if !addr.as_u64().is_multiple_of(align as u64) {
352        return Err(DmaError::AlignMismatch {
353            required: align,
354            address: addr,
355        });
356    }
357    Ok(())
358}
dma_api/lib.rs

dma_api/
lib.rs
