1#![cfg_attr(target_os = "none", no_std)]
2#![doc = include_str!("../README.md")]
3
4extern crate alloc;
5
6use core::{num::NonZeroUsize, ptr::NonNull};
7
8mod op;
9
10mod array;
11mod common;
12mod dbox;
13mod def;
14mod owned;
15mod pool;
16mod streaming;
17
18pub use array::*;
19pub use dbox::*;
20pub use def::*;
21pub use op::DmaOp;
22pub use owned::*;
23pub use pool::*;
24pub use streaming::*;
25
26#[derive(Clone)]
27pub struct DeviceDma {
28 op: &'static dyn DmaOp,
29 info: DmaDeviceInfo,
30}
31
32impl DeviceDma {
33 pub const fn new(info: DmaDeviceInfo, op: &'static dyn DmaOp) -> Self {
34 Self { info, op }
35 }
36
37 pub fn with_constraints(&self, constraints: DmaConstraints) -> Self {
38 Self {
39 op: self.op,
40 info: self.info.with_constraints(constraints),
41 }
42 }
43
44 pub const fn info(&self) -> DmaDeviceInfo {
45 self.info
46 }
47
48 pub fn page_size(&self) -> usize {
49 self.op.page_size()
50 }
51
52 pub(crate) unsafe fn alloc_contiguous(
53 &self,
54 layout: core::alloc::Layout,
55 ) -> Result<DmaAllocHandle, DmaError> {
56 let mut constraints = self.info.constraints();
57 constraints.align = constraints.align.max(layout.align());
58 let res =
59 unsafe { self.op.alloc_contiguous(constraints, layout) }.ok_or(DmaError::NoMemory)?;
60 match self.check_alloc_handle(&res, constraints) {
61 Ok(()) => Ok(res),
62 Err(e) => {
63 unsafe { self.op.dealloc_contiguous(res) };
64 Err(e)
65 }
66 }
67 }
68
69 pub(crate) unsafe fn dealloc_contiguous(&self, handle: DmaAllocHandle) {
70 unsafe { self.op.dealloc_contiguous(handle) }
71 }
72
73 pub(crate) unsafe fn alloc_coherent(
74 &self,
75 layout: core::alloc::Layout,
76 ) -> Result<DmaAllocHandle, DmaError> {
77 let mut constraints = self.info.constraints();
78 constraints.align = constraints.align.max(layout.align());
79 let res = match self.info.coherency() {
80 DmaCoherency::Coherent => unsafe { self.op.alloc_contiguous(constraints, layout) },
81 DmaCoherency::NonCoherent => unsafe { self.op.alloc_coherent(constraints, layout) },
82 }
83 .ok_or(DmaError::NoMemory)?;
84 match self.check_alloc_handle(&res, constraints) {
85 Ok(()) => Ok(res),
86 Err(e) => {
87 match self.info.coherency() {
88 DmaCoherency::Coherent => unsafe { self.op.dealloc_contiguous(res) },
89 DmaCoherency::NonCoherent => {
90 if let Err(release_err) = unsafe { self.op.dealloc_coherent(res) } {
91 log::error!(
92 "failed to release invalid coherent DMA allocation; allocation \
93 quarantined: {release_err}"
94 );
95 }
96 }
97 }
98 Err(e)
99 }
100 }
101 }
102
103 pub(crate) unsafe fn dealloc_coherent(&self, handle: DmaAllocHandle) -> Result<(), DmaError> {
104 match self.info.coherency() {
105 DmaCoherency::Coherent => {
106 unsafe { self.op.dealloc_contiguous(handle) };
107 Ok(())
108 }
109 DmaCoherency::NonCoherent => unsafe { self.op.dealloc_coherent(handle) },
110 }
111 }
112
113 pub(crate) unsafe fn map_streaming(
114 &self,
115 addr: NonNull<u8>,
116 size: NonZeroUsize,
117 align: usize,
118 direction: DmaDirection,
119 ) -> Result<DmaMapHandle, DmaError> {
120 let mut constraints = self.info.constraints();
121 constraints.align = constraints.align.max(align);
122 let res = unsafe { self.op.map_streaming(constraints, addr, size, direction) }?;
123 match self.check_map_handle(&res, constraints) {
124 Ok(()) => Ok(res),
125 Err(e) => {
126 unsafe { self.op.unmap_streaming(res) };
127 Err(e)
128 }
129 }
130 }
131
132 pub(crate) unsafe fn unmap_streaming(&self, handle: DmaMapHandle) {
133 unsafe { self.op.unmap_streaming(handle) }
134 }
135
136 pub(crate) fn sync_alloc_for_device(
137 &self,
138 handle: &DmaAllocHandle,
139 offset: usize,
140 size: usize,
141 direction: DmaDirection,
142 ) {
143 if self.info.coherency() == DmaCoherency::NonCoherent {
144 self.op
145 .sync_alloc_for_device(handle, offset, size, direction);
146 }
147 }
148
149 pub(crate) fn sync_alloc_for_cpu(
150 &self,
151 handle: &DmaAllocHandle,
152 offset: usize,
153 size: usize,
154 direction: DmaDirection,
155 ) {
156 if self.info.coherency() == DmaCoherency::NonCoherent {
157 self.op.sync_alloc_for_cpu(handle, offset, size, direction);
158 }
159 }
160
161 pub(crate) fn sync_map_for_device(
162 &self,
163 handle: &DmaMapHandle,
164 offset: usize,
165 size: usize,
166 direction: DmaDirection,
167 ) {
168 self.op
169 .sync_map_for_device(handle, offset, size, direction, self.info.coherency());
170 }
171
172 pub(crate) fn sync_map_for_cpu(
173 &self,
174 handle: &DmaMapHandle,
175 offset: usize,
176 size: usize,
177 direction: DmaDirection,
178 ) {
179 self.op
180 .sync_map_for_cpu(handle, offset, size, direction, self.info.coherency());
181 }
182
183 pub fn coherent_array_zero<T: DmaPod>(&self, len: usize) -> Result<CoherentArray<T>, DmaError> {
184 CoherentArray::new_zero(self, len)
185 }
186
187 pub fn coherent_array_zero_with_align<T: DmaPod>(
188 &self,
189 len: usize,
190 align: usize,
191 ) -> Result<CoherentArray<T>, DmaError> {
192 CoherentArray::new_zero_with_align(self, len, align)
193 }
194
195 pub fn contiguous_array_zero<T: DmaPod>(
196 &self,
197 len: usize,
198 direction: DmaDirection,
199 ) -> Result<ContiguousArray<T>, DmaError> {
200 ContiguousArray::new_zero(self, len, direction)
201 }
202
203 pub fn contiguous_array_zero_with_align<T: DmaPod>(
204 &self,
205 len: usize,
206 align: usize,
207 direction: DmaDirection,
208 ) -> Result<ContiguousArray<T>, DmaError> {
209 ContiguousArray::new_zero_with_align(self, len, align, direction)
210 }
211
212 pub fn coherent_box_zero<T: DmaPod>(&self) -> Result<CoherentBox<T>, DmaError> {
213 CoherentBox::new_zero(self)
214 }
215
216 pub fn coherent_box_zero_with_align<T: DmaPod>(
217 &self,
218 align: usize,
219 ) -> Result<CoherentBox<T>, DmaError> {
220 CoherentBox::new_zero_with_align(self, align)
221 }
222
223 pub fn contiguous_box_zero<T: DmaPod>(
224 &self,
225 direction: DmaDirection,
226 ) -> Result<ContiguousBox<T>, DmaError> {
227 ContiguousBox::new_zero(self, direction)
228 }
229
230 pub fn contiguous_box_zero_with_align<T: DmaPod>(
231 &self,
232 align: usize,
233 direction: DmaDirection,
234 ) -> Result<ContiguousBox<T>, DmaError> {
235 ContiguousBox::new_zero_with_align(self, align, direction)
236 }
237
238 pub fn map_streaming_slice<T: DmaPod>(
239 &self,
240 buff: &mut [T],
241 align: usize,
242 direction: DmaDirection,
243 ) -> Result<StreamingMap<T>, DmaError> {
244 StreamingMap::map(self, buff, align, direction)
245 }
246
247 pub fn map_streaming_slice_for_device<T: DmaPod>(
248 &self,
249 buff: &mut [T],
250 align: usize,
251 direction: DmaDirection,
252 ) -> Result<StreamingMap<T>, DmaError> {
253 let map = self.map_streaming_slice(buff, align, direction)?;
254 map.prepare_for_device(0..map.bytes_len());
255 Ok(map)
256 }
257
258 #[cfg(feature = "pool")]
259 pub fn contiguous_buffer_pool(
260 &self,
261 layout: core::alloc::Layout,
262 direction: DmaDirection,
263 cap: usize,
264 ) -> ContiguousBufferPool {
265 let config = ContiguousBufferConfig {
266 size: layout.size(),
267 align: layout.align(),
268 direction,
269 };
270 ContiguousBufferPool::with_capacity(self.clone(), config, cap)
271 }
272
273 fn check_alloc_handle(
274 &self,
275 handle: &DmaAllocHandle,
276 constraints: DmaConstraints,
277 ) -> Result<(), DmaError> {
278 check_dma_range(handle.dma_addr(), handle.size(), constraints)?;
279 check_dma_align(handle.dma_addr(), handle.align().max(constraints.align))?;
280 Ok(())
281 }
282
283 fn check_map_handle(
284 &self,
285 handle: &DmaMapHandle,
286 constraints: DmaConstraints,
287 ) -> Result<(), DmaError> {
288 check_dma_range(handle.dma_addr(), handle.size(), constraints)?;
289 check_dma_align(handle.dma_addr(), handle.align().max(constraints.align))?;
290 Ok(())
291 }
292}
293
294fn check_dma_range(
295 addr: DmaAddr,
296 size: usize,
297 constraints: DmaConstraints,
298) -> Result<(), DmaError> {
299 let start = addr.as_u64();
300 let in_mask = if size == 0 {
301 start <= constraints.addr_mask
302 } else {
303 start
304 .checked_add(size.saturating_sub(1) as u64)
305 .map(|end| end <= constraints.addr_mask)
306 .unwrap_or(false)
307 };
308
309 if !in_mask {
310 return Err(DmaError::DmaMaskNotMatch {
311 addr,
312 mask: constraints.addr_mask,
313 });
314 }
315
316 if let Some(max) = constraints.max_segment_size
317 && size > max
318 {
319 return Err(DmaError::SegmentTooLarge { size, max });
320 }
321
322 if let Some(boundary) = constraints.boundary
323 && size > 0
324 {
325 let boundary = boundary as u64;
326 let end = start + size.saturating_sub(1) as u64;
327 if start / boundary != end / boundary {
328 return Err(DmaError::BoundaryCross {
329 addr,
330 size,
331 boundary: boundary as usize,
332 });
333 }
334 }
335
336 Ok(())
337}
338
339fn check_dma_align(addr: DmaAddr, align: usize) -> Result<(), DmaError> {
340 let align = align.max(1);
341 if !addr.as_u64().is_multiple_of(align as u64) {
342 return Err(DmaError::AlignMismatch {
343 required: align,
344 address: addr,
345 });
346 }
347 Ok(())
348}