1#![cfg_attr(target_os = "none", no_std)]
2#![doc = include_str!("../README.md")]
3
4extern crate alloc;
5
6use core::{num::NonZeroUsize, ptr::NonNull};
7
8mod op;
9
10mod array;
11mod common;
12mod dbox;
13mod def;
14mod owned;
15mod pool;
16mod streaming;
17
18pub use array::*;
19pub use dbox::*;
20pub use def::*;
21pub use op::DmaOp;
22pub use owned::*;
23pub use pool::*;
24pub use streaming::*;
25
26#[derive(Clone)]
27pub struct DeviceDma {
28 op: &'static dyn DmaOp,
29 info: DmaDeviceInfo,
30}
31
32impl DeviceDma {
33 pub const fn new(info: DmaDeviceInfo, op: &'static dyn DmaOp) -> Self {
34 Self { info, op }
35 }
36
37 pub fn with_constraints(&self, constraints: DmaConstraints) -> Self {
38 Self {
39 op: self.op,
40 info: self.info.with_constraints(constraints),
41 }
42 }
43
44 pub const fn info(&self) -> DmaDeviceInfo {
45 self.info
46 }
47
48 pub fn page_size(&self) -> usize {
49 self.op.page_size()
50 }
51
52 pub(crate) unsafe fn alloc_contiguous(
53 &self,
54 layout: core::alloc::Layout,
55 ) -> Result<DmaAllocHandle, DmaError> {
56 let mut constraints = self.info.constraints();
57 constraints.align = constraints.align.max(layout.align());
58 let res =
59 unsafe { self.op.alloc_contiguous(constraints, layout) }.ok_or(DmaError::NoMemory)?;
60 match self.check_alloc_handle(&res, constraints) {
61 Ok(()) => Ok(res),
62 Err(e) => {
63 unsafe { self.op.dealloc_contiguous(res) };
64 Err(e)
65 }
66 }
67 }
68
69 pub(crate) unsafe fn dealloc_contiguous(&self, handle: DmaAllocHandle) {
70 unsafe { self.op.dealloc_contiguous(handle) }
71 }
72
73 pub(crate) unsafe fn alloc_coherent(
74 &self,
75 layout: core::alloc::Layout,
76 ) -> Result<DmaAllocHandle, DmaError> {
77 let mut constraints = self.info.constraints();
78 constraints.align = constraints.align.max(layout.align());
79 let res = match self.info.coherency() {
80 DmaCoherency::Coherent => unsafe { self.op.alloc_contiguous(constraints, layout) },
81 DmaCoherency::NonCoherent => unsafe { self.op.alloc_coherent(constraints, layout) },
82 }
83 .ok_or(DmaError::NoMemory)?;
84 match self.check_alloc_handle(&res, constraints) {
85 Ok(()) => Ok(res),
86 Err(e) => {
87 match self.info.coherency() {
88 DmaCoherency::Coherent => unsafe { self.op.dealloc_contiguous(res) },
89 DmaCoherency::NonCoherent => {
90 if let Err(release_err) = unsafe { self.op.dealloc_coherent(res) } {
91 log::error!(
92 "failed to release invalid coherent DMA allocation; allocation \
93 quarantined: {release_err}"
94 );
95 }
96 }
97 }
98 Err(e)
99 }
100 }
101 }
102
103 pub(crate) unsafe fn dealloc_coherent(&self, handle: DmaAllocHandle) -> Result<(), DmaError> {
104 match self.info.coherency() {
105 DmaCoherency::Coherent => {
106 unsafe { self.op.dealloc_contiguous(handle) };
107 Ok(())
108 }
109 DmaCoherency::NonCoherent => unsafe { self.op.dealloc_coherent(handle) },
110 }
111 }
112
113 pub(crate) unsafe fn map_streaming(
114 &self,
115 addr: NonNull<u8>,
116 size: NonZeroUsize,
117 align: usize,
118 direction: DmaDirection,
119 ) -> Result<DmaMapHandle, DmaError> {
120 let mut constraints = self.info.constraints();
121 constraints.align = constraints.align.max(align);
122 let res = unsafe { self.op.map_streaming(constraints, addr, size, direction) }?;
123 match self.check_map_handle(&res, constraints) {
124 Ok(()) => Ok(res),
125 Err(e) => {
126 unsafe { self.op.unmap_streaming(res) };
127 Err(e)
128 }
129 }
130 }
131
132 pub(crate) unsafe fn unmap_streaming(&self, handle: DmaMapHandle) {
133 unsafe { self.op.unmap_streaming(handle) }
134 }
135
136 pub(crate) fn sync_alloc_for_device(
137 &self,
138 handle: &DmaAllocHandle,
139 offset: usize,
140 size: usize,
141 direction: DmaDirection,
142 ) {
143 if self.info.coherency() == DmaCoherency::NonCoherent {
144 self.op
145 .sync_alloc_for_device(handle, offset, size, direction);
146 }
147 }
148
149 pub(crate) fn sync_alloc_for_cpu(
150 &self,
151 handle: &DmaAllocHandle,
152 offset: usize,
153 size: usize,
154 direction: DmaDirection,
155 ) {
156 if self.info.coherency() == DmaCoherency::NonCoherent {
157 self.op.sync_alloc_for_cpu(handle, offset, size, direction);
158 }
159 }
160
161 pub(crate) fn sync_map_for_device(
162 &self,
163 handle: &DmaMapHandle,
164 offset: usize,
165 size: usize,
166 direction: DmaDirection,
167 ) {
168 self.op
169 .sync_map_for_device(handle, offset, size, direction, self.info.coherency());
170 }
171
172 pub(crate) fn sync_map_for_cpu(
173 &self,
174 handle: &DmaMapHandle,
175 offset: usize,
176 size: usize,
177 direction: DmaDirection,
178 ) {
179 self.op
180 .sync_map_for_cpu(handle, offset, size, direction, self.info.coherency());
181 }
182
183 pub fn coherent_array_zero<T: DmaPod>(&self, len: usize) -> Result<CoherentArray<T>, DmaError> {
184 CoherentArray::new_zero(self, len)
185 }
186
187 pub fn coherent_array_zero_with_align<T: DmaPod>(
188 &self,
189 len: usize,
190 align: usize,
191 ) -> Result<CoherentArray<T>, DmaError> {
192 CoherentArray::new_zero_with_align(self, len, align)
193 }
194
195 pub fn contiguous_array_zero<T: DmaPod>(
196 &self,
197 len: usize,
198 direction: DmaDirection,
199 ) -> Result<ContiguousArray<T>, DmaError> {
200 ContiguousArray::new_zero(self, len, direction)
201 }
202
203 pub fn contiguous_array_zero_with_align<T: DmaPod>(
204 &self,
205 len: usize,
206 align: usize,
207 direction: DmaDirection,
208 ) -> Result<ContiguousArray<T>, DmaError> {
209 ContiguousArray::new_zero_with_align(self, len, align, direction)
210 }
211
212 pub fn coherent_box_zero<T: DmaPod>(&self) -> Result<CoherentBox<T>, DmaError> {
213 CoherentBox::new_zero(self)
214 }
215
216 pub fn coherent_box_zero_with_align<T: DmaPod>(
217 &self,
218 align: usize,
219 ) -> Result<CoherentBox<T>, DmaError> {
220 CoherentBox::new_zero_with_align(self, align)
221 }
222
223 pub fn contiguous_box_zero<T: DmaPod>(
224 &self,
225 direction: DmaDirection,
226 ) -> Result<ContiguousBox<T>, DmaError> {
227 ContiguousBox::new_zero(self, direction)
228 }
229
230 pub fn contiguous_box_zero_with_align<T: DmaPod>(
231 &self,
232 align: usize,
233 direction: DmaDirection,
234 ) -> Result<ContiguousBox<T>, DmaError> {
235 ContiguousBox::new_zero_with_align(self, align, direction)
236 }
237
238 pub fn map_streaming_slice<T: DmaPod>(
239 &self,
240 buff: &mut [T],
241 align: usize,
242 direction: DmaDirection,
243 ) -> Result<StreamingMap<T>, DmaError> {
244 StreamingMap::map(self, buff, align, direction)
245 }
246
247 pub fn map_streaming_slice_for_device<T: DmaPod>(
248 &self,
249 buff: &mut [T],
250 align: usize,
251 direction: DmaDirection,
252 ) -> Result<StreamingMap<T>, DmaError> {
253 let map = self.map_streaming_slice(buff, align, direction)?;
254 map.prepare_for_device(0..map.bytes_len());
255 Ok(map)
256 }
257
258 pub fn contiguous_buffer_pool(
259 &self,
260 layout: core::alloc::Layout,
261 direction: DmaDirection,
262 cap: usize,
263 ) -> ContiguousBufferPool {
264 let config = ContiguousBufferConfig {
265 size: layout.size(),
266 align: layout.align(),
267 direction,
268 };
269 ContiguousBufferPool::with_capacity(self.clone(), config, cap)
270 }
271
272 fn check_alloc_handle(
273 &self,
274 handle: &DmaAllocHandle,
275 constraints: DmaConstraints,
276 ) -> Result<(), DmaError> {
277 check_dma_range(handle.dma_addr(), handle.size(), constraints)?;
278 check_dma_align(handle.dma_addr(), handle.align().max(constraints.align))?;
279 Ok(())
280 }
281
282 fn check_map_handle(
283 &self,
284 handle: &DmaMapHandle,
285 constraints: DmaConstraints,
286 ) -> Result<(), DmaError> {
287 check_dma_range(handle.dma_addr(), handle.size(), constraints)?;
288 check_dma_align(handle.dma_addr(), handle.align().max(constraints.align))?;
289 Ok(())
290 }
291}
292
293fn check_dma_range(
294 addr: DmaAddr,
295 size: usize,
296 constraints: DmaConstraints,
297) -> Result<(), DmaError> {
298 let start = addr.as_u64();
299 let in_mask = if size == 0 {
300 start <= constraints.addr_mask
301 } else {
302 start
303 .checked_add(size.saturating_sub(1) as u64)
304 .map(|end| end <= constraints.addr_mask)
305 .unwrap_or(false)
306 };
307
308 if !in_mask {
309 return Err(DmaError::DmaMaskNotMatch {
310 addr,
311 mask: constraints.addr_mask,
312 });
313 }
314
315 if let Some(max) = constraints.max_segment_size
316 && size > max
317 {
318 return Err(DmaError::SegmentTooLarge { size, max });
319 }
320
321 if let Some(boundary) = constraints.boundary
322 && size > 0
323 {
324 let boundary = boundary as u64;
325 let end = start + size.saturating_sub(1) as u64;
326 if start / boundary != end / boundary {
327 return Err(DmaError::BoundaryCross {
328 addr,
329 size,
330 boundary: boundary as usize,
331 });
332 }
333 }
334
335 Ok(())
336}
337
338fn check_dma_align(addr: DmaAddr, align: usize) -> Result<(), DmaError> {
339 let align = align.max(1);
340 if !addr.as_u64().is_multiple_of(align as u64) {
341 return Err(DmaError::AlignMismatch {
342 required: align,
343 address: addr,
344 });
345 }
346 Ok(())
347}