1#![no_std]
2
3extern crate alloc;
4
5use alloc::{boxed::Box, collections::BTreeMap, sync::Arc};
6use core::{alloc::Layout, cell::UnsafeCell};
7
8use dma_api::{ContiguousBuffer, ContiguousBufferPool, DeviceDma, DmaDirection, DmaOp};
9use futures::task::AtomicWaker;
10pub use rdif_eth::{IrqHandler as InterfaceIrqHandler, *};
11
12fn other_error(msg: &'static str) -> NetError {
13 NetError::Other(Box::new(KError::Unknown(msg)))
14}
15
16struct QueueWakerMap(UnsafeCell<BTreeMap<usize, Arc<AtomicWaker>>>);
17
18impl QueueWakerMap {
19 fn new() -> Self {
20 Self(UnsafeCell::new(BTreeMap::new()))
21 }
22
23 fn register(&self, queue_id: usize) -> Arc<AtomicWaker> {
24 let waker = Arc::new(AtomicWaker::new());
25 unsafe { &mut *self.0.get() }.insert(queue_id, waker.clone());
26 waker
27 }
28
29 fn wake(&self, queue_id: usize) {
30 if let Some(waker) = unsafe { &*self.0.get() }.get(&queue_id) {
31 waker.wake();
32 }
33 }
34}
35
36struct NetInner {
37 interface: UnsafeCell<Box<dyn Interface>>,
38 dma_op: &'static dyn DmaOp,
39 tx_wakers: QueueWakerMap,
40 rx_wakers: QueueWakerMap,
41}
42
43unsafe impl Send for NetInner {}
44unsafe impl Sync for NetInner {}
45
46struct IrqGuard<'a> {
47 enabled: bool,
48 inner: &'a Net,
49}
50
51impl Drop for IrqGuard<'_> {
52 fn drop(&mut self) {
53 if self.enabled {
54 self.inner.interface().enable_irq();
55 }
56 }
57}
58
59pub struct Net {
60 inner: Arc<NetInner>,
61}
62
63impl DriverGeneric for Net {
64 fn name(&self) -> &str {
65 self.interface().name()
66 }
67
68 fn raw_any(&self) -> Option<&dyn core::any::Any> {
69 Some(self)
70 }
71
72 fn raw_any_mut(&mut self) -> Option<&mut dyn core::any::Any> {
73 Some(self)
74 }
75}
76
77impl Net {
78 pub fn new(interface: impl Interface, dma_op: &'static dyn DmaOp) -> Self {
79 Self {
80 inner: Arc::new(NetInner {
81 interface: UnsafeCell::new(Box::new(interface)),
82 dma_op,
83 tx_wakers: QueueWakerMap::new(),
84 rx_wakers: QueueWakerMap::new(),
85 }),
86 }
87 }
88
89 #[allow(clippy::mut_from_ref)]
90 fn interface(&self) -> &mut dyn Interface {
91 unsafe { &mut **self.inner.interface.get() }
92 }
93
94 fn irq_guard(&self) -> IrqGuard<'_> {
95 let enabled = self.interface().is_irq_enabled();
96 if enabled {
97 self.interface().disable_irq();
98 }
99 IrqGuard {
100 enabled,
101 inner: self,
102 }
103 }
104
105 pub fn mac_address(&self) -> [u8; 6] {
106 self.interface().mac_address()
107 }
108
109 #[allow(clippy::mut_from_ref)]
116 pub fn wifi_control(&self) -> Option<&mut dyn WifiControl> {
117 self.interface().wifi_control()
118 }
119
120 pub fn enable_irq(&mut self) {
121 self.interface().enable_irq();
122 }
123
124 pub fn disable_irq(&mut self) {
125 self.interface().disable_irq();
126 }
127
128 pub fn is_irq_enabled(&self) -> bool {
129 self.interface().is_irq_enabled()
130 }
131
132 pub fn create_tx_queue(&mut self) -> Result<TxQueue, NetError> {
133 let irq_guard = self.irq_guard();
134 let queue = self
135 .interface()
136 .create_tx_queue()
137 .ok_or_else(|| other_error("failed to create tx queue"))?;
138 let config = queue.config();
139 let pool = make_pool(self.inner.dma_op, config, DmaDirection::ToDevice)?;
140 let waker = self.inner.tx_wakers.register(queue.id());
141 drop(irq_guard);
142
143 Ok(TxQueue {
144 interface: queue,
145 pool,
146 inflight: BTreeMap::new(),
147 config,
148 _waker: waker,
149 })
150 }
151
152 pub fn create_rx_queue(&mut self) -> Result<RxQueue, NetError> {
153 let irq_guard = self.irq_guard();
154 let queue = self
155 .interface()
156 .create_rx_queue()
157 .ok_or_else(|| other_error("failed to create rx queue"))?;
158 let config = queue.config();
159 let pool = make_pool(self.inner.dma_op, config, DmaDirection::FromDevice)?;
160 let waker = self.inner.rx_wakers.register(queue.id());
161 drop(irq_guard);
162
163 let mut rx = RxQueue {
164 interface: queue,
165 pool,
166 inflight: BTreeMap::new(),
167 config,
168 _waker: waker,
169 };
170 rx.prefill()?;
171 Ok(rx)
172 }
173
174 pub fn take_irq_handler(&mut self) -> Option<IrqHandler> {
175 let irq_guard = self.irq_guard();
176 let handler = self.interface().take_irq_handler();
177 drop(irq_guard);
178
179 handler.map(|handler| IrqHandler {
180 inner: self.inner.clone(),
181 handler,
182 })
183 }
184
185 pub fn wifi_control_handle(&self) -> Option<WifiControlHandle> {
193 self.wifi_control().is_some().then(|| WifiControlHandle {
194 inner: self.inner.clone(),
195 })
196 }
197}
198
199pub struct WifiControlHandle {
205 inner: Arc<NetInner>,
206}
207
208impl Clone for WifiControlHandle {
209 fn clone(&self) -> Self {
210 Self {
211 inner: self.inner.clone(),
212 }
213 }
214}
215
216unsafe impl Send for WifiControlHandle {}
217unsafe impl Sync for WifiControlHandle {}
218
219impl WifiControlHandle {
220 #[allow(clippy::mut_from_ref)]
230 pub fn wifi_control(&self) -> Option<&mut dyn WifiControl> {
231 let iface = unsafe { &mut **self.inner.interface.get() };
232 iface.wifi_control()
233 }
234
235 pub fn mac_address(&self) -> [u8; 6] {
238 let iface = unsafe { &mut **self.inner.interface.get() };
239 iface.mac_address()
240 }
241}
242
243fn make_pool(
244 dma_op: &'static dyn DmaOp,
245 config: QueueConfig,
246 direction: DmaDirection,
247) -> Result<ContiguousBufferPool, NetError> {
248 let layout = Layout::from_size_align(config.buf_size, config.align.max(1))
249 .map_err(|_| other_error("invalid queue layout"))?;
250 let dma = DeviceDma::new_legacy(config.dma_mask, dma_op);
251 Ok(dma.contiguous_buffer_pool(layout, direction, config.ring_size))
252}
253
254pub struct IrqHandler {
255 inner: Arc<NetInner>,
256 handler: rdif_eth::BIrqHandler,
257}
258
259unsafe impl Send for IrqHandler {}
260unsafe impl Sync for IrqHandler {}
261
262impl IrqHandler {
263 pub fn enable(&self) {
264 let iface = unsafe { &mut **self.inner.interface.get() };
265 iface.enable_irq();
266 }
267
268 pub fn disable(&self) {
269 let iface = unsafe { &mut **self.inner.interface.get() };
270 iface.disable_irq();
271 }
272
273 pub fn handle_irq(&mut self) -> rdif_eth::Event {
281 self.handler.handle_irq()
282 }
283
284 pub fn handle(&mut self) {
289 let event = self.handle_irq();
290 for id in event.tx_queue.iter() {
291 self.inner.tx_wakers.wake(id);
292 }
293 for id in event.rx_queue.iter() {
294 self.inner.rx_wakers.wake(id);
295 }
296 }
297
298 pub fn enable_irq(&self) {
299 let iface = unsafe { &mut **self.inner.interface.get() };
300 iface.enable_irq();
301 }
302
303 pub fn disable_irq(&self) {
304 let iface = unsafe { &mut **self.inner.interface.get() };
305 iface.disable_irq();
306 }
307
308 pub fn is_irq_enabled(&self) -> bool {
309 let iface = unsafe { &mut **self.inner.interface.get() };
310 iface.is_irq_enabled()
311 }
312}
313
314pub struct TxQueue {
315 interface: Box<dyn ITxQueue>,
316 pool: ContiguousBufferPool,
317 inflight: BTreeMap<u64, ContiguousBuffer>,
318 config: QueueConfig,
319 _waker: Arc<AtomicWaker>,
320}
321
322impl TxQueue {
323 fn capacity(&self) -> usize {
324 self.config.ring_size.saturating_sub(1)
325 }
326
327 fn reclaim_bounded(&mut self, limit: usize) -> Result<usize, NetError> {
328 let mut reclaimed = 0;
329 while reclaimed < limit {
330 let Some(bus_addr) = self.interface.reclaim() else {
331 break;
332 };
333 let Some(buff) = self.inflight.remove(&bus_addr) else {
334 return Err(other_error("reclaimed unknown tx buffer"));
335 };
336 drop(buff);
337 reclaimed += 1;
338 }
339 Ok(reclaimed)
340 }
341
342 pub fn id(&self) -> usize {
343 self.interface.id()
344 }
345
346 pub fn buf_size(&self) -> usize {
347 self.config.buf_size
348 }
349
350 pub fn prepare_send<R>(
351 &mut self,
352 len: usize,
353 f: impl FnOnce(&mut [u8]) -> R,
354 ) -> Result<(R, TxPending<'_>), NetError> {
355 if len > self.config.buf_size {
356 return Err(other_error("tx packet too large"));
357 }
358
359 self.reclaim_bounded(self.capacity().max(1))?;
360
361 let mut buff = self.pool.alloc()?;
362 let bus_addr = buff.dma_addr().as_u64();
363 let ret = buff.write_with_cpu(len, f);
364 Ok((
365 ret,
366 TxPending {
367 queue: self,
368 len,
369 bus_addr,
370 buff: Some(buff),
371 },
372 ))
373 }
374}
375
376pub struct TxPending<'a> {
377 queue: &'a mut TxQueue,
378 len: usize,
379 bus_addr: u64,
380 buff: Option<ContiguousBuffer>,
381}
382
383impl TxPending<'_> {
384 pub fn bus_addr(&self) -> u64 {
385 self.bus_addr
386 }
387
388 pub fn len(&self) -> usize {
389 self.len
390 }
391
392 pub fn is_empty(&self) -> bool {
393 self.len == 0
394 }
395
396 pub fn try_submit(&mut self) -> Result<(), NetError> {
397 self.queue.reclaim_bounded(self.queue.capacity().max(1))?;
398 let buff = self
399 .buff
400 .as_ref()
401 .expect("tx pending buffer should exist until submit succeeds");
402 buff.prepare_for_device(0, self.len);
403 self.queue.interface.submit(DmaBuffer {
404 virt: buff.as_ptr(),
405 bus_addr: self.bus_addr,
406 len: self.len,
407 })?;
408 let buff = self
409 .buff
410 .take()
411 .expect("tx pending buffer should exist until submit succeeds");
412 self.queue.inflight.insert(self.bus_addr, buff);
413 Ok(())
414 }
415}
416
417pub struct RxQueue {
418 interface: Box<dyn IRxQueue>,
419 pool: ContiguousBufferPool,
420 inflight: BTreeMap<u64, ContiguousBuffer>,
421 config: QueueConfig,
422 _waker: Arc<AtomicWaker>,
423}
424
425impl RxQueue {
426 fn capacity(&self) -> usize {
427 self.config.ring_size.saturating_sub(1)
428 }
429
430 fn prefill(&mut self) -> Result<(), NetError> {
431 while self.inflight.len() < self.capacity() {
432 let buff = self.pool.alloc()?;
433 if let Err(err) = self.submit_buffer(buff) {
434 if matches!(err, NetError::Retry) {
435 break;
436 }
437 return Err(err);
438 }
439 }
440 Ok(())
441 }
442
443 fn submit_buffer(&mut self, buff: ContiguousBuffer) -> Result<(), NetError> {
444 let bus_addr = buff.dma_addr().as_u64();
445 let len = self.config.buf_size.min(buff.len());
446 buff.prepare_for_device(0, len);
447 self.interface.submit(DmaBuffer {
448 virt: buff.as_ptr(),
449 bus_addr,
450 len,
451 })?;
452 self.inflight.insert(bus_addr, buff);
453 Ok(())
454 }
455
456 fn reclaim_packet(&mut self) -> Result<Option<(ContiguousBuffer, usize)>, NetError> {
457 let Some((bus_addr, len)) = self.interface.reclaim() else {
458 return Ok(None);
459 };
460 let Some(buff) = self.inflight.remove(&bus_addr) else {
461 return Err(other_error("reclaimed unknown rx buffer"));
462 };
463 let packet_len = len.min(self.config.buf_size).min(buff.len());
464 buff.complete_for_cpu(0, packet_len);
465 Ok(Some((buff, packet_len)))
466 }
467
468 pub fn id(&self) -> usize {
469 self.interface.id()
470 }
471
472 pub fn buf_size(&self) -> usize {
473 self.config.buf_size
474 }
475
476 pub fn try_receive(&mut self) -> Option<RxPacket<'_>> {
477 match self.reclaim_packet() {
478 Ok(Some((buff, len))) => Some(RxPacket {
479 queue: self,
480 len,
481 buff: Some(buff),
482 }),
483 Ok(None) | Err(_) => None,
484 }
485 }
486
487 pub fn receive<R>(&mut self, f: impl FnOnce(&[u8]) -> R) -> Option<R> {
488 let packet = self.try_receive()?;
489 Some(packet.consume(f))
490 }
491}
492
493pub struct RxPacket<'a> {
494 queue: &'a mut RxQueue,
495 len: usize,
496 buff: Option<ContiguousBuffer>,
497}
498
499impl RxPacket<'_> {
500 pub fn len(&self) -> usize {
501 self.len
502 }
503
504 pub fn is_empty(&self) -> bool {
505 self.len == 0
506 }
507
508 pub fn consume<R>(mut self, f: impl FnOnce(&[u8]) -> R) -> R {
509 let buff = self.buff.as_ref().expect("rx packet buffer should exist");
510 let ret = buff.read_with_cpu(self.len, f);
511 if let Some(buff) = self.buff.take() {
512 let _ = self.queue.submit_buffer(buff);
513 }
514 ret
515 }
516}
517
518impl Drop for RxPacket<'_> {
519 fn drop(&mut self) {
520 if let Some(buff) = self.buff.take() {
521 let _ = self.queue.submit_buffer(buff);
522 }
523 }
524}
525
526#[cfg(test)]
527mod tests {
528 use alloc::{boxed::Box, sync::Arc};
529 use core::{
530 any::Any,
531 num::NonZeroUsize,
532 ptr::NonNull,
533 sync::atomic::{AtomicUsize, Ordering},
534 task::{RawWaker, RawWakerVTable, Waker},
535 };
536
537 use dma_api::{DmaAllocHandle, DmaConstraints, DmaDirection, DmaError, DmaMapHandle};
538 use rdif_eth::{DriverGeneric, Event, IRxQueue, ITxQueue, IdList, Interface};
539
540 use super::*;
541
542 struct TestDma;
543
544 impl dma_api::DmaOp for TestDma {
545 fn page_size(&self) -> usize {
546 4096
547 }
548
549 unsafe fn alloc_contiguous(
550 &self,
551 _constraints: DmaConstraints,
552 _layout: core::alloc::Layout,
553 ) -> Option<DmaAllocHandle> {
554 panic!("test should not allocate contiguous DMA")
555 }
556
557 unsafe fn dealloc_contiguous(&self, _handle: DmaAllocHandle) {
558 panic!("test should not deallocate contiguous DMA")
559 }
560
561 unsafe fn alloc_coherent(
562 &self,
563 _constraints: DmaConstraints,
564 _layout: core::alloc::Layout,
565 ) -> Option<DmaAllocHandle> {
566 panic!("test should not allocate coherent DMA")
567 }
568
569 unsafe fn dealloc_coherent(&self, _handle: DmaAllocHandle) {
570 panic!("test should not deallocate coherent DMA")
571 }
572
573 unsafe fn map_streaming(
574 &self,
575 _constraints: DmaConstraints,
576 _addr: NonNull<u8>,
577 _size: NonZeroUsize,
578 _direction: DmaDirection,
579 ) -> Result<DmaMapHandle, DmaError> {
580 panic!("test should not map streaming DMA")
581 }
582
583 unsafe fn unmap_streaming(&self, _handle: DmaMapHandle) {
584 panic!("test should not unmap streaming DMA")
585 }
586 }
587
588 struct TestInterface {
589 irq_events: Event,
590 handle_calls: Arc<AtomicUsize>,
591 owned_irq_handler: Option<rdif_eth::BIrqHandler>,
592 }
593
594 impl DriverGeneric for TestInterface {
595 fn name(&self) -> &str {
596 "test-net"
597 }
598
599 fn raw_any(&self) -> Option<&dyn Any> {
600 Some(self)
601 }
602
603 fn raw_any_mut(&mut self) -> Option<&mut dyn Any> {
604 Some(self)
605 }
606 }
607
608 impl Interface for TestInterface {
609 fn mac_address(&self) -> [u8; 6] {
610 [0x02, 0, 0, 0, 0, 1]
611 }
612
613 fn create_tx_queue(&mut self) -> Option<Box<dyn ITxQueue>> {
614 panic!("test should not create TX queue")
615 }
616
617 fn create_rx_queue(&mut self) -> Option<Box<dyn IRxQueue>> {
618 panic!("test should not create RX queue")
619 }
620
621 fn enable_irq(&mut self) {}
622
623 fn disable_irq(&mut self) {}
624
625 fn is_irq_enabled(&self) -> bool {
626 false
627 }
628
629 fn handle_irq(&mut self) -> Event {
630 self.handle_calls.fetch_add(1, Ordering::AcqRel);
631 self.irq_events
632 }
633
634 fn take_irq_handler(&mut self) -> Option<rdif_eth::BIrqHandler> {
635 self.owned_irq_handler.take()
636 }
637 }
638
639 struct OwnedTestIrqHandler {
640 irq_events: Event,
641 handle_calls: Arc<AtomicUsize>,
642 }
643
644 impl rdif_eth::IrqHandler for OwnedTestIrqHandler {
645 fn handle_irq(&mut self) -> Event {
646 self.handle_calls.fetch_add(1, Ordering::AcqRel);
647 self.irq_events
648 }
649 }
650
651 fn count_waker(counter: Arc<AtomicUsize>) -> Waker {
652 unsafe fn clone(data: *const ()) -> RawWaker {
653 let counter = unsafe { Arc::<AtomicUsize>::from_raw(data.cast()) };
654 let cloned = Arc::clone(&counter);
655 let _ = Arc::into_raw(counter);
656 RawWaker::new(Arc::into_raw(cloned).cast(), &VTABLE)
657 }
658
659 unsafe fn wake(data: *const ()) {
660 let counter = unsafe { Arc::<AtomicUsize>::from_raw(data.cast()) };
661 counter.fetch_add(1, Ordering::AcqRel);
662 }
663
664 unsafe fn wake_by_ref(data: *const ()) {
665 let counter = unsafe { Arc::<AtomicUsize>::from_raw(data.cast()) };
666 counter.fetch_add(1, Ordering::AcqRel);
667 let _ = Arc::into_raw(counter);
668 }
669
670 unsafe fn drop(data: *const ()) {
671 let _ = unsafe { Arc::<AtomicUsize>::from_raw(data.cast()) };
672 }
673
674 static VTABLE: RawWakerVTable = RawWakerVTable::new(clone, wake, wake_by_ref, drop);
675 let raw = RawWaker::new(Arc::into_raw(counter).cast(), &VTABLE);
676 unsafe { Waker::from_raw(raw) }
677 }
678
679 #[test]
680 fn irq_handler_fast_path_returns_events_without_waking_registered_wakers() {
681 static DMA: TestDma = TestDma;
682 let mut rx = IdList::none();
683 rx.insert(3);
684 let mut tx = IdList::none();
685 tx.insert(5);
686 let interface_calls = Arc::new(AtomicUsize::new(0));
687 let irq_calls = Arc::new(AtomicUsize::new(0));
688 let mut net = Net::new(
689 TestInterface {
690 irq_events: Event::none(),
691 handle_calls: Arc::clone(&interface_calls),
692 owned_irq_handler: Some(Box::new(OwnedTestIrqHandler {
693 irq_events: Event {
694 tx_queue: tx,
695 rx_queue: rx,
696 },
697 handle_calls: Arc::clone(&irq_calls),
698 })),
699 },
700 &DMA,
701 );
702 let rx_wake_count = Arc::new(AtomicUsize::new(0));
703 let tx_wake_count = Arc::new(AtomicUsize::new(0));
704 net.inner
705 .rx_wakers
706 .register(3)
707 .register(&count_waker(Arc::clone(&rx_wake_count)));
708 net.inner
709 .tx_wakers
710 .register(5)
711 .register(&count_waker(Arc::clone(&tx_wake_count)));
712
713 let mut irq = net.take_irq_handler().unwrap();
714 let events = irq.handle_irq();
715
716 assert!(events.rx_queue.contains(3));
717 assert!(events.tx_queue.contains(5));
718 assert_eq!(irq_calls.load(Ordering::Acquire), 1);
719 assert_eq!(interface_calls.load(Ordering::Acquire), 0);
720 assert_eq!(rx_wake_count.load(Ordering::Acquire), 0);
721 assert_eq!(tx_wake_count.load(Ordering::Acquire), 0);
722 }
723
724 #[test]
725 fn irq_handler_requires_owned_endpoint() {
726 static DMA: TestDma = TestDma;
727 let handle_calls = Arc::new(AtomicUsize::new(0));
728 let mut net = Net::new(
729 TestInterface {
730 irq_events: Event::none(),
731 handle_calls,
732 owned_irq_handler: None,
733 },
734 &DMA,
735 );
736
737 assert!(net.take_irq_handler().is_none());
738 }
739
740 #[test]
741 fn irq_handler_uses_owned_endpoint() {
742 static DMA: TestDma = TestDma;
743 let mut rx = IdList::none();
744 rx.insert(1);
745 let mut tx = IdList::none();
746 tx.insert(2);
747 let fallback_calls = Arc::new(AtomicUsize::new(0));
748 let owned_calls = Arc::new(AtomicUsize::new(0));
749 let mut net = Net::new(
750 TestInterface {
751 irq_events: Event::none(),
752 handle_calls: Arc::clone(&fallback_calls),
753 owned_irq_handler: Some(Box::new(OwnedTestIrqHandler {
754 irq_events: Event {
755 tx_queue: tx,
756 rx_queue: rx,
757 },
758 handle_calls: Arc::clone(&owned_calls),
759 })),
760 },
761 &DMA,
762 );
763
764 let mut irq = net.take_irq_handler().unwrap();
765 let events = irq.handle_irq();
766
767 assert!(events.rx_queue.contains(1));
768 assert!(events.tx_queue.contains(2));
769 assert_eq!(owned_calls.load(Ordering::Acquire), 1);
770 assert_eq!(fallback_calls.load(Ordering::Acquire), 0);
771 }
772}