Skip to main content

rusty_rtos_kernel_core/
queue.rs

1//! `queue.c`: the queue, and the semaphore and mutex built on it.
2//!
3//! In FreeRTOS a semaphore *is* a queue with a zero-size item, and a mutex
4//! is that plus a holder and priority inheritance. Kairos keeps the same
5//! shape, because the trace does: a semaphore give fires `QUEUE_SEND` and a
6//! mutex take fires `QUEUE_RECEIVE`, and a conformance diff would notice
7//! any tidier arrangement.
8//!
9//! # A blocking call with no stack to block on
10//!
11//! `xQueueReceive( q, &v, 100 )` blocks: the C task stops inside the call,
12//! and when it is woken it loops back to the top, re-checks the queue, and
13//! either returns the item or blocks again. This kernel has no stack to
14//! suspend, so the loop is turned inside out — a blocking call returns
15//! [`Blocked`] and the caller invokes it again, unchanged, when it next
16//! runs:
17//!
18//! ```ignore
19//! // in a task body, at some `pc`:
20//! match k.queue_receive(q, 100)? {
21//!     Blocked => {}                 // stay at this pc; the kernel parked us
22//!     Ready(value) => { ...; self.pc = next; }
23//! }
24//! ```
25//!
26//! The task is off the ready list when `Blocked` comes back, so the runner
27//! will not step it again until the kernel wakes it — which is exactly when
28//! the C thread would have resumed. What the C keeps on its stack across
29//! the block (the timeout, the remaining ticks, whether inheritance
30//! happened) this kernel keeps in the TCB, in [`Wait`], because that *is*
31//! the stack frame.
32
33use rusty_rtos_core::config::Config;
34use rusty_rtos_core::error::{Error, Result};
35use rusty_rtos_core::handle::{QueueHandle, TaskHandle};
36use rusty_rtos_core::hooks::TickHook;
37use rusty_rtos_core::isr::Woken;
38use rusty_rtos_core::port::Port;
39use rusty_rtos_core::trace::{Event, Trace};
40
41use crate::kernel::{Kernel, OwedTrace};
42
43/// What a blocking call answers.
44#[derive(Debug, Clone, Copy, PartialEq, Eq)]
45pub enum Wait<T> {
46    /// The kernel parked the calling task. Leave the program counter where
47    /// it is and make the same call again when the task next runs.
48    Blocked,
49    /// The call completed.
50    Ready(T),
51}
52
53pub use Wait::{Blocked, Ready};
54
55/// Where an item goes (`xCopyPosition`).
56#[derive(Debug, Clone, Copy, PartialEq, Eq)]
57pub enum Position {
58    /// `queueSEND_TO_BACK`.
59    Back,
60    /// `queueSEND_TO_FRONT`.
61    Front,
62    /// `queueOVERWRITE`: write to the front of a length-one queue whether
63    /// or not it already holds something, and leave the message count
64    /// where it was rather than raising it.
65    Overwrite,
66}
67
68/// What a queue is underneath (`ucQueueType`).
69#[derive(Debug, Clone, Copy, PartialEq, Eq)]
70pub enum Kind {
71    /// A queue of values.
72    Queue,
73    /// A binary or counting semaphore: a queue with a zero-size item, so
74    /// only the count matters.
75    Semaphore,
76    /// A mutex: a one-deep semaphore with a holder and priority
77    /// inheritance.
78    Mutex,
79    /// A recursive mutex: as above, and the holder may take it again.
80    RecursiveMutex,
81    /// A queue set: a queue whose items are the handles of the queues in
82    /// it. `xQueueSelectFromSet` is a receive from this one.
83    Set,
84}
85
86impl Kind {
87    /// `uxItemSize == 0`: semaphores and mutexes carry a count, not data.
88    pub(crate) const fn carries_data(self) -> bool {
89        matches!(self, Self::Queue | Self::Set)
90    }
91
92    /// The two mutex kinds, which inherit priority.
93    pub(crate) const fn is_mutex(self) -> bool {
94        matches!(self, Self::Mutex | Self::RecursiveMutex)
95    }
96}
97
98/// One queue: the C `Queue_t` minus the byte pointers. Storage is a range
99/// of the kernel's shared slot pool, and the read and write cursors follow
100/// C's `pcReadFrom` / `pcWriteTo` exactly — `pcReadFrom` points at the item
101/// last read, which is what makes `queueSEND_TO_FRONT` a single step back.
102#[derive(Debug, Clone, Copy)]
103pub(crate) struct Queue {
104    pub(crate) base: usize,
105    /// `uxLength`.
106    pub(crate) length: usize,
107    /// `uxMessagesWaiting`.
108    pub(crate) waiting: usize,
109    /// `pcReadFrom`, as an index: the item last read.
110    pub(crate) read_from: usize,
111    /// `pcWriteTo`, as an index: where the next back-send writes.
112    pub(crate) write_to: usize,
113    pub(crate) kind: Kind,
114    /// `u.xSemaphore.xMutexHolder`.
115    pub(crate) holder: TaskHandle,
116    /// `u.xSemaphore.uxRecursiveCallCount`.
117    pub(crate) recursions: u32,
118    /// `cTxLock`: [`UNLOCKED`] when no task holds the queue locked;
119    /// otherwise how many sends an interrupt made while it was locked, and
120    /// therefore how many receivers `prvUnlockQueue` still owes a wake-up.
121    pub(crate) tx_lock: i8,
122    /// `cRxLock`, the same for receives and the tasks waiting to send.
123    pub(crate) rx_lock: i8,
124    /// `pxQueueSetContainer`: the set this queue belongs to, if any. An
125    /// item arriving here is announced there.
126    pub(crate) set_container: QueueHandle,
127}
128
129/// `queueUNLOCKED`.
130pub(crate) const UNLOCKED: i8 = -1;
131/// `queueLOCKED_UNMODIFIED`.
132pub(crate) const LOCKED_UNMODIFIED: i8 = 0;
133
134impl Queue {
135    pub(crate) const fn new(base: usize, length: usize, kind: Kind) -> Self {
136        Self {
137            base,
138            length,
139            waiting: 0,
140            // `xQueueGenericReset`: pcReadFrom = pcHead + (length - 1) * itemSize.
141            read_from: length.saturating_sub(1),
142            write_to: 0,
143            kind,
144            holder: TaskHandle::NULL,
145            recursions: 0,
146            tx_lock: UNLOCKED,
147            rx_lock: UNLOCKED,
148            set_container: QueueHandle::NULL,
149        }
150    }
151}
152
153impl<
154    C: Config,
155    P: Port,
156    T: Trace,
157    H,
158    const TASKS: usize,
159    const ITEMS: usize,
160    const LISTS: usize,
161    const QUEUES: usize,
162    const SLOTS: usize,
163    const BUFFERS: usize,
164    const BYTES: usize,
165    const TIMERS: usize,
166    const GROUPS: usize,
167> Kernel<C, P, T, H, TASKS, ITEMS, LISTS, QUEUES, SLOTS, BUFFERS, BYTES, TIMERS, GROUPS>
168where
169    H: TickHook<Self>,
170{
171    // ------------------------------------------------------------ create --
172
173    /// `xQueueCreate`.
174    ///
175    /// # Errors
176    /// [`Error::Full`] when the queue arena or the shared slot pool is
177    /// exhausted; [`Error::InvalidArgument`] for a zero length.
178    pub fn queue_create(&mut self, length: usize) -> Result<QueueHandle> {
179        self.new_queue(length, Kind::Queue)
180    }
181
182    /// `vQueueDelete`, and `vSemaphoreDelete`, which the C defines as the
183    /// same function.
184    ///
185    /// # It traces nothing and still costs time
186    ///
187    /// `vQueueDelete` takes no critical section of its own — but it ends in
188    /// `vPortFree( pxQueue )`, and every `heap_N.c` wraps free in
189    /// `vTaskSuspendAll` / `xTaskResumeAll` exactly as it wraps malloc. So
190    /// a delete costs **one outermost critical-section exit**, which under
191    /// the sim contract is one unit of time, while emitting no trace line
192    /// at all (this harness defines no `traceQUEUE_DELETE`).
193    ///
194    /// `AbortDelay`'s remake left the deletes out on the reasoning that an
195    /// event-less call cannot move the trace. Every event still agreed and
196    /// the exit column was one short from the first delete onwards. The
197    /// sibling `event_group_delete` already carried this note; the queue
198    /// had no delete at all.
199    ///
200    /// # Errors
201    /// [`Error::Gone`] for a stale handle.
202    pub fn queue_delete(&mut self, queue: QueueHandle) -> Result<()> {
203        self.queues.resolve(queue)?;
204        let _ = self.queues.remove(queue);
205        // `vPortFree( pxQueue )`.
206        self.account_for_allocation();
207        Ok(())
208    }
209
210    /// `xSemaphoreCreateBinary`: a one-deep semaphore, created empty.
211    ///
212    /// # Errors
213    /// As [`Kernel::queue_create`].
214    pub fn semaphore_create_binary(&mut self) -> Result<QueueHandle> {
215        self.new_queue(1, Kind::Semaphore)
216    }
217
218    /// `xSemaphoreCreateCounting`.
219    ///
220    /// # Errors
221    /// As [`Kernel::queue_create`]; [`Error::InvalidArgument`] if the
222    /// initial count exceeds the maximum.
223    pub fn semaphore_create_counting(&mut self, max: usize, initial: usize) -> Result<QueueHandle> {
224        if initial > max {
225            return Err(Error::InvalidArgument);
226        }
227        let handle = self.new_queue(max, Kind::Semaphore)?;
228        if let Ok(q) = self.queues.resolve_mut(handle) {
229            q.waiting = initial;
230        }
231        Ok(handle)
232    }
233
234    /// `xSemaphoreCreateMutex`: created **available**, unlike a semaphore.
235    ///
236    /// # Errors
237    /// As [`Kernel::queue_create`].
238    pub fn mutex_create(&mut self) -> Result<QueueHandle> {
239        self.new_mutex(Kind::Mutex)
240    }
241
242    /// `xSemaphoreCreateRecursiveMutex`.
243    ///
244    /// # Errors
245    /// As [`Kernel::queue_create`].
246    pub fn mutex_create_recursive(&mut self) -> Result<QueueHandle> {
247        self.new_mutex(Kind::RecursiveMutex)
248    }
249
250    fn new_mutex(&mut self, kind: Kind) -> Result<QueueHandle> {
251        let handle = self.new_queue(1, kind)?;
252        // `prvInitialiseMutex` gives the mutex once so it starts available,
253        // and it does so through `xQueueGenericSend` — which fires
254        // `traceQUEUE_SEND`. A mutex therefore has a `QUEUE_SEND` line
255        // immediately after its `QUEUE_CREATE`, before any task exists.
256        let _ = self.queue_send_generic(handle, 0, 0, Position::Back)?;
257        Ok(handle)
258    }
259
260    fn new_queue(&mut self, length: usize, kind: Kind) -> Result<QueueHandle> {
261        if length == 0 {
262            return Err(Error::InvalidArgument);
263        }
264        // A zero-item-size queue needs no storage; only real queues do.
265        let slots = if kind.carries_data() { length } else { 0 };
266        let base = self.slots_used;
267        let end = base.checked_add(slots).ok_or(Error::Full)?;
268        if end > SLOTS {
269            return Err(Error::Full);
270        }
271        let handle = self
272            .queues
273            .try_insert(Queue::new(base, length, kind))
274            .map_err(|_| Error::Full)?;
275        self.slots_used = end;
276        // `xQueueGenericCreate` takes the queue and its storage from the
277        // heap before it initialises anything.
278        self.account_for_allocation();
279        // `xQueueGenericReset` runs in a critical section before
280        // `traceQUEUE_CREATE` fires at the end of `prvInitialiseNewQueue`.
281        self.enter_critical();
282        self.exit_critical();
283        let tick = self.tick;
284        self.trace.note_exits(self.port.exits());
285        self.trace.event(
286            tick,
287            Event::QueueCreate {
288                queue: handle,
289                name: "",
290                length,
291            },
292        );
293        Ok(handle)
294    }
295
296    // ----------------------------------------------------------- getters --
297
298    /// `uxQueueMessagesWaiting`, which takes a critical section.
299    ///
300    /// # Errors
301    /// [`Error::Gone`] for a stale handle.
302    pub fn queue_messages_waiting(&mut self, queue: QueueHandle) -> Result<usize> {
303        self.enter_critical();
304        let n = self.queues.resolve(queue).map(|q| q.waiting);
305        self.exit_critical();
306        n
307    }
308
309    /// `uxSemaphoreGetCount`, which is the same thing.
310    ///
311    /// # Errors
312    /// [`Error::Gone`] for a stale handle.
313    pub fn semaphore_count(&mut self, semaphore: QueueHandle) -> Result<usize> {
314        self.queue_messages_waiting(semaphore)
315    }
316
317    /// `xSemaphoreGetMutexHolderFromISR`, which is
318    /// `xQueueGetMutexHolderFromISR`: the same answer, read without a
319    /// critical section because an ISR is already inside one.
320    ///
321    /// # Errors
322    /// [`Error::Gone`] for a stale handle.
323    pub fn mutex_holder_from_isr(&mut self, mutex: QueueHandle) -> Result<TaskHandle> {
324        self.queues.resolve(mutex).map(|q| q.holder)
325    }
326
327    /// `xSemaphoreGetMutexHolder`, which takes a critical section.
328    ///
329    /// # Errors
330    /// [`Error::Gone`] for a stale handle.
331    pub fn mutex_holder(&mut self, mutex: QueueHandle) -> Result<TaskHandle> {
332        self.enter_critical();
333        let holder = self.queues.resolve(mutex).map(|q| q.holder);
334        self.exit_critical();
335        holder
336    }
337
338    // -------------------------------------------------------------- send --
339
340    /// `xQueueSend` / `xQueueSendToBack`.
341    ///
342    /// # Errors
343    /// [`Error::Full`] when the block time expires with the queue still
344    /// full; [`Error::Gone`] for a stale handle.
345    pub fn queue_send(&mut self, queue: QueueHandle, value: u64, ticks: u64) -> Result<Wait<()>> {
346        self.queue_send_generic(queue, value, ticks, Position::Back)
347    }
348
349    /// `xQueueReset`, which is `xQueueGenericReset( q, pdFALSE )`: empty
350    /// the queue and unlock it, waking one waiting sender if there is one.
351    ///
352    /// The event lists survive — only a brand-new queue re-initialises
353    /// those — so a task blocked on this queue stays blocked, and the one
354    /// woken here is woken because the reset made room.
355    ///
356    /// # Errors
357    /// [`Error::Gone`] for a stale handle.
358    pub fn queue_reset(&mut self, queue: QueueHandle) -> Result<()> {
359        self.enter_critical();
360        let result = (|| {
361            let length = self.queues.resolve(queue)?.length;
362            {
363                let q = self.queues.resolve_mut(queue)?;
364                q.waiting = 0;
365                q.write_to = 0;
366                q.read_from = length.saturating_sub(1);
367                q.rx_lock = UNLOCKED;
368                q.tx_lock = UNLOCKED;
369            }
370            let senders = Self::queue_send_list(queue);
371            if self.lists.is_empty(senders) == Ok(false) && self.remove_from_event_list(senders)? {
372                // queueYIELD_IF_USING_PREEMPTION(), inside the section.
373                self.port_yield();
374            }
375            Ok(())
376        })();
377        self.exit_critical();
378        result
379    }
380
381    /// `uxQueueSpacesAvailable`: how many more items would fit.
382    ///
383    /// # Errors
384    /// [`Error::Gone`] for a stale handle.
385    pub fn queue_spaces_available(&mut self, queue: QueueHandle) -> Result<usize> {
386        self.enter_critical();
387        let spaces = self
388            .queues
389            .resolve(queue)
390            .map(|q| q.length.saturating_sub(q.waiting));
391        self.exit_critical();
392        spaces
393    }
394
395    // --------------------------------------------------- queue sets --
396    //
397    // A queue set is a queue whose items are the handles of the queues in
398    // it. Nothing is *moved* into the set: an item arriving on a member
399    // queue puts that queue's handle on the set, and the task that wakes on
400    // the set reads the handle and then reads the member queue itself. So a
401    // set holds an announcement, not the data — which is why adding a queue
402    // that already has items in it is refused, and why the two counts would
403    // otherwise drift apart forever.
404
405    /// `xQueueCreateSet`: a set that can hold `length` announcements.
406    ///
407    /// # Errors
408    /// As [`Kernel::queue_create`].
409    pub fn queue_create_set(&mut self, length: usize) -> Result<QueueHandle> {
410        self.new_queue(length, Kind::Set)
411    }
412
413    /// `xQueueAddToSet`.
414    ///
415    /// `false` is the C's `pdFAIL`: the target is not a set, the queue is
416    /// already in one, or it has items waiting that the set never saw.
417    ///
418    /// # Errors
419    /// [`Error::Gone`] for a stale handle.
420    pub fn queue_add_to_set(&mut self, queue: QueueHandle, set: QueueHandle) -> Result<bool> {
421        self.enter_critical();
422        let result = (|| {
423            if self.queues.resolve(set)?.kind != Kind::Set {
424                return Ok(false);
425            }
426            let member = self.queues.resolve(queue)?;
427            if member.set_container != QueueHandle::NULL || member.waiting != 0 {
428                return Ok(false);
429            }
430            self.queues.resolve_mut(queue)?.set_container = set;
431            Ok(true)
432        })();
433        self.exit_critical();
434        result
435    }
436
437    /// `xQueueRemoveFromSet`.
438    ///
439    /// # Errors
440    /// [`Error::Gone`] for a stale handle.
441    pub fn queue_remove_from_set(&mut self, queue: QueueHandle, set: QueueHandle) -> Result<bool> {
442        self.enter_critical();
443        let result = (|| {
444            let member = self.queues.resolve(queue)?;
445            if member.set_container != set || member.waiting != 0 {
446                return Ok(false);
447            }
448            self.queues.resolve_mut(queue)?.set_container = QueueHandle::NULL;
449            Ok(true)
450        })();
451        self.exit_critical();
452        result
453    }
454
455    /// `xQueueSelectFromSet`: which member queue has something on it.
456    ///
457    /// The C is one line — `xQueueReceive( xQueueSet, &xReturn, xTicksToWait )`
458    /// — so this blocks, traces and costs exactly what a receive does, and
459    /// `None` is the C's `NULL`.
460    ///
461    /// # Errors
462    /// As [`Kernel::queue_receive`].
463    pub fn queue_select_from_set(
464        &mut self,
465        set: QueueHandle,
466        ticks: u64,
467    ) -> Result<Wait<Option<QueueHandle>>> {
468        match self.queue_receive(set, ticks) {
469            Ok(Blocked) => Ok(Blocked),
470            Ok(Ready(raw)) => {
471                let handle = QueueHandle::from_raw(u32::try_from(raw).unwrap_or(0));
472                Ok(Ready((handle != QueueHandle::NULL).then_some(handle)))
473            }
474            Err(Error::Empty) => Ok(Ready(None)),
475            Err(e) => Err(e),
476        }
477    }
478
479    /// `prvNotifyQueueSetContainer`: put `queue`'s handle on the set it
480    /// belongs to. `true` when that woke a task that outranks the current
481    /// one.
482    fn notify_queue_set_container(&mut self, queue: QueueHandle) -> Result<bool> {
483        let set = self.queues.resolve(queue)?.set_container;
484        let container = *self.queues.resolve(set)?;
485        if container.waiting >= container.length {
486            // The C asserts this cannot happen and does nothing if it does.
487            return Ok(false);
488        }
489        let tx_lock = container.tx_lock;
490        let tick = self.tick;
491        self.trace.note_exits(self.port.exits());
492        // `traceQUEUE_SET_SEND` is `traceQUEUE_SEND` unless a port says
493        // otherwise, and this port does not.
494        self.trace.event(
495            tick,
496            Event::QueueSend {
497                queue: set,
498                name: "",
499            },
500        );
501        let mut woke = self.copy_data_to_queue(set, u64::from(queue.to_raw()), Position::Back)?;
502        if tx_lock == UNLOCKED {
503            let receivers = Self::queue_receive_list(set);
504            if self.lists.is_empty(receivers) == Ok(false)
505                && self.remove_from_event_list(receivers)?
506            {
507                woke = true;
508            }
509        } else {
510            self.increment_tx_lock(set, tx_lock);
511        }
512        Ok(woke)
513    }
514
515    /// `xQueueOverwrite`: write to a length-one queue whether or not it
516    /// already holds something.
517    ///
518    /// The C is `xQueueGenericSend( q, item, 0, queueOVERWRITE )` — a send
519    /// with no block time, because a queue that is always writable can
520    /// never make the caller wait.
521    ///
522    /// # Errors
523    /// As [`Kernel::queue_send`].
524    pub fn queue_overwrite(&mut self, queue: QueueHandle, value: u64) -> Result<Wait<()>> {
525        self.queue_send_generic(queue, value, 0, Position::Overwrite)
526    }
527
528    // ------------------------------------------------------- from an ISR --
529    //
530    // The `FromISR` half of the API. Three things make it a different
531    // animal from the task half, and all three are visible in a trace:
532    //
533    //   * It never blocks and never yields. It reports that a higher
534    //     priority task woke, by returning a [`Woken`], and the interrupt
535    //     decides what to do about it on the way out.
536    //   * Its critical section is `portSET_INTERRUPT_MASK_FROM_ISR`, not
537    //     `portENTER_CRITICAL`. On the Posix port both are empty, so a call
538    //     here costs no sim time at all — which is exactly what the C does
539    //     and therefore what a matching trace requires.
540    //   * If a task has the queue locked it may not touch the event lists.
541    //     It counts what it did in `cTxLock` / `cRxLock` instead, and
542    //     `prvUnlockQueue` pays the wake-ups back when the task is done.
543
544    /// `xQueueGenericSendFromISR`.
545    ///
546    /// # Errors
547    /// [`Error::Full`] when the queue is full and this is not an overwrite;
548    /// [`Error::Gone`] for a stale handle.
549    pub fn queue_send_generic_from_isr(
550        &mut self,
551        queue: QueueHandle,
552        value: u64,
553        position: Position,
554    ) -> Result<Woken> {
555        let mask = self.port.enter_critical_from_isr();
556        let result = self.send_from_isr_locked(queue, value, position);
557        self.port.exit_critical_from_isr(mask);
558        result
559    }
560
561    fn send_from_isr_locked(
562        &mut self,
563        queue: QueueHandle,
564        value: u64,
565        position: Position,
566    ) -> Result<Woken> {
567        let snapshot = *self.queues.resolve(queue)?;
568        if snapshot.waiting >= snapshot.length && position != Position::Overwrite {
569            // `traceQUEUE_SEND_FROM_ISR_FAILED` is not one of the harness's
570            // hooks, so a full queue says nothing on either side.
571            return Err(Error::Full);
572        }
573        let tx_lock = snapshot.tx_lock;
574        let tick = self.tick;
575        self.trace.note_exits(self.port.exits());
576        self.trace
577            .event(tick, Event::QueueSendFromIsr { queue, name: "" });
578        let previously_waiting = snapshot.waiting;
579        let _ = self.copy_data_to_queue(queue, value, position)?;
580        let mut woken = Woken::NO;
581        if tx_lock == UNLOCKED {
582            if snapshot.set_container != QueueHandle::NULL {
583                let overwrote = position == Position::Overwrite && previously_waiting > 0;
584                if !overwrote && self.notify_queue_set_container(queue)? {
585                    woken = Woken::YES;
586                }
587                return Ok(woken);
588            }
589            let receivers = Self::queue_receive_list(queue);
590            if self.lists.is_empty(receivers) == Ok(false)
591                && self.remove_from_event_list(receivers)?
592            {
593                woken = Woken::YES;
594            }
595        } else {
596            self.increment_tx_lock(queue, tx_lock);
597        }
598        Ok(woken)
599    }
600
601    /// `xQueueSendToBackFromISR`.
602    ///
603    /// # Errors
604    /// As [`Kernel::queue_send_generic_from_isr`].
605    pub fn queue_send_from_isr(&mut self, queue: QueueHandle, value: u64) -> Result<Woken> {
606        self.queue_send_generic_from_isr(queue, value, Position::Back)
607    }
608
609    /// `xQueueSendToFrontFromISR`.
610    ///
611    /// # Errors
612    /// As [`Kernel::queue_send_generic_from_isr`].
613    pub fn queue_send_to_front_from_isr(
614        &mut self,
615        queue: QueueHandle,
616        value: u64,
617    ) -> Result<Woken> {
618        self.queue_send_generic_from_isr(queue, value, Position::Front)
619    }
620
621    /// `xQueueOverwriteFromISR`. Only for a queue of length one.
622    ///
623    /// # Errors
624    /// As [`Kernel::queue_send_generic_from_isr`].
625    pub fn queue_overwrite_from_isr(&mut self, queue: QueueHandle, value: u64) -> Result<Woken> {
626        self.queue_send_generic_from_isr(queue, value, Position::Overwrite)
627    }
628
629    /// `xSemaphoreGiveFromISR`.
630    ///
631    /// # Errors
632    /// As [`Kernel::queue_send_generic_from_isr`].
633    pub fn semaphore_give_from_isr(&mut self, semaphore: QueueHandle) -> Result<Woken> {
634        self.queue_send_generic_from_isr(semaphore, 0, Position::Back)
635    }
636
637    /// `xQueueReceiveFromISR`: the value, and whether a waiting sender that
638    /// this made room for outranks the interrupted task.
639    ///
640    /// # Errors
641    /// [`Error::Empty`] when the queue is empty; [`Error::Gone`] for a
642    /// stale handle.
643    pub fn queue_receive_from_isr(&mut self, queue: QueueHandle) -> Result<(u64, Woken)> {
644        let mask = self.port.enter_critical_from_isr();
645        let result = self.receive_from_isr_locked(queue);
646        self.port.exit_critical_from_isr(mask);
647        result
648    }
649
650    fn receive_from_isr_locked(&mut self, queue: QueueHandle) -> Result<(u64, Woken)> {
651        let snapshot = *self.queues.resolve(queue)?;
652        if snapshot.waiting == 0 {
653            return Err(Error::Empty);
654        }
655        let rx_lock = snapshot.rx_lock;
656        let tick = self.tick;
657        self.trace.note_exits(self.port.exits());
658        self.trace
659            .event(tick, Event::QueueReceiveFromIsr { queue, name: "" });
660        let value = self.copy_data_from_queue(queue, &snapshot, false)?;
661        let mut woken = Woken::NO;
662        if rx_lock == UNLOCKED {
663            let senders = Self::queue_send_list(queue);
664            if self.lists.is_empty(senders) == Ok(false) && self.remove_from_event_list(senders)? {
665                woken = Woken::YES;
666            }
667        } else {
668            self.increment_rx_lock(queue, rx_lock);
669        }
670        Ok((value, woken))
671    }
672
673    /// `xQueuePeekFromISR`: the head without removing it.
674    ///
675    /// It wakes nobody and traces nothing — the C has no hook for it —
676    /// which is why it returns a bare value rather than a [`Woken`].
677    ///
678    /// # Errors
679    /// [`Error::Empty`] when the queue is empty; [`Error::Gone`] for a
680    /// stale handle.
681    pub fn queue_peek_from_isr(&mut self, queue: QueueHandle) -> Result<u64> {
682        let mask = self.port.enter_critical_from_isr();
683        let result = match self.queues.resolve(queue) {
684            Ok(q) if q.waiting > 0 => {
685                let snapshot = *q;
686                self.copy_data_from_queue(queue, &snapshot, true)
687            }
688            Ok(_) => Err(Error::Empty),
689            Err(e) => Err(e),
690        };
691        self.port.exit_critical_from_isr(mask);
692        result
693    }
694
695    /// `prvIncrementQueueTxLock`, capped at the task count as the C caps it.
696    fn increment_tx_lock(&mut self, queue: QueueHandle, tx_lock: i8) {
697        let tasks = self.task_count();
698        if let Ok(q) = self.queues.resolve_mut(queue) {
699            if i64::from(tx_lock) < tasks as i64 {
700                q.tx_lock = tx_lock.saturating_add(1);
701            }
702        }
703    }
704
705    /// `prvIncrementQueueRxLock`.
706    fn increment_rx_lock(&mut self, queue: QueueHandle, rx_lock: i8) {
707        let tasks = self.task_count();
708        if let Ok(q) = self.queues.resolve_mut(queue) {
709            if i64::from(rx_lock) < tasks as i64 {
710                q.rx_lock = rx_lock.saturating_add(1);
711            }
712        }
713    }
714
715    /// `xQueueSendToFront`.
716    ///
717    /// # Errors
718    /// As [`Kernel::queue_send`].
719    pub fn queue_send_to_front(
720        &mut self,
721        queue: QueueHandle,
722        value: u64,
723        ticks: u64,
724    ) -> Result<Wait<()>> {
725        self.queue_send_generic(queue, value, ticks, Position::Front)
726    }
727
728    /// `xSemaphoreGive`: a send with no data.
729    ///
730    /// # Errors
731    /// [`Error::Full`] when the semaphore is already at its maximum count.
732    pub fn semaphore_give(&mut self, semaphore: QueueHandle) -> Result<Wait<()>> {
733        self.queue_send_generic(semaphore, 0, 0, Position::Back)
734    }
735
736    /// `xQueueGenericSend`, one pass of its `for(;;)`.
737    ///
738    /// # Errors
739    /// [`Error::Full`] on timeout; [`Error::Gone`] for a stale handle.
740    pub fn queue_send_generic(
741        &mut self,
742        queue: QueueHandle,
743        value: u64,
744        ticks: u64,
745        position: Position,
746    ) -> Result<Wait<()>> {
747        let caller = self.current;
748        self.enter_critical();
749        let snapshot = match self.queues.resolve(queue) {
750            Ok(q) => *q,
751            Err(e) => {
752                self.exit_critical();
753                return Err(e);
754            }
755        };
756        // The wait frame is set where the C sets it: `xQueueReceive` and
757        // `xQueueGenericSend` call `vTaskInternalSetTimeOutState` only
758        // after finding the queue unusable AND the block time non-zero,
759        // guarded by `xEntryTimeSet`. Setting it up here instead meant
760        // every *successful* call wrote a six-field frame and then had
761        // `end_wait` wipe it on the way out.
762        //
763        // It also has to be after the resolve, so that a handle naming
764        // nothing leaves the caller exactly as it found it -- which is
765        // what `configASSERT( pxQueue )` means in the C.
766        // `( uxMessagesWaiting < uxLength ) || ( xCopyPosition == queueOVERWRITE )`
767        if snapshot.waiting < snapshot.length || position == Position::Overwrite {
768            self.trace.note_exits(self.port.exits());
769            let tick = self.tick;
770            self.trace.event(tick, Event::QueueSend { queue, name: "" });
771            let previously_waiting = snapshot.waiting;
772            let yield_required = self.copy_data_to_queue(queue, value, position)?;
773            if snapshot.set_container != QueueHandle::NULL {
774                // A queue in a set announces the arrival there, not here.
775                // An overwrite of an item that was already present is not an
776                // arrival: the count did not change, so the set is not told.
777                let overwrote = position == Position::Overwrite && previously_waiting > 0;
778                if !overwrote && self.notify_queue_set_container(queue)? {
779                    self.port_yield();
780                }
781                self.exit_critical();
782                self.end_wait(caller);
783                return Ok(Ready(()));
784            }
785            let receivers = Self::queue_receive_list(queue);
786            let woke_higher = if self.lists.is_empty(receivers) == Ok(false) {
787                self.remove_from_event_list(receivers)?
788            } else {
789                false
790            };
791            // queueYIELD_IF_USING_PREEMPTION(), inside the section.
792            // `yield_required` first: it is false on every send that is
793            // not a mutex give, and it is a local, so on the common path
794            // the list is not read a second time at all. Both operands are
795            // pure, so the order is free to choose. (The read cannot be
796            // hoisted above the removal above it -- removing the last
797            // waiter is exactly what changes the answer.)
798            if woke_higher || (yield_required && self.lists.is_empty(receivers) != Ok(false)) {
799                self.port_yield();
800            }
801            self.exit_critical();
802            self.end_wait(caller);
803            return Ok(Ready(()));
804        }
805        if let Err(e) = self.begin_wait(caller, queue, ticks) {
806            self.exit_critical();
807            return Err(e);
808        }
809        if self.remaining_ticks(caller) == 0 {
810            self.exit_critical();
811            self.trace_failure_or_owe(caller, OwedTrace::SendFailed(queue));
812            self.end_wait(caller);
813            return Err(Error::Full);
814        }
815        self.exit_critical();
816        self.suspend_all();
817        self.lock_queue(queue);
818        if self.check_for_timeout(caller) {
819            self.unlock_queue(queue)?;
820            let _ = self.resume_all();
821            self.trace_failure_or_owe(caller, OwedTrace::SendFailed(queue));
822            self.end_wait(caller);
823            return Err(Error::Full);
824        }
825        if self.is_queue_full(queue) {
826            self.trace.note_exits(self.port.exits());
827            let tick = self.tick;
828            self.trace
829                .event(tick, Event::BlockingOnQueueSend { queue, name: "" });
830            let ticks_left = self.remaining_ticks(caller);
831            self.place_on_event_list(Self::queue_send_list(queue), ticks_left)?;
832            self.unlock_queue(queue)?;
833            if !self.resume_all() {
834                self.yield_or_owe(caller);
835            }
836        } else {
837            self.unlock_queue(queue)?;
838            let _ = self.resume_all();
839        }
840        Ok(Blocked)
841    }
842
843    /// `prvCopyDataToQueue`; `true` when giving a mutex back lowered the
844    /// giver's priority and a yield is therefore wanted.
845    /// `prvCopyDataToQueue`.
846    ///
847    /// This resolves rather than taking the caller's snapshot, and
848    /// [`Kernel::copy_data_from_queue`] does the opposite. That is not an
849    /// inconsistency: both forms were measured on both helpers and they
850    /// disagree, because the writer's caller does not keep its snapshot
851    /// live across the call and the reader's does.
852    fn copy_data_to_queue(
853        &mut self,
854        queue: QueueHandle,
855        value: u64,
856        position: Position,
857    ) -> Result<bool> {
858        let snapshot = *self.queues.resolve(queue)?;
859        let mut yield_required = false;
860        // `prvCopyDataToQueue` counts the item in — except on an overwrite
861        // of a queue that already held one, where it decrements first so
862        // the increment cancels and the count stays where it was.
863        //
864        // Decided once, up here, so that each arm below can fold the count
865        // into the resolve it is already making. Deciding it after the arms
866        // is what forced a further resolve for one field.
867        let counted = !(position == Position::Overwrite && snapshot.waiting > 0);
868        if snapshot.kind.carries_data() {
869            let (index, next_read, next_write) = match position {
870                Position::Back => (
871                    snapshot.write_to,
872                    snapshot.read_from,
873                    wrap_next(snapshot.write_to, snapshot.length),
874                ),
875                // Write at the item last read, then step back, so the next
876                // read's pre-increment lands on it. An overwrite takes the
877                // same path; only the message count differs.
878                Position::Front | Position::Overwrite => (
879                    snapshot.read_from,
880                    wrap_prev(snapshot.read_from, snapshot.length),
881                    snapshot.write_to,
882                ),
883            };
884            if let Some(slot) = self.slots.get_mut(snapshot.base.saturating_add(index)) {
885                *slot = value;
886            }
887            let q = self.queues.resolve_mut(queue)?;
888            q.read_from = next_read;
889            q.write_to = next_write;
890            if counted {
891                q.waiting = q.waiting.saturating_add(1);
892            }
893        } else if snapshot.kind.is_mutex() {
894            // Giving a mutex back: the holder drops any inherited
895            // priority. The disinherit touches the holder's TCB, not this
896            // queue, so the resolve after it is still the first one.
897            yield_required = self.priority_disinherit(snapshot.holder)?;
898            let q = self.queues.resolve_mut(queue)?;
899            q.holder = TaskHandle::NULL;
900            if counted {
901                q.waiting = q.waiting.saturating_add(1);
902            }
903        } else if counted {
904            // A counting or binary semaphore: only the count moves.
905            let q = self.queues.resolve_mut(queue)?;
906            q.waiting = q.waiting.saturating_add(1);
907        }
908        Ok(yield_required)
909    }
910
911    // ----------------------------------------------------------- receive --
912
913    /// `xQueueReceive`.
914    ///
915    /// # Errors
916    /// [`Error::Empty`] when the block time expires with the queue still
917    /// empty; [`Error::Gone`] for a stale handle.
918    pub fn queue_receive(&mut self, queue: QueueHandle, ticks: u64) -> Result<Wait<u64>> {
919        self.queue_take(queue, ticks, false)
920    }
921
922    /// `xQueuePeek`: read the head without removing it.
923    ///
924    /// # Errors
925    /// As [`Kernel::queue_receive`].
926    pub fn queue_peek(&mut self, queue: QueueHandle, ticks: u64) -> Result<Wait<u64>> {
927        self.queue_take(queue, ticks, true)
928    }
929
930    /// `xSemaphoreTake`, which is `xQueueSemaphoreTake`: a receive with no
931    /// data, and — on a mutex — with priority inheritance while it waits.
932    ///
933    /// # Errors
934    /// [`Error::Empty`] on timeout; [`Error::Gone`] for a stale handle.
935    pub fn semaphore_take(&mut self, semaphore: QueueHandle, ticks: u64) -> Result<Wait<()>> {
936        match self.queue_take(semaphore, ticks, false)? {
937            Blocked => Ok(Blocked),
938            Ready(_) => Ok(Ready(())),
939        }
940    }
941
942    /// The shared body of `xQueueReceive`, `xQueuePeek` and
943    /// `xQueueSemaphoreTake` — one pass of the C `for(;;)`.
944    fn queue_take(&mut self, queue: QueueHandle, ticks: u64, peek: bool) -> Result<Wait<u64>> {
945        let caller = self.current;
946        self.enter_critical();
947        let snapshot = match self.queues.resolve(queue) {
948            Ok(q) => *q,
949            Err(e) => {
950                self.exit_critical();
951                return Err(e);
952            }
953        };
954        // The wait frame is set where the C sets it: `xQueueReceive` and
955        // `xQueueGenericSend` call `vTaskInternalSetTimeOutState` only
956        // after finding the queue unusable AND the block time non-zero,
957        // guarded by `xEntryTimeSet`. Setting it up here instead meant
958        // every *successful* call wrote a six-field frame and then had
959        // `end_wait` wipe it on the way out.
960        //
961        // It also has to be after the resolve, so that a handle naming
962        // nothing leaves the caller exactly as it found it -- which is
963        // what `configASSERT( pxQueue )` means in the C.
964        if snapshot.waiting > 0 {
965            let value = self.copy_data_from_queue(queue, &snapshot, peek)?;
966            self.trace.note_exits(self.port.exits());
967            let tick = self.tick;
968            // `xQueuePeek` is its own function in the C with its own trace
969            // macro; `xQueueSemaphoreTake` shares `xQueueReceive`'s.
970            let event = if peek {
971                Event::QueuePeek { queue, name: "" }
972            } else {
973                Event::QueueReceive { queue, name: "" }
974            };
975            self.trace.event(tick, event);
976            if peek {
977                // A peek wakes another *receiver*, not a sender: the item
978                // is still there.
979                let receivers = Self::queue_receive_list(queue);
980                if self.lists.is_empty(receivers) == Ok(false)
981                    && self.remove_from_event_list(receivers)?
982                {
983                    self.port_yield();
984                }
985            } else {
986                if snapshot.kind.is_mutex() {
987                    self.queues.resolve_mut(queue)?.holder = caller;
988                    self.increment_mutexes_held(caller);
989                }
990                let senders = Self::queue_send_list(queue);
991                if self.lists.is_empty(senders) == Ok(false)
992                    && self.remove_from_event_list(senders)?
993                {
994                    self.port_yield();
995                }
996            }
997            self.exit_critical();
998            self.end_wait(caller);
999            return Ok(Ready(value));
1000        }
1001        if let Err(e) = self.begin_wait(caller, queue, ticks) {
1002            self.exit_critical();
1003            return Err(e);
1004        }
1005        if self.remaining_ticks(caller) == 0 {
1006            self.exit_critical();
1007            // `traceQUEUE_PEEK_FAILED` is not one of the harness's hooks,
1008            // so a failed peek says nothing on either side.
1009            if !peek {
1010                self.trace_failure_or_owe(caller, OwedTrace::ReceiveFailed(queue));
1011            }
1012            self.end_wait(caller);
1013            return Err(Error::Empty);
1014        }
1015        self.exit_critical();
1016        self.suspend_all();
1017        self.lock_queue(queue);
1018        if self.check_for_timeout(caller) {
1019            self.unlock_queue(queue)?;
1020            let _ = self.resume_all();
1021            if self.is_queue_empty(queue) {
1022                if snapshot.kind.is_mutex() && self.wait_inherited(caller) {
1023                    // `vTaskPriorityDisinheritAfterTimeout`: the holder
1024                    // keeps only what the still-waiting tasks justify.
1025                    self.enter_critical();
1026                    let highest = self.highest_waiting_priority(queue)?;
1027                    let holder = self.queues.resolve(queue).map(|q| q.holder)?;
1028                    self.priority_disinherit_after_timeout(holder, highest)?;
1029                    self.exit_critical();
1030                }
1031                if !peek {
1032                    self.trace_failure_or_owe(caller, OwedTrace::ReceiveFailed(queue));
1033                }
1034                self.end_wait(caller);
1035                return Err(Error::Empty);
1036            }
1037            return Ok(Blocked);
1038        }
1039        if self.is_queue_empty(queue) {
1040            let event = if peek {
1041                Event::BlockingOnQueuePeek { queue, name: "" }
1042            } else {
1043                Event::BlockingOnQueueReceive { queue, name: "" }
1044            };
1045            self.trace.note_exits(self.port.exits());
1046            let tick = self.tick;
1047            self.trace.event(tick, event);
1048            if snapshot.kind.is_mutex() {
1049                self.enter_critical();
1050                let holder = self.queues.resolve(queue).map(|q| q.holder)?;
1051                let inherited = self.priority_inherit(holder)?;
1052                self.exit_critical();
1053                if inherited {
1054                    self.set_wait_inherited(caller);
1055                }
1056            }
1057            let ticks_left = self.remaining_ticks(caller);
1058            self.place_on_event_list(Self::queue_receive_list(queue), ticks_left)?;
1059            self.unlock_queue(queue)?;
1060            if !self.resume_all() {
1061                self.yield_or_owe(caller);
1062            }
1063        } else {
1064            self.unlock_queue(queue)?;
1065            let _ = self.resume_all();
1066        }
1067        Ok(Blocked)
1068    }
1069
1070    /// `prvCopyDataFromQueue`, with the peek variant putting the cursor
1071    /// back as `xQueuePeek` does.
1072    /// `prvCopyDataFromQueue`, over a snapshot its caller already took.
1073    ///
1074    /// Every caller resolves this queue to decide whether there is
1075    /// anything to read; this used to resolve it again to do the reading.
1076    /// The snapshot cannot be stale — nothing between the two touches this
1077    /// queue, and both are inside the same critical section.
1078    ///
1079    /// The same change to [`Kernel::copy_data_to_queue`] is **refused**,
1080    /// and the pair is why this bench exists: threading the snapshot into
1081    /// the writer is worth -245,389 Ir inside it and +211,717 in
1082    /// `queue_send_generic`, because a `&Queue` argument forces the
1083    /// caller's local to be addressable and it stops living in registers.
1084    /// Here the caller is `queue_take`, which keeps its snapshot alive
1085    /// across the call anyway, so there is no spill to pay for.
1086    fn copy_data_from_queue(
1087        &mut self,
1088        queue: QueueHandle,
1089        snapshot: &Queue,
1090        peek: bool,
1091    ) -> Result<u64> {
1092        if !snapshot.kind.carries_data() {
1093            if !peek {
1094                let q = self.queues.resolve_mut(queue)?;
1095                q.waiting = q.waiting.saturating_sub(1);
1096            }
1097            return Ok(0);
1098        }
1099        let next = wrap_next(snapshot.read_from, snapshot.length);
1100        let value = self
1101            .slots
1102            .get(snapshot.base.saturating_add(next))
1103            .copied()
1104            .unwrap_or(0);
1105        // `xQueuePeek` saves and restores `pcReadFrom` — which is to
1106        // say it leaves it exactly as it found it. Writing it back stored
1107        // the value already there, and the resolve that reached it was a
1108        // whole handle validation for a no-op.
1109        if !peek {
1110            let q = self.queues.resolve_mut(queue)?;
1111            q.read_from = next;
1112            q.waiting = q.waiting.saturating_sub(1);
1113        }
1114        Ok(value)
1115    }
1116
1117    // ------------------------------------------------------ recursive mutex --
1118
1119    /// `xSemaphoreTakeRecursive`.
1120    ///
1121    /// # Errors
1122    /// [`Error::Empty`] on timeout; [`Error::Gone`] for a stale handle.
1123    pub fn mutex_take_recursive(&mut self, mutex: QueueHandle, ticks: u64) -> Result<Wait<()>> {
1124        let caller = self.current;
1125        let holder = self.queues.resolve(mutex)?.holder;
1126        if holder == caller {
1127            let q = self.queues.resolve_mut(mutex)?;
1128            q.recursions = q.recursions.saturating_add(1);
1129            return Ok(Ready(()));
1130        }
1131        match self.semaphore_take(mutex, ticks)? {
1132            Blocked => Ok(Blocked),
1133            Ready(()) => {
1134                let q = self.queues.resolve_mut(mutex)?;
1135                q.recursions = q.recursions.saturating_add(1);
1136                Ok(Ready(()))
1137            }
1138        }
1139    }
1140
1141    /// `xSemaphoreGiveRecursive`.
1142    ///
1143    /// # Errors
1144    /// [`Error::NotActive`] when the caller does not hold the mutex.
1145    pub fn mutex_give_recursive(&mut self, mutex: QueueHandle) -> Result<()> {
1146        let caller = self.current;
1147        if self.queues.resolve(mutex)?.holder != caller {
1148            return Err(Error::NotActive);
1149        }
1150        let remaining = {
1151            let q = self.queues.resolve_mut(mutex)?;
1152            q.recursions = q.recursions.saturating_sub(1);
1153            q.recursions
1154        };
1155        if remaining == 0 {
1156            let _ = self.semaphore_give(mutex)?;
1157        }
1158        Ok(())
1159    }
1160
1161    // -------------------------------------------------------- queue locks --
1162
1163    /// `prvLockQueue`: one critical section. The lock counters themselves
1164    /// only matter to an ISR, and the sim has none — but the section is
1165    /// where sim time passes, so it is here.
1166    /// `prvLockQueue`: stop interrupts taking tasks off this queue's event
1167    /// lists while a task walks them. An interrupt that finds the queue
1168    /// locked counts what it did instead, and [`Kernel::unlock_queue`] pays
1169    /// it back.
1170    fn lock_queue(&mut self, queue: QueueHandle) {
1171        self.enter_critical();
1172        if let Ok(q) = self.queues.resolve_mut(queue) {
1173            if q.rx_lock == UNLOCKED {
1174                q.rx_lock = LOCKED_UNMODIFIED;
1175            }
1176            if q.tx_lock == UNLOCKED {
1177                q.tx_lock = LOCKED_UNMODIFIED;
1178            }
1179        }
1180        self.exit_critical();
1181    }
1182
1183    /// `prvUnlockQueue`: two critical sections, one per lock counter. The
1184    /// loops inside them wake tasks that an ISR queued while the lock was
1185    /// held, and on the sim there are none.
1186    /// `prvUnlockQueue`: two critical sections, one per lock count, each
1187    /// waking one task per send or receive an interrupt made while the
1188    /// queue was locked. The yields are *missed* yields — pended, not
1189    /// taken — because a task is walking the list.
1190    fn unlock_queue(&mut self, queue: QueueHandle) -> Result<()> {
1191        self.enter_critical();
1192        {
1193            // One resolve for both fields. They were two, and a resolve
1194            // is a null test, a bounds check, a generation compare and an
1195            // `Option` unwrap -- all of it repeated to reach the same slot.
1196            let (mut tx_lock, container) = self
1197                .queues
1198                .resolve(queue)
1199                .map_or((UNLOCKED, QueueHandle::NULL), |q| {
1200                    (q.tx_lock, q.set_container)
1201                });
1202            while tx_lock > LOCKED_UNMODIFIED {
1203                if container != QueueHandle::NULL {
1204                    if self.notify_queue_set_container(queue)? {
1205                        self.missed_yield();
1206                    }
1207                    tx_lock = tx_lock.saturating_sub(1);
1208                    continue;
1209                }
1210                let receivers = Self::queue_receive_list(queue);
1211                if self.lists.is_empty(receivers) == Ok(true) {
1212                    break;
1213                }
1214                if self.remove_from_event_list(receivers)? {
1215                    self.missed_yield();
1216                }
1217                tx_lock = tx_lock.saturating_sub(1);
1218            }
1219            if let Ok(q) = self.queues.resolve_mut(queue) {
1220                q.tx_lock = UNLOCKED;
1221            }
1222        }
1223        self.exit_critical();
1224        self.enter_critical();
1225        {
1226            let mut rx_lock = self
1227                .queues
1228                .resolve(queue)
1229                .map(|q| q.rx_lock)
1230                .unwrap_or(UNLOCKED);
1231            while rx_lock > LOCKED_UNMODIFIED {
1232                let senders = Self::queue_send_list(queue);
1233                if self.lists.is_empty(senders) == Ok(true) {
1234                    break;
1235                }
1236                if self.remove_from_event_list(senders)? {
1237                    self.missed_yield();
1238                }
1239                rx_lock = rx_lock.saturating_sub(1);
1240            }
1241            if let Ok(q) = self.queues.resolve_mut(queue) {
1242                q.rx_lock = UNLOCKED;
1243            }
1244        }
1245        self.exit_critical();
1246        Ok(())
1247    }
1248
1249    /// `vQueueWaitForMessageRestricted`: what the timer service task does
1250    /// when it has no timer to run — block on the command queue without
1251    /// suspending the scheduler.
1252    ///
1253    /// `wait_indefinitely` is the C third argument: when set the wait
1254    /// becomes `portMAX_DELAY` and the task goes to the suspended list
1255    /// rather than a delayed one. The queue locks around it are C's
1256    /// `prvLockQueue` and `prvUnlockQueue` — three critical sections, and
1257    /// therefore three sixteenths of a tick's worth of sim time.
1258    ///
1259    /// # Errors
1260    /// A list error surfaces as itself.
1261    pub fn wait_for_message_restricted(
1262        &mut self,
1263        queue: QueueHandle,
1264        ticks: u64,
1265        wait_indefinitely: bool,
1266    ) -> Result<()> {
1267        self.lock_queue(queue);
1268        let empty = self
1269            .queues
1270            .resolve(queue)
1271            .map(|q| q.waiting == 0)
1272            .unwrap_or(false);
1273        if empty {
1274            // vTaskPlaceOnEventListRestricted
1275            let current = self.current;
1276            let event = Self::event_item(current);
1277            self.lists
1278                .insert_end(Self::queue_receive_list(queue), event)?;
1279            let ticks = if wait_indefinitely {
1280                Self::MAX_DELAY
1281            } else {
1282                ticks
1283            };
1284            let wake_at = self.tick.wrapping_add(ticks) & Self::MAX_DELAY;
1285            self.trace_task(current, |task, name| Event::TaskDelayUntil {
1286                task,
1287                name,
1288                wake_at,
1289            });
1290            self.add_current_task_to_delayed_list(ticks, wait_indefinitely)?;
1291        }
1292        self.unlock_queue(queue)?;
1293        Ok(())
1294    }
1295
1296    /// `prvIsQueueEmpty`, critical section and all — it is one of the
1297    /// four the blocking path spends, and therefore a sixteenth of a tick
1298    /// every four calls.
1299    fn is_queue_empty(&mut self, queue: QueueHandle) -> bool {
1300        self.enter_critical();
1301        let empty = self
1302            .queues
1303            .resolve(queue)
1304            .map(|q| q.waiting == 0)
1305            .unwrap_or(true);
1306        self.exit_critical();
1307        empty
1308    }
1309
1310    /// `prvIsQueueFull`, likewise.
1311    fn is_queue_full(&mut self, queue: QueueHandle) -> bool {
1312        self.enter_critical();
1313        let full = self
1314            .queues
1315            .resolve(queue)
1316            .map(|q| q.waiting >= q.length)
1317            .unwrap_or(false);
1318        self.exit_critical();
1319        full
1320    }
1321
1322    /// `prvGetHighestPriorityOfWaitToReceiveList`.
1323    fn highest_waiting_priority(&self, queue: QueueHandle) -> Result<u8> {
1324        let list = Self::queue_receive_list(queue);
1325        match self.lists.head(list)? {
1326            // The event item's value is `configMAX_PRIORITIES - priority`.
1327            Some(item) => {
1328                let value = self.lists.value(item)?;
1329                let max = u64::from(C::MAX_PRIORITIES);
1330                Ok(u8::try_from(max.saturating_sub(value)).unwrap_or(0))
1331            }
1332            None => Ok(0),
1333        }
1334    }
1335}
1336
1337/// The next index in a ring of `length`.
1338const fn wrap_next(index: usize, length: usize) -> usize {
1339    let next = index.saturating_add(1);
1340    if next >= length { 0 } else { next }
1341}
1342
1343/// The previous index in a ring of `length`.
1344const fn wrap_prev(index: usize, length: usize) -> usize {
1345    match index.checked_sub(1) {
1346        Some(prev) => prev,
1347        None => length.saturating_sub(1),
1348    }
1349}
1350
1351/// The Rust face's raw surface: see [`crate::typed`].
1352///
1353/// Every method here is one of the kernel's own calls under another name.
1354/// The one that is not — [`Raw::raw_queue_has_room`] — is the kernel
1355/// reading its own queue rather than a task asking, so it takes no
1356/// critical section and costs no sim time, which is what lets the typed
1357/// face be free.
1358impl<
1359    C: Config,
1360    P: Port,
1361    T: Trace,
1362    H,
1363    const TASKS: usize,
1364    const ITEMS: usize,
1365    const LISTS: usize,
1366    const QUEUES: usize,
1367    const SLOTS: usize,
1368    const BUFFERS: usize,
1369    const BYTES: usize,
1370    const TIMERS: usize,
1371    const GROUPS: usize,
1372> crate::typed::Raw
1373    for Kernel<C, P, T, H, TASKS, ITEMS, LISTS, QUEUES, SLOTS, BUFFERS, BYTES, TIMERS, GROUPS>
1374where
1375    H: TickHook<Self>,
1376{
1377    fn raw_queue_create(&mut self, length: usize) -> Result<QueueHandle> {
1378        self.queue_create(length)
1379    }
1380
1381    fn raw_queue_send(&mut self, queue: QueueHandle, value: u64, ticks: u64) -> Result<Wait<()>> {
1382        self.queue_send(queue, value, ticks)
1383    }
1384
1385    fn raw_queue_receive(&mut self, queue: QueueHandle, ticks: u64) -> Result<Wait<u64>> {
1386        self.queue_receive(queue, ticks)
1387    }
1388
1389    fn raw_queue_messages_waiting(&mut self, queue: QueueHandle) -> Result<usize> {
1390        self.queue_messages_waiting(queue)
1391    }
1392
1393    fn raw_queue_has_room(&self, queue: QueueHandle) -> bool {
1394        self.queues
1395            .resolve(queue)
1396            .is_ok_and(|q| q.waiting < q.length)
1397    }
1398
1399    fn raw_in_isr(&self) -> bool {
1400        self.port.in_isr()
1401    }
1402
1403    fn raw_queue_send_from_isr(&mut self, queue: QueueHandle, value: u64) -> Result<Woken> {
1404        self.queue_send_from_isr(queue, value)
1405    }
1406
1407    fn raw_mutex_create(&mut self) -> Result<QueueHandle> {
1408        self.mutex_create()
1409    }
1410
1411    fn raw_mutex_take(&mut self, mutex: QueueHandle, ticks: u64) -> Result<Wait<()>> {
1412        self.semaphore_take(mutex, ticks)
1413    }
1414
1415    fn raw_mutex_give(&mut self, mutex: QueueHandle) -> Result<()> {
1416        self.semaphore_give(mutex).map(|_| ())
1417    }
1418}