Skip to main content

rusty_rtos_kernel_core/
queue.rs

1//! `queue.c`: the queue, and the semaphore and mutex built on it.
2//!
3//! In FreeRTOS a semaphore *is* a queue with a zero-size item, and a mutex
4//! is that plus a holder and priority inheritance. Kairos keeps the same
5//! shape, because the trace does: a semaphore give fires `QUEUE_SEND` and a
6//! mutex take fires `QUEUE_RECEIVE`, and a conformance diff would notice
7//! any tidier arrangement.
8//!
9//! # A blocking call with no stack to block on
10//!
11//! `xQueueReceive( q, &v, 100 )` blocks: the C task stops inside the call,
12//! and when it is woken it loops back to the top, re-checks the queue, and
13//! either returns the item or blocks again. This kernel has no stack to
14//! suspend, so the loop is turned inside out — a blocking call returns
15//! [`Blocked`] and the caller invokes it again, unchanged, when it next
16//! runs:
17//!
18//! ```ignore
19//! // in a task body, at some `pc`:
20//! match k.queue_receive(q, 100)? {
21//!     Blocked => {}                 // stay at this pc; the kernel parked us
22//!     Ready(value) => { ...; self.pc = next; }
23//! }
24//! ```
25//!
26//! The task is off the ready list when `Blocked` comes back, so the runner
27//! will not step it again until the kernel wakes it — which is exactly when
28//! the C thread would have resumed. What the C keeps on its stack across
29//! the block (the timeout, the remaining ticks, whether inheritance
30//! happened) this kernel keeps in the TCB, in [`Wait`], because that *is*
31//! the stack frame.
32
33use rusty_rtos_core::config::Config;
34use rusty_rtos_core::error::{Error, Result};
35use rusty_rtos_core::handle::{QueueHandle, TaskHandle};
36use rusty_rtos_core::hooks::TickHook;
37use rusty_rtos_core::isr::Woken;
38use rusty_rtos_core::port::Port;
39use rusty_rtos_core::trace::{Event, Trace};
40
41use crate::kernel::{Kernel, OwedTrace, QueueResume};
42
43/// What a blocking call answers.
44#[derive(Debug, Clone, Copy, PartialEq, Eq)]
45pub enum Wait<T> {
46    /// The kernel parked the calling task. Leave the program counter where
47    /// it is and make the same call again when the task next runs.
48    Blocked,
49    /// The call completed.
50    Ready(T),
51}
52
53pub use Wait::{Blocked, Ready};
54
55/// Where an item goes (`xCopyPosition`).
56#[derive(Debug, Clone, Copy, PartialEq, Eq)]
57pub enum Position {
58    /// `queueSEND_TO_BACK`.
59    Back,
60    /// `queueSEND_TO_FRONT`.
61    Front,
62    /// `queueOVERWRITE`: write to the front of a length-one queue whether
63    /// or not it already holds something, and leave the message count
64    /// where it was rather than raising it.
65    Overwrite,
66}
67
68/// What a queue is underneath (`ucQueueType`).
69#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
70pub enum Kind {
71    // ORDER IS LOAD-BEARING, and only for the two predicates below. The kinds
72    // that carry data are first and the two mutex kinds are last, so
73    // `carries_data` and `is_mutex` are each a range the compiler can test
74    // with one comparison instead of two. Nothing reads a discriminant, so
75    // the order carries no other meaning.
76    /// A queue of values. `#[default]` so a free arena slot holds something
77    /// real without unsafe; the ORDER above is untouched, and nothing reads
78    /// a free slot.
79    #[default]
80    Queue,
81    /// A queue set: a queue whose items are the handles of the queues in
82    /// it. `xQueueSelectFromSet` is a receive from this one.
83    Set,
84    /// A binary or counting semaphore: a queue with a zero-size item, so
85    /// only the count matters.
86    Semaphore,
87    /// A mutex: a one-deep semaphore with a holder and priority
88    /// inheritance.
89    Mutex,
90    /// A recursive mutex: as above, and the holder may take it again.
91    RecursiveMutex,
92}
93
94impl Kind {
95    /// `uxItemSize == 0`: semaphores and mutexes carry a count, not data.
96    pub(crate) const fn carries_data(self) -> bool {
97        matches!(self, Self::Queue | Self::Set)
98    }
99
100    /// The two mutex kinds, which inherit priority.
101    pub(crate) const fn is_mutex(self) -> bool {
102        matches!(self, Self::Mutex | Self::RecursiveMutex)
103    }
104}
105
106/// One queue: the C `Queue_t` minus the byte pointers. Storage is a range
107/// of the kernel's shared slot pool, and the read and write cursors follow
108/// C's `pcReadFrom` / `pcWriteTo` exactly — `pcReadFrom` points at the item
109/// last read, which is what makes `queueSEND_TO_FRONT` a single step back.
110#[derive(Debug, Clone, Copy, Default)]
111pub(crate) struct Queue {
112    pub(crate) base: usize,
113    /// `uxLength`.
114    pub(crate) length: usize,
115    /// `uxMessagesWaiting`.
116    pub(crate) waiting: usize,
117    /// `pcReadFrom`, as an index: the item last read.
118    pub(crate) read_from: usize,
119    /// `pcWriteTo`, as an index: where the next back-send writes.
120    pub(crate) write_to: usize,
121    pub(crate) kind: Kind,
122    /// `u.xSemaphore.xMutexHolder`.
123    pub(crate) holder: TaskHandle,
124    /// `u.xSemaphore.uxRecursiveCallCount`.
125    pub(crate) recursions: u32,
126    /// `cTxLock`: [`UNLOCKED`] when no task holds the queue locked;
127    /// otherwise how many sends an interrupt made while it was locked, and
128    /// therefore how many receivers `prvUnlockQueue` still owes a wake-up.
129    pub(crate) tx_lock: i8,
130    /// `cRxLock`, the same for receives and the tasks waiting to send.
131    pub(crate) rx_lock: i8,
132    /// `pxQueueSetContainer`: the set this queue belongs to, if any. An
133    /// item arriving here is announced there.
134    pub(crate) set_container: QueueHandle,
135}
136
137/// `queueUNLOCKED`.
138pub(crate) const UNLOCKED: i8 = -1;
139/// `queueLOCKED_UNMODIFIED`.
140pub(crate) const LOCKED_UNMODIFIED: i8 = 0;
141
142impl Queue {
143    // A3: ONE caller, so the out-of-line body pays a prologue and epilogue for
144    // a single call. Inlining moves the body rather than duplicating it.
145    #[inline]
146    pub(crate) const fn new(base: usize, length: usize, kind: Kind) -> Self {
147        Self {
148            base,
149            length,
150            waiting: 0,
151            // `xQueueGenericReset`: pcReadFrom = pcHead + (length - 1) * itemSize.
152            read_from: length.saturating_sub(1),
153            write_to: 0,
154            kind,
155            holder: TaskHandle::NULL,
156            recursions: 0,
157            tx_lock: UNLOCKED,
158            rx_lock: UNLOCKED,
159            set_container: QueueHandle::NULL,
160        }
161    }
162}
163
164impl<
165    C: Config,
166    P: Port,
167    T: Trace,
168    H,
169    const TASKS: usize,
170    const ITEMS: usize,
171    const LISTS: usize,
172    const QUEUES: usize,
173    const SLOTS: usize,
174    const BUFFERS: usize,
175    const BYTES: usize,
176    const TIMERS: usize,
177    const GROUPS: usize,
178    const TIMER_CMDS: usize,
179> Kernel<C, P, T, H, TASKS, ITEMS, LISTS, QUEUES, SLOTS, BUFFERS, BYTES, TIMERS, GROUPS, TIMER_CMDS>
180where
181    H: TickHook<Self>,
182{
183    // ------------------------------------------------------------ create --
184
185    /// `xQueueCreate`.
186    ///
187    /// # Errors
188    /// [`Error::Full`] when the queue arena or the shared slot pool is
189    /// exhausted; [`Error::InvalidArgument`] for a zero length.
190    pub fn queue_create(&mut self, length: usize) -> Result<QueueHandle> {
191        self.new_queue(length, Kind::Queue)
192    }
193
194    /// `vQueueDelete`, and `vSemaphoreDelete`, which the C defines as the
195    /// same function.
196    ///
197    /// # It traces nothing and still costs time
198    ///
199    /// `vQueueDelete` takes no critical section of its own — but it ends in
200    /// `vPortFree( pxQueue )`, and every `heap_N.c` wraps free in
201    /// `vTaskSuspendAll` / `xTaskResumeAll` exactly as it wraps malloc. So
202    /// a delete costs **one outermost critical-section exit**, which under
203    /// the sim contract is one unit of time, while emitting no trace line
204    /// at all (this harness defines no `traceQUEUE_DELETE`).
205    ///
206    /// `AbortDelay`'s remake left the deletes out on the reasoning that an
207    /// event-less call cannot move the trace. Every event still agreed and
208    /// the exit column was one short from the first delete onwards. The
209    /// sibling `event_group_delete` already carried this note; the queue
210    /// had no delete at all.
211    ///
212    /// # Errors
213    /// [`Error::Gone`] for a stale handle.
214    pub fn queue_delete(&mut self, queue: QueueHandle) -> Result<()> {
215        let q = *self.queues.resolve(queue)?;
216        // Give the item storage back when it is the last thing allocated.
217        //
218        // `new_queue` bump-allocates from `slots_used` and, until
219        // 2026-09-21, nothing ever lowered it: the DESCRIPTOR came back and
220        // the storage did not, so a create/delete loop exhausted `SLOTS`
221        // permanently. `AbortDelay` hits it after ~108 passes, which is how
222        // it was found, and any application with a queue per connection or
223        // per job hits it too.
224        //
225        // This is the same move `give_bytes` already makes for the byte
226        // arena — that allocator exists because "a bump allocator would run
227        // out in a few hundred ticks", and the identical argument applies
228        // here and was simply never made.
229        //
230        // Deliberately the END-OF-ARENA case only, and not a free list.
231        // The pattern that needs this is create-then-delete of the same
232        // object, where the deleted extent IS the last one, and it is what
233        // the corpus exercises. An out-of-order delete still strands its
234        // slots; that wants the full `free_blocks` treatment and is a
235        // larger change with more to prove.
236        if q.kind.carries_data() {
237            self.give_slots(q.base, q.length);
238        }
239        let _ = self.queues.discard(queue);
240        // `vPortFree( pxQueue )`.
241        self.account_for_allocation();
242        Ok(())
243    }
244
245    /// `xSemaphoreCreateBinary`: a one-deep semaphore, created empty.
246    ///
247    /// # Errors
248    /// As [`Kernel::queue_create`].
249    pub fn semaphore_create_binary(&mut self) -> Result<QueueHandle> {
250        self.new_queue(1, Kind::Semaphore)
251    }
252
253    /// `xSemaphoreCreateCounting`.
254    ///
255    /// # Errors
256    /// As [`Kernel::queue_create`]; [`Error::InvalidArgument`] if the
257    /// initial count exceeds the maximum.
258    pub fn semaphore_create_counting(&mut self, max: usize, initial: usize) -> Result<QueueHandle> {
259        if initial > max {
260            return Err(Error::InvalidArgument);
261        }
262        let handle = self.new_queue(max, Kind::Semaphore)?;
263        if let Ok(q) = self.queues.resolve_mut(handle) {
264            q.waiting = initial;
265        }
266        Ok(handle)
267    }
268
269    /// `xSemaphoreCreateMutex`: created **available**, unlike a semaphore.
270    ///
271    /// # Errors
272    /// As [`Kernel::queue_create`].
273    pub fn mutex_create(&mut self) -> Result<QueueHandle> {
274        self.new_mutex(Kind::Mutex)
275    }
276
277    /// `xSemaphoreCreateRecursiveMutex`.
278    ///
279    /// # Errors
280    /// As [`Kernel::queue_create`].
281    pub fn mutex_create_recursive(&mut self) -> Result<QueueHandle> {
282        self.new_mutex(Kind::RecursiveMutex)
283    }
284
285    /// `prvInitialiseMutex`: give the mutex once so it starts available.
286    ///
287    /// This goes through the general send for the same reason the C does — and
288    /// the byte count is the argument. Until 2026-09-25 a private `prime_mutex`
289    /// carried a hand-written copy of the send, specialised on the four
290    /// constants supplied below, because inlining the general body here cost
291    /// **534 bytes**. That defence was aimed at the wrong thing: the choice is
292    /// not "specialise or inline", it is *which symbol to call*, and
293    /// [`Self::send_generic_outlined`] — the A4 handle that already existed for
294    /// the cold callers — costs a five-argument frame, not a body.
295    ///
296    /// Calling it instead and deleting the specialisation measured
297    /// **−316 bytes** of flash (19,302 → 18,986), `mv` − 8, `slli` − 5,
298    /// `andi` − 2 and 99 fewer rv32 instructions — every pin down, none up.
299    /// `conform --all` stays at 26/26, which is what proves the trace is
300    /// unchanged: one critical section, `QueueSend` at the same point, the same
301    /// `exits`. A mutex therefore still shows a `QUEUE_SEND` line immediately
302    /// after its `QUEUE_CREATE`, before any task exists.
303    ///
304    /// The law: a specialisation written to escape an unwanted INLINE is only
305    /// worth its bytes if no out-of-line handle on the general body exists.
306    /// Check for the handle first.
307    fn new_mutex(&mut self, kind: Kind) -> Result<QueueHandle> {
308        let handle = self.new_queue(1, kind)?;
309        let _ = self.send_generic_outlined(handle, 0, 0, Position::Back)?;
310        Ok(handle)
311    }
312
313    /// Take `length` slots: first fit from what deleted queues gave back,
314    /// and only then from the untouched end.
315    ///
316    /// First fit rather than best fit for the same reason `take_bytes` uses
317    /// it — the pattern that needs an allocator at all is create-then-delete
318    /// of the *same size*, which first fit serves exactly and without
319    /// fragmenting.
320    fn take_slots(&mut self, length: usize) -> Option<usize> {
321        if length == 0 {
322            return Some(self.slots_used);
323        }
324        for i in 0..self.free_slot_count {
325            let (base, free) = *self.free_slots.get(i)?;
326            if free < length {
327                continue;
328            }
329            if free == length {
330                self.drop_free_slot(i);
331            } else if let Some(slot) = self.free_slots.get_mut(i) {
332                *slot = (base.saturating_add(length), free.wrapping_sub(length));
333            }
334            return Some(base);
335        }
336        let end = self.slots_used.checked_add(length)?;
337        if end > SLOTS {
338            return None;
339        }
340        let base = self.slots_used;
341        self.slots_used = end;
342        Some(base)
343    }
344
345    /// Give an extent back, coalescing with whichever neighbours touch it so
346    /// the list cannot grow past one entry per hole.
347    pub(crate) fn give_slots(&mut self, base: usize, length: usize) {
348        if length == 0 {
349            return;
350        }
351        let mut base = base;
352        let mut length = length;
353        let mut i = 0;
354        while i < self.free_slot_count {
355            let Some(&(other_base, other_len)) = self.free_slots.get(i) else {
356                break;
357            };
358            if other_base.saturating_add(other_len) == base {
359                base = other_base;
360                length = length.saturating_add(other_len);
361                self.drop_free_slot(i);
362                continue;
363            }
364            if base.saturating_add(length) == other_base {
365                length = length.saturating_add(other_len);
366                self.drop_free_slot(i);
367                continue;
368            }
369            i = i.wrapping_add(1);
370        }
371        // An extent at the very end goes back to the bump pointer instead of
372        // the list, which is what keeps a create/delete loop free.
373        if base.saturating_add(length) == self.slots_used {
374            self.slots_used = base;
375            return;
376        }
377        if let Some(slot) = self.free_slots.get_mut(self.free_slot_count) {
378            *slot = (base, length);
379            self.free_slot_count = self.free_slot_count.wrapping_add(1);
380        }
381    }
382
383    fn drop_free_slot(&mut self, index: usize) {
384        let last = self.free_slot_count.saturating_sub(1);
385        if index < last {
386            if let Some(&moved) = self.free_slots.get(last) {
387                if let Some(slot) = self.free_slots.get_mut(index) {
388                    *slot = moved;
389                }
390            }
391        }
392        self.free_slot_count = last;
393    }
394
395    fn new_queue(&mut self, length: usize, kind: Kind) -> Result<QueueHandle> {
396        if length == 0 {
397            return Err(Error::InvalidArgument);
398        }
399        // A zero-item-size queue needs no storage; only real queues do.
400        let slots = if kind.carries_data() { length } else { 0 };
401        let base = self.take_slots(slots).ok_or(Error::Full)?;
402        let handle = match self.queues.try_insert(Queue::new(base, length, kind)) {
403            Ok(handle) => handle,
404            Err(_) => {
405                // The descriptor arena refused after the slots were taken,
406                // so give them straight back rather than stranding them.
407                self.give_slots(base, slots);
408                return Err(Error::Full);
409            }
410        };
411        // `xQueueGenericCreate` takes the queue and its storage from the
412        // heap before it initialises anything.
413        self.account_for_allocation();
414        // `xQueueGenericReset` runs in a critical section before
415        // `traceQUEUE_CREATE` fires at the end of `prvInitialiseNewQueue`.
416        self.enter_critical();
417        self.exit_critical();
418        let tick = self.tick;
419        self.note_exits();
420        self.trace.event(
421            tick,
422            Event::QueueCreate {
423                queue: handle,
424                name: "",
425                length,
426            },
427        );
428        Ok(handle)
429    }
430
431    // ----------------------------------------------------------- getters --
432
433    /// `uxQueueMessagesWaiting`, which takes a critical section.
434    ///
435    /// # Errors
436    /// [`Error::Gone`] for a stale handle.
437    pub fn queue_messages_waiting(&mut self, queue: QueueHandle) -> Result<usize> {
438        self.enter_critical();
439        let n = self.queues.resolve(queue).map(|q| q.waiting);
440        self.exit_critical();
441        n
442    }
443
444    /// `xQueueIsQueueFullFromISR`, which takes NO critical section.
445    ///
446    /// The C compares `uxMessagesWaiting` against `uxLength` and returns,
447    /// with nothing around it — it is meant to be called from an interrupt,
448    /// where a critical section would be the wrong thing. That is why this
449    /// takes `&self` and [`Kernel::queue_messages_waiting`] takes `&mut
450    /// self`: the borrow is the proof that this one cannot charge the
451    /// clock, and on the sim a charged section is a sixteenth of a tick.
452    ///
453    /// # Errors
454    /// [`Error::Gone`] for a stale handle.
455    pub fn queue_is_full_from_isr(&self, queue: QueueHandle) -> Result<bool> {
456        self.queues.resolve(queue).map(|q| q.waiting == q.length)
457    }
458
459    /// `uxSemaphoreGetCount`, which is the same thing.
460    ///
461    /// # Errors
462    /// [`Error::Gone`] for a stale handle.
463    pub fn semaphore_count(&mut self, semaphore: QueueHandle) -> Result<usize> {
464        self.queue_messages_waiting(semaphore)
465    }
466
467    /// `xSemaphoreGetMutexHolderFromISR`, which is
468    /// `xQueueGetMutexHolderFromISR`: the same answer, read without a
469    /// critical section because an ISR is already inside one.
470    ///
471    /// # Errors
472    /// [`Error::Gone`] for a stale handle.
473    pub fn mutex_holder_from_isr(&mut self, mutex: QueueHandle) -> Result<TaskHandle> {
474        self.queues.resolve(mutex).map(|q| q.holder)
475    }
476
477    /// `xSemaphoreGetMutexHolder`, which takes a critical section.
478    ///
479    /// # Errors
480    /// [`Error::Gone`] for a stale handle.
481    pub fn mutex_holder(&mut self, mutex: QueueHandle) -> Result<TaskHandle> {
482        self.enter_critical();
483        let holder = self.queues.resolve(mutex).map(|q| q.holder);
484        self.exit_critical();
485        holder
486    }
487
488    // -------------------------------------------------------------- send --
489
490    /// `xQueueSend` / `xQueueSendToBack`.
491    ///
492    /// # Errors
493    /// [`Error::Full`] when the block time expires with the queue still
494    /// full; [`Error::Gone`] for a stale handle.
495    pub fn queue_send(&mut self, queue: QueueHandle, value: u64, ticks: u64) -> Result<Wait<()>> {
496        self.queue_send_generic(queue, value, ticks, Position::Back)
497    }
498
499    /// `xQueueReset`, which is `xQueueGenericReset( q, pdFALSE )`: empty
500    /// the queue and unlock it, waking one waiting sender if there is one.
501    ///
502    /// The event lists survive — only a brand-new queue re-initialises
503    /// those — so a task blocked on this queue stays blocked, and the one
504    /// woken here is woken because the reset made room.
505    ///
506    /// # Errors
507    /// [`Error::Gone`] for a stale handle.
508    pub fn queue_reset(&mut self, queue: QueueHandle) -> Result<()> {
509        self.enter_critical();
510        let result = (|| {
511            let length = self.queues.resolve(queue)?.length;
512            {
513                let q = self.queues.resolve_mut(queue)?;
514                q.waiting = 0;
515                q.write_to = 0;
516                q.read_from = length.saturating_sub(1);
517                q.rx_lock = UNLOCKED;
518                q.tx_lock = UNLOCKED;
519            }
520            let senders = Self::queue_send_list(queue);
521            if !self.lists.is_empty_of(senders) && self.remove_from_event_list(senders)? {
522                // queueYIELD_IF_USING_PREEMPTION(), inside the section.
523                self.port_yield();
524            }
525            Ok(())
526        })();
527        self.exit_critical();
528        result
529    }
530
531    /// `uxQueueSpacesAvailable`: how many more items would fit.
532    ///
533    /// # Errors
534    /// [`Error::Gone`] for a stale handle.
535    pub fn queue_spaces_available(&mut self, queue: QueueHandle) -> Result<usize> {
536        self.enter_critical();
537        let spaces = self
538            .queues
539            .resolve(queue)
540            .map(|q| q.length.wrapping_sub(q.waiting));
541        self.exit_critical();
542        spaces
543    }
544
545    // --------------------------------------------------- queue sets --
546    //
547    // A queue set is a queue whose items are the handles of the queues in
548    // it. Nothing is *moved* into the set: an item arriving on a member
549    // queue puts that queue's handle on the set, and the task that wakes on
550    // the set reads the handle and then reads the member queue itself. So a
551    // set holds an announcement, not the data — which is why adding a queue
552    // that already has items in it is refused, and why the two counts would
553    // otherwise drift apart forever.
554
555    /// `xQueueCreateSet`: a set that can hold `length` announcements.
556    ///
557    /// # Errors
558    /// As [`Kernel::queue_create`].
559    pub fn queue_create_set(&mut self, length: usize) -> Result<QueueHandle> {
560        self.new_queue(length, Kind::Set)
561    }
562
563    /// `xQueueAddToSet`.
564    ///
565    /// `false` is the C's `pdFAIL`: the target is not a set, the queue is
566    /// already in one, or it has items waiting that the set never saw.
567    ///
568    /// # Errors
569    /// [`Error::Gone`] for a stale handle.
570    pub fn queue_add_to_set(&mut self, queue: QueueHandle, set: QueueHandle) -> Result<bool> {
571        self.enter_critical();
572        let result = (|| {
573            if self.queues.resolve(set)?.kind != Kind::Set {
574                return Ok(false);
575            }
576            let member = self.queues.resolve(queue)?;
577            if !member.set_container.is_null() || member.waiting != 0 {
578                return Ok(false);
579            }
580            self.queues.resolve_mut(queue)?.set_container = set;
581            Ok(true)
582        })();
583        self.exit_critical();
584        result
585    }
586
587    /// `xQueueRemoveFromSet`.
588    ///
589    /// # Errors
590    /// [`Error::Gone`] for a stale handle.
591    pub fn queue_remove_from_set(&mut self, queue: QueueHandle, set: QueueHandle) -> Result<bool> {
592        // Both refusals are tested OUTSIDE the section and only the removal
593        // itself is inside it, which is what the C does -- and it is NOT
594        // what `xQueueAddToSet` does, three functions up, where the whole
595        // body including both refusals is inside one section. The asymmetry
596        // is upstream's and it is load-bearing here: a failing remove costs
597        // the clock nothing, and `QueueSet`'s setup makes one deliberately,
598        // to prove a queue cannot be removed from a set it is not in.
599        // Charging for it put a tick one event early, 30 lines in.
600        let member = *self.queues.resolve(queue)?;
601        if member.set_container != set || member.waiting != 0 {
602            return Ok(false);
603        }
604        self.enter_critical();
605        let result = self
606            .queues
607            .resolve_mut(queue)
608            .map(|q| q.set_container = QueueHandle::NULL);
609        self.exit_critical();
610        result?;
611        Ok(true)
612    }
613
614    /// `xQueueSelectFromSet`: which member queue has something on it.
615    ///
616    /// The C is one line — `xQueueReceive( xQueueSet, &xReturn, xTicksToWait )`
617    /// — so this blocks, traces and costs exactly what a receive does, and
618    /// `None` is the C's `NULL`.
619    ///
620    /// # Errors
621    /// As [`Kernel::queue_receive`].
622    pub fn queue_select_from_set(
623        &mut self,
624        set: QueueHandle,
625        ticks: u64,
626    ) -> Result<Wait<Option<QueueHandle>>> {
627        match self.queue_receive(set, ticks) {
628            Ok(Blocked) => Ok(Blocked),
629            Ok(Ready(raw)) => {
630                let handle = QueueHandle::from_raw(u32::try_from(raw).unwrap_or(0));
631                Ok(Ready((!handle.is_null()).then_some(handle)))
632            }
633            Err(Error::Empty) => Ok(Ready(None)),
634            Err(e) => Err(e),
635        }
636    }
637
638    /// `xQueueSelectFromSetFromISR`: which queue in `set` has data, from
639    /// an interrupt.
640    ///
641    /// The C is three lines -- `xQueueReceiveFromISR` on the set itself,
642    /// with a NULL handle meaning "none" -- because a set IS a queue of
643    /// member handles. This is that, and it takes no critical section for
644    /// the same reason the rest of the FromISR surface does not.
645    ///
646    /// # Errors
647    /// [`Error::Gone`] for a stale handle.
648    pub fn queue_select_from_set_from_isr(
649        &mut self,
650        set: QueueHandle,
651    ) -> Result<Option<QueueHandle>> {
652        match self.queue_receive_from_isr(set) {
653            Ok((raw, _woken)) => {
654                let handle = QueueHandle::from_raw(u32::try_from(raw).unwrap_or(0));
655                Ok((!handle.is_null()).then_some(handle))
656            }
657            // An empty set is "no queue has data", which is what the C's
658            // NULL means; it is not a failure.
659            Err(Error::Empty) => Ok(None),
660            Err(e) => Err(e),
661        }
662    }
663
664    /// `prvNotifyQueueSetContainer`: put `queue`'s handle on the set it
665    /// belongs to. `true` when that woke a task that outranks the current
666    /// one.
667    /// `#[cold]` because the call is GUARDED and reached from many sites on
668    /// one hot path, which is the shape that pays: it lets LLVM keep the
669    /// caller's frame setup out of the likely route. Measured on
670    /// `riscv32-qemu-tick-work`, one attribute at a time.
671    /// Worth queue -1 on its own.
672    #[cold]
673    fn notify_queue_set_container(&mut self, queue: QueueHandle) -> Result<bool> {
674        let set = self.queues.resolve(queue)?.set_container;
675        let container = *self.queues.resolve(set)?;
676        if container.waiting >= container.length {
677            // The C asserts this cannot happen and does nothing if it does.
678            return Ok(false);
679        }
680        let tx_lock = container.tx_lock;
681        let tick = self.tick;
682        self.note_exits();
683        // `traceQUEUE_SET_SEND` is `traceQUEUE_SEND` unless a port says
684        // otherwise, and this port does not.
685        self.trace.event(
686            tick,
687            Event::QueueSend {
688                queue: set,
689                name: "",
690            },
691        );
692        let mut woke =
693            self.copy_data_to_queue(set, &container, u64::from(queue.to_raw()), Position::Back)?;
694        if tx_lock == UNLOCKED {
695            let receivers = Self::queue_receive_list(set);
696            if !self.lists.is_empty_of(receivers) && self.remove_from_event_list(receivers)? {
697                woke = true;
698            }
699        } else {
700            self.increment_tx_lock(set, tx_lock);
701        }
702        Ok(woke)
703    }
704
705    /// `xQueueOverwrite`: write to a length-one queue whether or not it
706    /// already holds something.
707    ///
708    /// The C is `xQueueGenericSend( q, item, 0, queueOVERWRITE )` — a send
709    /// with no block time, because a queue that is always writable can
710    /// never make the caller wait.
711    ///
712    /// # Errors
713    /// As [`Kernel::queue_send`].
714    pub fn queue_overwrite(&mut self, queue: QueueHandle, value: u64) -> Result<Wait<()>> {
715        self.send_generic_outlined(queue, value, 0, Position::Overwrite)
716    }
717
718    // ------------------------------------------------------- from an ISR --
719    //
720    // The `FromISR` half of the API. Three things make it a different
721    // animal from the task half, and all three are visible in a trace:
722    //
723    //   * It never blocks and never yields. It reports that a higher
724    //     priority task woke, by returning a [`Woken`], and the interrupt
725    //     decides what to do about it on the way out.
726    //   * Its critical section is `portSET_INTERRUPT_MASK_FROM_ISR`, not
727    //     `portENTER_CRITICAL`. On the Posix port both are empty, so a call
728    //     here costs no sim time at all — which is exactly what the C does
729    //     and therefore what a matching trace requires.
730    //   * If a task has the queue locked it may not touch the event lists.
731    //     It counts what it did in `cTxLock` / `cRxLock` instead, and
732    //     `prvUnlockQueue` pays the wake-ups back when the task is done.
733
734    /// `xQueueGenericSendFromISR`.
735    ///
736    /// # Errors
737    /// [`Error::Full`] when the queue is full and this is not an overwrite;
738    /// [`Error::Gone`] for a stale handle.
739    pub fn queue_send_generic_from_isr(
740        &mut self,
741        queue: QueueHandle,
742        value: u64,
743        position: Position,
744    ) -> Result<Woken> {
745        let mask = self.port.enter_critical_from_isr();
746        let result = self.send_from_isr_locked(queue, value, position);
747        self.port.exit_critical_from_isr(mask);
748        result
749    }
750
751    fn send_from_isr_locked(
752        &mut self,
753        queue: QueueHandle,
754        value: u64,
755        position: Position,
756    ) -> Result<Woken> {
757        let snapshot = *self.queues.resolve(queue)?;
758        if snapshot.waiting >= snapshot.length && position != Position::Overwrite {
759            // `traceQUEUE_SEND_FROM_ISR_FAILED` is not one of the harness's
760            // hooks, so a full queue says nothing on either side.
761            return Err(Error::Full);
762        }
763        let tx_lock = snapshot.tx_lock;
764        let tick = self.tick;
765        self.note_exits();
766        self.trace
767            .event(tick, Event::QueueSendFromIsr { queue, name: "" });
768        let previously_waiting = snapshot.waiting;
769        let _ = self.copy_data_to_queue(queue, &snapshot, value, position)?;
770        let mut woken = Woken::NO;
771        if tx_lock == UNLOCKED {
772            if C::USE_QUEUE_SETS && !snapshot.set_container.is_null() {
773                let overwrote = position == Position::Overwrite && previously_waiting > 0;
774                if !overwrote && self.notify_queue_set_container(queue)? {
775                    woken = Woken::YES;
776                }
777                return Ok(woken);
778            }
779            let receivers = Self::queue_receive_list(queue);
780            if !self.lists.is_empty_of(receivers) && self.remove_from_event_list(receivers)? {
781                woken = Woken::YES;
782            }
783        } else {
784            self.increment_tx_lock(queue, tx_lock);
785        }
786        Ok(woken)
787    }
788
789    /// `xQueueSendToBackFromISR`.
790    ///
791    /// # Errors
792    /// As [`Kernel::queue_send_generic_from_isr`].
793    pub fn queue_send_from_isr(&mut self, queue: QueueHandle, value: u64) -> Result<Woken> {
794        self.queue_send_generic_from_isr(queue, value, Position::Back)
795    }
796
797    /// `xQueueSendToFrontFromISR`.
798    ///
799    /// # Errors
800    /// As [`Kernel::queue_send_generic_from_isr`].
801    pub fn queue_send_to_front_from_isr(
802        &mut self,
803        queue: QueueHandle,
804        value: u64,
805    ) -> Result<Woken> {
806        self.queue_send_generic_from_isr(queue, value, Position::Front)
807    }
808
809    /// `xQueueOverwriteFromISR`. Only for a queue of length one.
810    ///
811    /// # Errors
812    /// As [`Kernel::queue_send_generic_from_isr`].
813    pub fn queue_overwrite_from_isr(&mut self, queue: QueueHandle, value: u64) -> Result<Woken> {
814        self.queue_send_generic_from_isr(queue, value, Position::Overwrite)
815    }
816
817    /// `xSemaphoreGiveFromISR`.
818    ///
819    /// # Errors
820    /// As [`Kernel::queue_send_generic_from_isr`].
821    pub fn semaphore_give_from_isr(&mut self, semaphore: QueueHandle) -> Result<Woken> {
822        self.queue_send_generic_from_isr(semaphore, 0, Position::Back)
823    }
824
825    /// `xQueueReceiveFromISR`: the value, and whether a waiting sender that
826    /// this made room for outranks the interrupted task.
827    ///
828    /// # Errors
829    /// [`Error::Empty`] when the queue is empty; [`Error::Gone`] for a
830    /// stale handle.
831    pub fn queue_receive_from_isr(&mut self, queue: QueueHandle) -> Result<(u64, Woken)> {
832        let mask = self.port.enter_critical_from_isr();
833        let result = self.receive_from_isr_locked(queue);
834        self.port.exit_critical_from_isr(mask);
835        result
836    }
837
838    fn receive_from_isr_locked(&mut self, queue: QueueHandle) -> Result<(u64, Woken)> {
839        let snapshot = *self.queues.resolve(queue)?;
840        if snapshot.waiting == 0 {
841            return Err(Error::Empty);
842        }
843        let rx_lock = snapshot.rx_lock;
844        let tick = self.tick;
845        self.note_exits();
846        self.trace
847            .event(tick, Event::QueueReceiveFromIsr { queue, name: "" });
848        let value = self.copy_data_from_queue(
849            queue,
850            snapshot.kind,
851            snapshot.base,
852            snapshot.length,
853            snapshot.read_from,
854            false,
855        )?;
856        let mut woken = Woken::NO;
857        if rx_lock == UNLOCKED {
858            let senders = Self::queue_send_list(queue);
859            if !self.lists.is_empty_of(senders) && self.remove_from_event_list(senders)? {
860                woken = Woken::YES;
861            }
862        } else {
863            self.increment_rx_lock(queue, rx_lock);
864        }
865        Ok((value, woken))
866    }
867
868    /// `xQueuePeekFromISR`: the head without removing it.
869    ///
870    /// It wakes nobody and traces nothing — the C has no hook for it —
871    /// which is why it returns a bare value rather than a [`Woken`].
872    ///
873    /// # Errors
874    /// [`Error::Empty`] when the queue is empty; [`Error::Gone`] for a
875    /// stale handle.
876    pub fn queue_peek_from_isr(&mut self, queue: QueueHandle) -> Result<u64> {
877        let mask = self.port.enter_critical_from_isr();
878        let result = match self.queues.resolve(queue) {
879            Ok(q) if q.waiting > 0 => {
880                let snapshot = *q;
881                self.copy_data_from_queue(
882                    queue,
883                    snapshot.kind,
884                    snapshot.base,
885                    snapshot.length,
886                    snapshot.read_from,
887                    true,
888                )
889            }
890            Ok(_) => Err(Error::Empty),
891            Err(e) => Err(e),
892        };
893        self.port.exit_critical_from_isr(mask);
894        result
895    }
896
897    /// `prvIncrementQueueTxLock`, capped at the task count as the C caps it.
898    fn increment_tx_lock(&mut self, queue: QueueHandle, tx_lock: i8) {
899        let tasks = self.task_count();
900        if let Ok(q) = self.queues.resolve_mut(queue) {
901            // 32-bit throughout. This was `i64::from(lock) < tasks as i64`,
902            // which is a SIXTY-FOUR-bit signed compare on a target whose
903            // registers are 32 -- to order an `i8` against a task count. The
904            // `< 0` arm is what makes the unsigned cast safe and is the
905            // UNLOCKED sentinel's case, which cannot reach the increment.
906            if tx_lock < 0 || (tx_lock as usize) < tasks {
907                q.tx_lock = tx_lock.wrapping_add(1);
908            }
909        }
910    }
911
912    /// `prvIncrementQueueRxLock`.
913    fn increment_rx_lock(&mut self, queue: QueueHandle, rx_lock: i8) {
914        let tasks = self.task_count();
915        if let Ok(q) = self.queues.resolve_mut(queue) {
916            // 32-bit throughout. This was `i64::from(lock) < tasks as i64`,
917            // which is a SIXTY-FOUR-bit signed compare on a target whose
918            // registers are 32 -- to order an `i8` against a task count. The
919            // `< 0` arm is what makes the unsigned cast safe and is the
920            // UNLOCKED sentinel's case, which cannot reach the increment.
921            if rx_lock < 0 || (rx_lock as usize) < tasks {
922                q.rx_lock = rx_lock.wrapping_add(1);
923            }
924        }
925    }
926
927    /// `xQueueSendToFront`.
928    ///
929    /// # Errors
930    /// As [`Kernel::queue_send`].
931    pub fn queue_send_to_front(
932        &mut self,
933        queue: QueueHandle,
934        value: u64,
935        ticks: u64,
936    ) -> Result<Wait<()>> {
937        self.send_generic_outlined(queue, value, ticks, Position::Front)
938    }
939
940    /// `xSemaphoreGive`: a send with no data.
941    ///
942    /// # Errors
943    /// [`Error::Full`] when the semaphore is already at its maximum count.
944    pub fn semaphore_give(&mut self, semaphore: QueueHandle) -> Result<Wait<()>> {
945        self.send_generic_outlined(semaphore, 0, 0, Position::Back)
946    }
947
948    /// The out-of-line handle on [`Kernel::queue_send_generic`]'s body.
949    ///
950    /// One shared copy for the wrappers that are not on a measured hot row --
951    /// `queue_send_to_front`, `queue_overwrite`, `semaphore_give` and the
952    /// timer daemon's post. `queue_send` keeps the inlined body because
953    /// `send_full`, `queue_roundtrip` and `block_cycle` all go through it and
954    /// all three pay for the call.
955    #[inline(never)]
956    pub(crate) fn send_generic_outlined(
957        &mut self,
958        queue: QueueHandle,
959        value: u64,
960        ticks: u64,
961        position: Position,
962    ) -> Result<Wait<()>> {
963        self.queue_send_generic(queue, value, ticks, position)
964    }
965
966    /// `xQueueGenericSend`, one pass of its `for(;;)`.
967    ///
968    /// # Errors
969    /// [`Error::Full`] on timeout; [`Error::Gone`] for a stale handle.
970    /// In line on purpose, for the same reason as
971    /// [`Kernel::queue_take`]: seven instructions of prologue and ten of
972    /// epilogue on every one of khot-ir's 48,000 sends, so that `xQueueSend`
973    /// and its siblings could pass four arguments through unchanged. This
974    /// one pays on every instrument, `kernel-ir` included.
975    ///
976    /// **But only the HOT site wants it.** With the flash probe fixed so that
977    /// real bodies link at all (2026-09-24), this was inlined at five places
978    /// and cost about 1,666 bytes of duplicate.
979    ///
980    /// RE-PRICED 2026-09-25, because that figure was taken at FIVE inline sites
981    /// and there is now one. Dropping the hint today measures:
982    ///
983    /// * flash **19,302 -> 18,704, i.e. -598 B**, with `mv` -31, `andi` -7,
984    ///   `slli` -8 and 221 fewer instructions;
985    /// * rv32 work rows **+118 instructions** over seven of them —
986    ///   `queue_roundtrip` 122 -> 169, `block_cycle` 983 -> 1022,
987    ///   `send_full` 36 -> 62, and +1 or +2 on four more.
988    ///
989    /// That is **5.07 bytes of flash per instruction**, a worse rate than the
990    /// `queue_peek` trade declined at 7.3 and than the 10.7 this hint was
991    /// originally kept at. So it stays — but the K3 flash row is the one
992    /// FAILING its target while the work rows are absolutes with no C
993    /// comparison, so if that priority ever inverts this is the single largest
994    /// item in the send path and the numbers are here. `rusty-compiler-leverage` A4: a function has one
995    /// body, so one inlining decision serves callers who want different
996    /// things -- give it two names instead of averaging them.
997    /// [`Kernel::send_generic_outlined`] is the out-of-line handle, and the
998    /// four colder wrappers call that.
999    #[cfg_attr(not(feature = "small"), inline(always))]
1000    #[cfg_attr(feature = "small", inline(never))]
1001    pub fn queue_send_generic(
1002        &mut self,
1003        queue: QueueHandle,
1004        value: u64,
1005        ticks: u64,
1006        position: Position,
1007    ) -> Result<Wait<()>> {
1008        let caller = self.cur();
1009        // Resuming below the sampling exit, as the C's thread does. See
1010        // `Kernel::take_queue_resume`: the prologue has already run, and
1011        // repeating it would spend a second critical-section exit that the
1012        // C never spends.
1013        if self.take_queue_resume(caller) != QueueResume::No {
1014            // The send's only resume point is below its sampling exit: its
1015            // timed-out branch has nothing after `resume_all` that costs an
1016            // exit, so being switched away there changes no count.
1017            return self.queue_send_blocking(caller, queue);
1018        }
1019        self.enter_critical();
1020        let snapshot = match self.queues.resolve(queue) {
1021            Ok(q) => *q,
1022            Err(e) => {
1023                self.exit_critical();
1024                return Err(e);
1025            }
1026        };
1027        // The wait frame is set where the C sets it: `xQueueReceive` and
1028        // `xQueueGenericSend` call `vTaskInternalSetTimeOutState` only
1029        // after finding the queue unusable AND the block time non-zero,
1030        // guarded by `xEntryTimeSet`. Setting it up here instead meant
1031        // every *successful* call wrote a six-field frame and then had
1032        // `end_wait` wipe it on the way out.
1033        //
1034        // It also has to be after the resolve, so that a handle naming
1035        // nothing leaves the caller exactly as it found it -- which is
1036        // what `configASSERT( pxQueue )` means in the C.
1037        // `( uxMessagesWaiting < uxLength ) || ( xCopyPosition == queueOVERWRITE )`
1038        if snapshot.waiting < snapshot.length || position == Position::Overwrite {
1039            self.note_exits();
1040            let tick = self.tick;
1041            self.trace.event(tick, Event::QueueSend { queue, name: "" });
1042            // An emitting sink is a CALL here, and the whole snapshot was held
1043            // live across it -- seven fields spilled and reloaded on every
1044            // send. Read again after the call instead: nothing in the sink can
1045            // reach the arena, so it is the same value, and a re-resolve is
1046            // cheaper than the spills. A silent sink has no call, and keeps
1047            // the one read.
1048            let snapshot = if T::EMITS {
1049                match self.queues.resolve(queue) {
1050                    Ok(q) => *q,
1051                    Err(e) => {
1052                        self.exit_critical();
1053                        return Err(e);
1054                    }
1055                }
1056            } else {
1057                snapshot
1058            };
1059            let previously_waiting = snapshot.waiting;
1060            let yield_required = self.copy_data_to_queue(queue, &snapshot, value, position)?;
1061            if C::USE_QUEUE_SETS && !snapshot.set_container.is_null() {
1062                // A queue in a set announces the arrival there, not here.
1063                // An overwrite of an item that was already present is not an
1064                // arrival: the count did not change, so the set is not told.
1065                let overwrote = position == Position::Overwrite && previously_waiting > 0;
1066                if !overwrote && self.notify_queue_set_container(queue)? {
1067                    self.port_yield();
1068                }
1069                self.exit_critical();
1070                self.end_wait(caller);
1071                return Ok(Ready(()));
1072            }
1073            let receivers = Self::queue_receive_list(queue);
1074            // The C's `if( waiters ) { ... } else if( xYieldRequired )`: the
1075            // emptiness is answered ONCE, BEFORE a waiter is removed. With
1076            // waiters, only "the woken task outranks me" yields; the
1077            // disinherit's yield request is consulted only when there were
1078            // none. This used to re-read the list AFTER the removal, so
1079            // removing the only waiter made the list empty and a give that
1080            // disinherited AND woke a lower-priority task yielded where the
1081            // C does not (one core; on two the disinherit yields the core
1082            // itself). Found by the API differential, P1.3, step 4105.
1083            let had_receivers = !self.lists.is_empty_of(receivers);
1084            let woke_higher = if had_receivers {
1085                self.remove_from_event_list(receivers)?
1086            } else {
1087                false
1088            };
1089            // queueYIELD_IF_USING_PREEMPTION(), inside the section.
1090            if woke_higher || (yield_required && !had_receivers) {
1091                self.port_yield();
1092            }
1093            self.exit_critical();
1094            self.end_wait(caller);
1095            return Ok(Ready(()));
1096        }
1097        let remaining = self.begin_wait(caller, queue, ticks);
1098        if remaining.unwrap_or(0) == 0 {
1099            self.exit_critical();
1100            self.trace_failure_or_owe(caller, OwedTrace::SendFailed(queue));
1101            if remaining.is_some() {
1102                // A frame was written, so it has to be torn down. `None` means
1103                // `begin_wait` took its zero-block-time exit and wrote
1104                // nothing, and then `end_wait` is a provable no-op.
1105                self.end_wait(caller);
1106            }
1107            return Err(Error::Full);
1108        }
1109        self.exit_critical();
1110        // That exit can release a tick, and a tick can switch this task
1111        // away. The C's thread stops INSIDE the exit, so everything below
1112        // runs when the task is scheduled again -- and names that task.
1113        if self.cur() != caller {
1114            self.set_queue_resume(caller, QueueResume::BelowSample);
1115            return Ok(Blocked);
1116        }
1117        self.queue_send_blocking(caller, queue)
1118    }
1119
1120    /// Everything `xQueueGenericSend` does below its sampling exit.
1121    ///
1122    /// Split out so a call preempted at that exit re-enters here rather
1123    /// than at the top. See [`Kernel::take_queue_resume`].
1124    #[cold]
1125    fn queue_send_blocking(&mut self, caller: TaskHandle, queue: QueueHandle) -> Result<Wait<()>> {
1126        self.suspend_all();
1127        self.lock_queue(queue);
1128        let left = self.check_for_timeout(caller);
1129        if left.is_none() {
1130            self.unlock_queue(queue)?;
1131            let _ = self.resume_all();
1132            self.trace_failure_or_owe(caller, OwedTrace::SendFailed(queue));
1133            self.end_wait(caller);
1134            return Err(Error::Full);
1135        }
1136        if self.is_queue_full(queue) {
1137            self.note_exits();
1138            let tick = self.tick;
1139            self.trace
1140                .event(tick, Event::BlockingOnQueueSend { queue, name: "" });
1141            // `check_for_timeout` above already resolved the TCB and wrote
1142            // this field; `remaining_ticks` resolved it a second time to read
1143            // it back.
1144            let ticks_left = left.map_or(0, core::num::NonZeroU64::get);
1145            self.place_on_event_list(Self::queue_send_list(queue), ticks_left)?;
1146            self.unlock_queue(queue)?;
1147            if !self.resume_all() {
1148                self.yield_or_owe(caller);
1149            }
1150        } else {
1151            self.unlock_queue(queue)?;
1152            let _ = self.resume_all();
1153        }
1154        Ok(Blocked)
1155    }
1156
1157    /// `prvCopyDataToQueue`; `true` when giving a mutex back lowered the
1158    /// giver's priority and a yield is therefore wanted.
1159    ///
1160    /// This resolves rather than taking the caller's snapshot, and
1161    /// [`Kernel::copy_data_from_queue`] does the opposite. That is not an
1162    /// inconsistency: both forms were measured on both helpers and they
1163    /// disagree, because the writer's caller does not keep its snapshot
1164    /// live across the call and the reader's does.
1165    ///
1166    /// In line on purpose, at ALL THREE call sites.
1167    ///
1168    /// The rationale here used to say `queue_send_generic` was its only caller.
1169    /// That is no longer true — `queue_add_to_set` and
1170    /// `queue_send_generic_from_isr` call it too — so an A4 split was tried on
1171    /// 2026-09-25, keeping the hint for the hot caller and routing the cold ones
1172    /// through an out-of-line handle. It measured **+72 B of flash**, `mv` +12
1173    /// and 49 more instructions, and was reverted. `prime_mutex` was a fourth
1174    /// caller when that was measured; it was deleted later the same day (see
1175    /// [`Self::new_mutex`]) and the split was **re-tested at three callers**:
1176    /// **+6 B**, still a loss but only a twelfth of the old one. So ~66 of the
1177    /// original 72 bytes were `prime_mutex`'s own site — an A4 verdict can be
1178    /// carried almost entirely by ONE caller, which is why §9's re-test law
1179    /// exists. At three callers this is now a near-tie held by 6 bytes.
1180    ///
1181    /// The reason is worth keeping: the cold callers pass CONSTANT arguments
1182    /// (`Position::Back` from `queue_add_to_set`), so each inlined copy is
1183    /// SPECIALISED and folds well below the 138 bytes C's `prvCopyDataToQueue`
1184    /// occupies. Out of line they pay a five-argument frame and force
1185    /// `&snapshot` into memory, which costs more than the copies they remove.
1186    ///
1187    /// The hot reason also still stands: out of line it paid a call and a frame
1188    /// on all 48,000 sends of `bench/kernel-ir` to move one value into a slot
1189    /// the caller had already resolved.
1190    #[inline(always)]
1191    fn copy_data_to_queue(
1192        &mut self,
1193        queue: QueueHandle,
1194        snapshot: &Queue,
1195        value: u64,
1196        position: Position,
1197    ) -> Result<bool> {
1198        // The snapshot comes from the caller, which has already resolved this
1199        // handle -- and nothing between that resolve and this call can reach
1200        // the arena. Resolving again here made every send pay for the same
1201        // four checks twice. `copy_data_from_queue` has always taken its
1202        // snapshot this way.
1203        let mut yield_required = false;
1204        // `prvCopyDataToQueue` counts the item in — except on an overwrite
1205        // of a queue that already held one, where it decrements first so
1206        // the increment cancels and the count stays where it was.
1207        //
1208        // Decided once, up here, so that each arm below can fold the count
1209        // into the resolve it is already making. Deciding it after the arms
1210        // is what forced a further resolve for one field.
1211        let counted = !(position == Position::Overwrite && snapshot.waiting > 0);
1212        if snapshot.kind.carries_data() {
1213            let (index, next_read, next_write) = match position {
1214                Position::Back => (
1215                    snapshot.write_to,
1216                    snapshot.read_from,
1217                    wrap_next(snapshot.write_to, snapshot.length),
1218                ),
1219                // Write at the item last read, then step back, so the next
1220                // read's pre-increment lands on it. An overwrite takes the
1221                // same path; only the message count differs.
1222                Position::Front | Position::Overwrite => (
1223                    snapshot.read_from,
1224                    wrap_prev(snapshot.read_from, snapshot.length),
1225                    snapshot.write_to,
1226                ),
1227            };
1228            // `base + index` is below `base + length`, which the geometry
1229            // put inside `SLOTS`; and the `get_mut` refuses anything it is
1230            // not, so nothing rests on the arithmetic either way.
1231            //
1232            // A silent sink writes the queue's fields FIRST: then nothing
1233            // stands between the caller's resolve and this one, they fold, and
1234            // the handle is not re-checked after the slot store, which LLVM
1235            // cannot prove misses the queue. With an emitting sink the trace
1236            // call already stands between them, and the reorder only cost
1237            // codegen (refuted R2: host +32,092).
1238            if !T::EMITS {
1239                let q = self.queues.resolve_mut(queue)?;
1240                q.read_from = next_read;
1241                q.write_to = next_write;
1242                if counted {
1243                    q.waiting = q.waiting.wrapping_add(1);
1244                }
1245            }
1246            if let Some(slot) = self.slots.get_mut(snapshot.base.wrapping_add(index)) {
1247                *slot = value;
1248            }
1249            if T::EMITS {
1250                let q = self.queues.resolve_mut(queue)?;
1251                q.read_from = next_read;
1252                q.write_to = next_write;
1253                if counted {
1254                    q.waiting = q.waiting.wrapping_add(1);
1255                }
1256            }
1257        } else if snapshot.kind.is_mutex() {
1258            // Giving a mutex back: the holder drops any inherited
1259            // priority. The disinherit touches the holder's TCB, not this
1260            // queue, so the resolve after it is still the first one.
1261            yield_required = self.priority_disinherit(snapshot.holder)?;
1262            let q = self.queues.resolve_mut(queue)?;
1263            q.holder = TaskHandle::NULL;
1264            if counted {
1265                q.waiting = q.waiting.wrapping_add(1);
1266            }
1267        } else if counted {
1268            // A counting or binary semaphore: only the count moves.
1269            let q = self.queues.resolve_mut(queue)?;
1270            q.waiting = q.waiting.wrapping_add(1);
1271        }
1272        Ok(yield_required)
1273    }
1274
1275    // ----------------------------------------------------------- receive --
1276
1277    /// The out-of-line handle on [`Kernel::queue_take`]s body.
1278    ///
1279    /// `peek` was a `const PEEK: bool`, which gave the whole take chain TWO
1280    /// monomorphisations and three inlined copies of a ~600-byte body that no
1281    /// A4 split could share, because `<true>` and `<false>` are different
1282    /// functions. As a runtime flag there is one body to share.
1283    ///
1284    /// Only `semaphore_take` uses this handle. `queue_receive` and `queue_peek`
1285    /// keep the inlined body because all four of `recv_empty`, `peek_ok`,
1286    /// `queue_roundtrip` and `block_cycle` go through one or the other --
1287    /// routing `queue_peek` here cost `peek_ok` **45 -> 108**, which is what a
1288    /// call and a frame are worth on a row that short.
1289    #[inline(never)]
1290    fn take_outlined(&mut self, queue: QueueHandle, ticks: u64, peek: bool) -> Result<Wait<u64>> {
1291        self.queue_take(queue, ticks, peek)
1292    }
1293
1294    /// `xQueueReceive`.
1295    ///
1296    /// # Errors
1297    /// [`Error::Empty`] when the block time expires with the queue still
1298    /// empty; [`Error::Gone`] for a stale handle.
1299    pub fn queue_receive(&mut self, queue: QueueHandle, ticks: u64) -> Result<Wait<u64>> {
1300        self.queue_take(queue, ticks, false)
1301    }
1302
1303    /// `xQueuePeek`: read the head without removing it.
1304    ///
1305    /// # Errors
1306    /// As [`Kernel::queue_receive`].
1307    pub fn queue_peek(&mut self, queue: QueueHandle, ticks: u64) -> Result<Wait<u64>> {
1308        self.queue_take(queue, ticks, true)
1309    }
1310
1311    /// `xSemaphoreTake`, which is `xQueueSemaphoreTake`: a receive with no
1312    /// data, and — on a mutex — with priority inheritance while it waits.
1313    ///
1314    /// # Errors
1315    /// [`Error::Empty`] on timeout; [`Error::Gone`] for a stale handle.
1316    // A3: ONE caller, so the out-of-line body pays a prologue and epilogue for
1317    // a single call. Inlining moves the body rather than duplicating it.
1318    #[inline]
1319    pub fn semaphore_take(&mut self, semaphore: QueueHandle, ticks: u64) -> Result<Wait<()>> {
1320        match self.take_outlined(semaphore, ticks, false)? {
1321            Blocked => Ok(Blocked),
1322            Ready(_) => Ok(Ready(())),
1323        }
1324    }
1325
1326    /// The shared body of `xQueueReceive`, `xQueuePeek` and
1327    /// `xQueueSemaphoreTake` — one pass of the C `for(;;)`.
1328    /// `peek` is a const because it is one at every call site: a receive
1329    /// passes `false`, a peek `true`, and a semaphore take `false`. As a
1330    /// parameter it cost an argument to set up and a branch at each of the
1331    /// ten places this function and its copy helper consult it.
1332    /// In line on purpose, into all three of `xQueueReceive`, `xQueuePeek`
1333    /// and `xSemaphoreTake`.
1334    ///
1335    /// Out of line it charged eight instructions of prologue and ten of
1336    /// epilogue to every take -- 48,000 of them in `khot-ir`, which is 7.6%
1337    /// of that instrument -- so that three wrappers could each hand it two
1338    /// arguments and return its result unchanged. LLVM declined on size; the
1339    /// frame is the reason to overrule that.
1340    ///
1341    /// The cost is stated rather than hidden: `kernel-ir` pays 214,804 for
1342    /// the larger wrappers. Splitting the body from the symbol does not
1343    /// recover it -- outlining at the semaphore site alone leaves `kernel-ir`
1344    /// 262,828 up and gives back 427,983 of `khot-ir`'s win, and outlining at
1345    /// the receive site leaves it 190,238 up and gives back 1,104,183 -- so
1346    /// the regression is the wrappers growing, not any one call site.
1347    #[cfg_attr(not(feature = "small"), inline(always))]
1348    #[cfg_attr(feature = "small", inline(never))]
1349    fn queue_take(&mut self, queue: QueueHandle, ticks: u64, peek: bool) -> Result<Wait<u64>> {
1350        let caller = self.cur();
1351        // Resuming below the sampling exit, as `queue_send_generic` does.
1352        // The snapshot is re-read rather than carried: nothing below uses
1353        // anything of it that a concurrent access could change, and the
1354        // read takes no critical section, so it costs the clock nothing.
1355        match self.take_queue_resume(caller) {
1356            QueueResume::No => {}
1357            QueueResume::BelowSample => {
1358                return self.queue_take_blocking(caller, queue, peek);
1359            }
1360            QueueResume::BelowTimedOutResume => {
1361                return self.queue_take_timed_out(queue, peek);
1362            }
1363        }
1364        self.enter_critical();
1365        // ONE resolve, mutable, for the whole fast path, reading only the
1366        // fields it uses. Two earlier shapes, both measured:
1367        //  - `*q`, the whole eleven-field descriptor copied to reach five of
1368        //    them: nine words of `usize` on rv32 with four dead on every
1369        //    successful receive, and more live fields than LLVM keeps in
1370        //    hand, so the function claimed callee-saved registers it had no
1371        //    use for;
1372        //  - the five fields read through `resolve`, then the item taken
1373        //    through `copy_data_from_queue`, which resolved the handle AGAIN
1374        //    to write `read_from` and `waiting` back. On rv32 LLVM merged the
1375        //    two at run time, but every inlined copy carried the second body
1376        //    (-242 B of flash, -395,353 kernel-ir), and on the Xtensa S3 the
1377        //    copy was an outlined call on every receive.
1378        // The blocking tail below still wants the whole descriptor and
1379        // re-resolves for it, outside the section and at no cost to the
1380        // clock, exactly as the resume arms above do.
1381        let q = match self.queues.resolve_mut(queue) {
1382            Ok(q) => q,
1383            Err(e) => {
1384                self.exit_critical();
1385                return Err(e);
1386            }
1387        };
1388        let (waiting, kind) = (q.waiting, q.kind);
1389        // The wait frame is set where the C sets it: `xQueueReceive` and
1390        // `xQueueGenericSend` call `vTaskInternalSetTimeOutState` only
1391        // after finding the queue unusable AND the block time non-zero,
1392        // guarded by `xEntryTimeSet`. Setting it up here instead meant
1393        // every *successful* call wrote a six-field frame and then had
1394        // `end_wait` wipe it on the way out.
1395        //
1396        // It also has to be after the resolve, so that a handle naming
1397        // nothing leaves the caller exactly as it found it -- which is
1398        // what `configASSERT( pxQueue )` means in the C.
1399        if waiting > 0 {
1400            // `copy_data_from_queue`, in place, on the reference above.
1401            let value = if kind.carries_data() {
1402                let next = wrap_next(q.read_from, q.length);
1403                let value = self
1404                    .slots
1405                    .get(q.base.wrapping_add(next))
1406                    .copied()
1407                    .unwrap_or(0);
1408                if !peek {
1409                    q.read_from = next;
1410                    // Wrapping: `waiting > 0` was read under this section.
1411                    q.waiting = q.waiting.wrapping_sub(1);
1412                }
1413                value
1414            } else {
1415                if !peek {
1416                    q.waiting = q.waiting.wrapping_sub(1);
1417                }
1418                0
1419            };
1420            self.note_exits();
1421            let tick = self.tick;
1422            // `xQueuePeek` is its own function in the C with its own trace
1423            // macro; `xQueueSemaphoreTake` shares `xQueueReceive`'s.
1424            let event = if peek {
1425                Event::QueuePeek { queue, name: "" }
1426            } else {
1427                Event::QueueReceive { queue, name: "" }
1428            };
1429            self.trace.event(tick, event);
1430            if peek {
1431                // A peek wakes another *receiver*, not a sender: the item
1432                // is still there.
1433                let receivers = Self::queue_receive_list(queue);
1434                if !self.lists.is_empty_of(receivers) && self.remove_from_event_list(receivers)? {
1435                    self.port_yield();
1436                }
1437            } else {
1438                if kind.is_mutex() {
1439                    self.queues.resolve_mut(queue)?.holder = caller;
1440                    self.increment_mutexes_held(caller);
1441                }
1442                let senders = Self::queue_send_list(queue);
1443                if !self.lists.is_empty_of(senders) && self.remove_from_event_list(senders)? {
1444                    self.port_yield();
1445                }
1446            }
1447            self.exit_critical();
1448            self.end_wait(caller);
1449            return Ok(Ready(value));
1450        }
1451        let remaining = self.begin_wait(caller, queue, ticks);
1452        if remaining.unwrap_or(0) == 0 {
1453            self.exit_critical();
1454            // `traceQUEUE_peek_FAILED` is not one of the harness's hooks,
1455            // so a failed peek says nothing on either side.
1456            if !peek {
1457                self.trace_failure_or_owe(caller, OwedTrace::ReceiveFailed(queue));
1458            }
1459            if remaining.is_some() {
1460                self.end_wait(caller);
1461            }
1462            return Err(Error::Empty);
1463        }
1464        self.exit_critical();
1465        // That exit can release a tick, and a tick can switch this task
1466        // away. The C's thread stops INSIDE the exit, so everything below
1467        // runs when the task is scheduled again -- and names that task.
1468        if self.cur() != caller {
1469            self.set_queue_resume(caller, QueueResume::BelowSample);
1470            return Ok(Blocked);
1471        }
1472        self.queue_take_blocking(caller, queue, peek)
1473    }
1474
1475    /// Everything `xQueueReceive` and `xQueuePeek` do below their sampling
1476    /// exit. See [`Kernel::take_queue_resume`].
1477    #[cold]
1478    fn queue_take_blocking(
1479        &mut self,
1480        caller: TaskHandle,
1481        queue: QueueHandle,
1482        peek: bool,
1483    ) -> Result<Wait<u64>> {
1484        // `kind` is read HERE rather than handed down from the caller, because
1485        // it is the one field of the descriptor that cannot change after
1486        // `new_queue` sets it -- so resolving it late reads the same byte, and
1487        // resolving it early only lengthened a live range across the hot
1488        // function's whole body.
1489        //
1490        // SINKING it below the timed-out branch -- which never reads it, and
1491        // whose callee resolves `kind` for itself -- removes a resolve from
1492        // that path and measured **+2 B** on 2026-09-25. Refuted: the resolve
1493        // is still emitted once at the later site and the branch around it grew
1494        // by more. Both paths are `#[cold]`, so there was no runtime case to
1495        // weigh against the bytes.
1496        let kind = self.queues.resolve(queue)?.kind;
1497        self.suspend_all();
1498        self.lock_queue(queue);
1499        let left = self.check_for_timeout(caller);
1500        if left.is_none() {
1501            self.unlock_queue(queue)?;
1502            let _ = self.resume_all();
1503            // `xTaskResumeAll` replays any tick that pended while the
1504            // scheduler was suspended, and replaying it can switch this task
1505            // away with `prvIsQueueEmpty` still to run. The C's thread stops
1506            // inside it; everything below runs when the task is next
1507            // scheduled. Without this, `prvIsQueueEmpty`'s section is spent
1508            // in the wrong window -- one exit, and IntQueue's 100,000-tick
1509            // run diverges at tick 16,260.
1510            if self.cur() != caller {
1511                self.set_queue_resume(caller, QueueResume::BelowTimedOutResume);
1512                return Ok(Blocked);
1513            }
1514            return self.queue_take_timed_out(queue, peek);
1515        }
1516        // `left` is `Some` here, and it carries the block time
1517        // `check_for_timeout` has just written into the TCB.
1518        self.queue_take_locked(
1519            caller,
1520            queue,
1521            peek,
1522            kind,
1523            left.map_or(0, core::num::NonZeroU64::get),
1524        )
1525    }
1526
1527    /// `xQueueReceive` below the `xTaskResumeAll` of its timed-out branch.
1528    /// `caller` is read here rather than passed, for the reason given on
1529    /// [`Kernel::queue_take_blocking`]: both call sites have already
1530    /// established that it is `self.cur()`.
1531    #[cold]
1532    fn queue_take_timed_out(&mut self, queue: QueueHandle, peek: bool) -> Result<Wait<u64>> {
1533        let caller = self.cur();
1534        // `kind` is read HERE rather than handed down from the caller,
1535        // because it is the one field of the descriptor that cannot change
1536        // after `new_queue` sets it -- so resolving it late reads the same
1537        // byte, and resolving it early only lengthened a live range across
1538        // the hot function's whole body.
1539        let kind = self.queues.resolve(queue)?.kind;
1540        {
1541            if self.is_queue_empty(queue) {
1542                if kind.is_mutex() && self.wait_inherited(caller) {
1543                    // `vTaskPriorityDisinheritAfterTimeout`: the holder
1544                    // keeps only what the still-waiting tasks justify.
1545                    self.enter_critical();
1546                    let highest = self.highest_waiting_priority(queue)?;
1547                    let holder = self.queues.resolve(queue).map(|q| q.holder)?;
1548                    self.priority_disinherit_after_timeout(holder, highest)?;
1549                    self.exit_critical();
1550                }
1551                if !peek {
1552                    self.trace_failure_or_owe(caller, OwedTrace::ReceiveFailed(queue));
1553                }
1554                self.end_wait(caller);
1555                return Err(Error::Empty);
1556            }
1557            Ok(Blocked)
1558        }
1559    }
1560
1561    /// `xQueueReceive` below its timeout test, with the queue still locked.
1562    fn queue_take_locked(
1563        &mut self,
1564        caller: TaskHandle,
1565        queue: QueueHandle,
1566        peek: bool,
1567        // The descriptor's `kind`, and ONLY that. All three of these
1568        // functions took the whole `Queue` by value -- thirty-six bytes on
1569        // rv32, copied at every hop of a three-deep chain -- to read
1570        // `kind.is_mutex()` twice. Nothing else of it was ever touched.
1571        kind: Kind,
1572        ticks_left: u64,
1573    ) -> Result<Wait<u64>> {
1574        // Both arms end the same way -- unlock, then resume -- and
1575        // `resume_all_inline` is exactly that: INLINE. Written as two arms it
1576        // was two copies of its body in one function. The sequence below is
1577        // the sequence both arms had, in the same order, because on the sim a
1578        // critical-section exit is the clock and reordering one is a
1579        // divergence.
1580        let was_empty = self.is_queue_empty(queue);
1581        if was_empty {
1582            let event = if peek {
1583                Event::BlockingOnQueuePeek { queue, name: "" }
1584            } else {
1585                Event::BlockingOnQueueReceive { queue, name: "" }
1586            };
1587            self.note_exits();
1588            let tick = self.tick;
1589            self.trace.event(tick, event);
1590            if kind.is_mutex() {
1591                self.enter_critical();
1592                let holder = self.queues.resolve(queue).map(|q| q.holder)?;
1593                let inherited = self.priority_inherit(holder)?;
1594                self.exit_critical();
1595                // Assigned, not latched: the C's `xInheritanceOccurred` is
1596                // whatever the LATEST block's inherit answered. Latching it
1597                // made a timeout disinherit after a re-block that inherited
1598                // nothing (API differential P1.4, two cores, step 2390: a
1599                // waiter lowered to priority 0 while it waited).
1600                self.set_wait_inherited(caller, inherited);
1601            }
1602            self.place_on_event_list(Self::queue_receive_list(queue), ticks_left)?;
1603        }
1604        self.unlock_queue(queue)?;
1605        let yielded = self.resume_all_inline();
1606        if was_empty && !yielded {
1607            self.yield_or_owe(caller);
1608        }
1609        Ok(Blocked)
1610    }
1611
1612    /// `prvCopyDataFromQueue`, with the peek variant putting the cursor
1613    /// back as `xQueuePeek` does -- over fields its caller already read.
1614    ///
1615    /// Every caller resolves this queue to decide whether there is
1616    /// anything to read; this used to resolve it again to do the reading.
1617    /// The snapshot cannot be stale — nothing between the two touches this
1618    /// queue, and both are inside the same critical section.
1619    ///
1620    /// The same change to [`Kernel::copy_data_to_queue`] is **refused**,
1621    /// and the pair is why this bench exists: threading the snapshot into
1622    /// the writer is worth -245,389 Ir inside it and +211,717 in
1623    /// `queue_send_generic`, because a `&Queue` argument forces the
1624    /// caller's local to be addressable and it stops living in registers.
1625    /// Here the caller is `queue_take`, which keeps its snapshot alive
1626    /// across the call anyway, so there is no spill to pay for.
1627    fn copy_data_from_queue(
1628        &mut self,
1629        queue: QueueHandle,
1630        kind: Kind,
1631        base: usize,
1632        length: usize,
1633        read_from: usize,
1634        peek: bool,
1635    ) -> Result<u64> {
1636        if !kind.carries_data() {
1637            if !peek {
1638                let q = self.queues.resolve_mut(queue)?;
1639                // Wrapping: all three callers reach here under
1640                // `waiting > 0`, and the doc above says the snapshot they
1641                // tested cannot be stale, so there is an item to take.
1642                q.waiting = q.waiting.wrapping_sub(1);
1643            }
1644            return Ok(0);
1645        }
1646        let next = wrap_next(read_from, length);
1647        let value = self
1648            .slots
1649            .get(base.wrapping_add(next))
1650            .copied()
1651            .unwrap_or(0);
1652        // `xQueuePeek` saves and restores `pcReadFrom` — which is to
1653        // say it leaves it exactly as it found it. Writing it back stored
1654        // the value already there, and the resolve that reached it was a
1655        // whole handle validation for a no-op.
1656        if !peek {
1657            let q = self.queues.resolve_mut(queue)?;
1658            q.read_from = next;
1659            // Wrapping, for the same reason as the branch above.
1660            q.waiting = q.waiting.wrapping_sub(1);
1661        }
1662        Ok(value)
1663    }
1664
1665    // ------------------------------------------------------ recursive mutex --
1666
1667    /// `xSemaphoreTakeRecursive`.
1668    ///
1669    /// # Errors
1670    /// [`Error::Empty`] on timeout; [`Error::Gone`] for a stale handle.
1671    pub fn mutex_take_recursive(&mut self, mutex: QueueHandle, ticks: u64) -> Result<Wait<()>> {
1672        let caller = self.cur();
1673        // One resolve reads the holder and counts the re-entry.
1674        // `resolve_mut` runs the same four checks and answers the same
1675        // errors, so asking twice only asked twice.
1676        {
1677            let q = self.queues.resolve_mut(mutex)?;
1678            if q.holder == caller {
1679                q.recursions = q.recursions.wrapping_add(1);
1680                return Ok(Ready(()));
1681            }
1682        }
1683        match self.semaphore_take(mutex, ticks)? {
1684            Blocked => Ok(Blocked),
1685            Ready(()) => {
1686                let q = self.queues.resolve_mut(mutex)?;
1687                q.recursions = q.recursions.wrapping_add(1);
1688                Ok(Ready(()))
1689            }
1690        }
1691    }
1692
1693    /// `xSemaphoreGiveRecursive`.
1694    ///
1695    /// # Errors
1696    /// [`Error::NotActive`] when the caller does not hold the mutex.
1697    pub fn mutex_give_recursive(&mut self, mutex: QueueHandle) -> Result<()> {
1698        let caller = self.cur();
1699        // One resolve, as in the take above.
1700        let remaining = {
1701            let q = self.queues.resolve_mut(mutex)?;
1702            if q.holder != caller {
1703                return Err(Error::NotActive);
1704            }
1705            q.recursions = q.recursions.wrapping_sub(1);
1706            q.recursions
1707        };
1708        if remaining == 0 {
1709            let _ = self.semaphore_give(mutex)?;
1710        }
1711        Ok(())
1712    }
1713
1714    // -------------------------------------------------------- queue locks --
1715
1716    /// `prvLockQueue`: one critical section. The lock counters themselves
1717    /// only matter to an ISR, and the sim has none — but the section is
1718    /// where sim time passes, so it is here.
1719    /// `prvLockQueue`: stop interrupts taking tasks off this queue's event
1720    /// lists while a task walks them. An interrupt that finds the queue
1721    /// locked counts what it did instead, and [`Kernel::unlock_queue`] pays
1722    /// it back.
1723    // In line at both callers (`rusty-compiler-leverage` A3): win 23, which
1724    // took this and `is_queue_empty`/`lock_queue`'s siblings together for
1725    // -48 B of flash. A two-caller body this small pays two frames for a
1726    // resolve and a critical section; `#[inline]` alone was declined.
1727    #[inline(always)]
1728    fn lock_queue(&mut self, queue: QueueHandle) {
1729        self.enter_critical();
1730        if let Ok(q) = self.queues.resolve_mut(queue) {
1731            if q.rx_lock == UNLOCKED {
1732                q.rx_lock = LOCKED_UNMODIFIED;
1733            }
1734            if q.tx_lock == UNLOCKED {
1735                q.tx_lock = LOCKED_UNMODIFIED;
1736            }
1737        }
1738        self.exit_critical();
1739    }
1740
1741    /// `prvUnlockQueue`: two critical sections, one per lock counter. The
1742    /// loops inside them wake tasks that an ISR queued while the lock was
1743    /// held, and on the sim there are none.
1744    /// `prvUnlockQueue`: two critical sections, one per lock count, each
1745    /// waking one task per send or receive an interrupt made while the
1746    /// queue was locked. The yields are *missed* yields — pended, not
1747    /// taken — because a task is walking the list.
1748    fn unlock_queue(&mut self, queue: QueueHandle) -> Result<()> {
1749        self.enter_critical();
1750        {
1751            // One resolve for both fields. They were two, and a resolve
1752            // is a null test, a bounds check, a generation compare and an
1753            // `Option` unwrap -- all of it repeated to reach the same slot.
1754            // The unlock is written on the SAME lookup that read the lock,
1755            // whenever the drain below is not going to run -- which is every
1756            // call where nothing was sent while the queue was held, i.e. the
1757            // common one. It used to take a second `resolve_mut` after the
1758            // loop to store a value already known here.
1759            let (tx_lock, container) = match self.queues.resolve_mut(queue) {
1760                Ok(q) => {
1761                    // The container is read only when queue sets exist. With
1762                    // `USE_QUEUE_SETS` false nothing ever writes `set_container`,
1763                    // so the const makes it a known NULL and the whole branch
1764                    // below folds -- the other two queue-set sites already put
1765                    // the const FIRST and this one tested the field first.
1766                    let read = (
1767                        q.tx_lock,
1768                        if C::USE_QUEUE_SETS {
1769                            q.set_container
1770                        } else {
1771                            QueueHandle::NULL
1772                        },
1773                    );
1774                    if read.0 <= LOCKED_UNMODIFIED {
1775                        q.tx_lock = UNLOCKED;
1776                    }
1777                    read
1778                }
1779                Err(_) => (UNLOCKED, QueueHandle::NULL),
1780            };
1781            if tx_lock > LOCKED_UNMODIFIED {
1782                self.unlock_drain::<false>(queue, tx_lock, container)?;
1783            }
1784        }
1785        self.exit_critical();
1786        self.enter_critical();
1787        {
1788            // Same trade as the tx half above.
1789            let rx_lock = match self.queues.resolve_mut(queue) {
1790                Ok(q) => {
1791                    let read = q.rx_lock;
1792                    if read <= LOCKED_UNMODIFIED {
1793                        q.rx_lock = UNLOCKED;
1794                    }
1795                    read
1796                }
1797                Err(_) => UNLOCKED,
1798            };
1799            if rx_lock > LOCKED_UNMODIFIED {
1800                self.unlock_drain::<true>(queue, rx_lock, QueueHandle::NULL)?;
1801            }
1802        }
1803        self.exit_critical();
1804        Ok(())
1805    }
1806
1807    /// Either half of [`Kernel::unlock_queue`] when an interrupt used the
1808    /// queue while it was locked: wake one waiter per operation it made
1809    /// (`RX` false: receivers, for its sends; true: senders, for its
1810    /// receives), then unlock that half.
1811    ///
1812    /// Out of line because the loop is what makes `unlock_queue` CALL
1813    /// anything, and a body that calls saves six registers on entry. Nearly
1814    /// every unlock finds nothing to drain (BlockQ: all 21,490), and it was
1815    /// paying that frame for a loop it never ran.
1816    ///
1817    /// In line under `small`: out of line it is +232 B on the flash profile
1818    /// (17,776 -> 18,008), a speed trade that profile does not make. In line
1819    /// it is +8 B there (17,784), which is LLVM picking a different compare
1820    /// for the loop counter; three arrangements of the same loop read +8,
1821    /// +8 and +16, so the 8 is layout, not work.
1822    #[cfg_attr(not(feature = "small"), cold, inline(never))]
1823    #[cfg_attr(feature = "small", inline(always))]
1824    fn unlock_drain<const RX: bool>(
1825        &mut self,
1826        queue: QueueHandle,
1827        mut lock: i8,
1828        container: QueueHandle,
1829    ) -> Result<()> {
1830        while lock > LOCKED_UNMODIFIED {
1831            if !RX && C::USE_QUEUE_SETS && !container.is_null() {
1832                if C::USE_QUEUE_SETS && self.notify_queue_set_container(queue)? {
1833                    self.missed_yield();
1834                }
1835                // Wrapping: the loop runs only while this is above
1836                // `LOCKED_UNMODIFIED`, so one off it stays in range.
1837                lock = lock.wrapping_sub(1);
1838                continue;
1839            }
1840            let waiters = if RX {
1841                Self::queue_send_list(queue)
1842            } else {
1843                Self::queue_receive_list(queue)
1844            };
1845            if self.lists.is_empty_of(waiters) {
1846                break;
1847            }
1848            if self.remove_from_event_list(waiters)? {
1849                self.missed_yield();
1850            }
1851            // Wrapping: bounded by the loop, as above.
1852            lock = lock.wrapping_sub(1);
1853        }
1854        if let Ok(q) = self.queues.resolve_mut(queue) {
1855            if RX {
1856                q.rx_lock = UNLOCKED;
1857            } else {
1858                q.tx_lock = UNLOCKED;
1859            }
1860        }
1861        Ok(())
1862    }
1863
1864    /// `vQueueWaitForMessageRestricted`: what the timer service task does
1865    /// when it has no timer to run — block on the command queue without
1866    /// suspending the scheduler.
1867    ///
1868    /// `wait_indefinitely` is the C third argument: when set the wait
1869    /// becomes `portMAX_DELAY` and the task goes to the suspended list
1870    /// rather than a delayed one. The queue locks around it are C's
1871    /// `prvLockQueue` and `prvUnlockQueue` — three critical sections, and
1872    /// therefore three sixteenths of a tick's worth of sim time.
1873    ///
1874    /// # Errors
1875    /// A list error surfaces as itself.
1876    pub fn wait_for_message_restricted(
1877        &mut self,
1878        queue: QueueHandle,
1879        ticks: u64,
1880        wait_indefinitely: bool,
1881    ) -> Result<()> {
1882        self.lock_queue(queue);
1883        let empty = self
1884            .queues
1885            .resolve(queue)
1886            .map(|q| q.waiting == 0)
1887            .unwrap_or(false);
1888        if empty {
1889            // vTaskPlaceOnEventListRestricted
1890            let current = self.cur();
1891            let event = Self::event_item(current);
1892            self.lists
1893                .insert_end(Self::queue_receive_list(queue), event)?;
1894            let ticks = if wait_indefinitely {
1895                Self::MAX_DELAY
1896            } else {
1897                ticks
1898            };
1899            let wake_at = self.tick.wrapping_add(ticks) & Self::MAX_DELAY;
1900            self.trace_task(current, |task, name| Event::TaskDelayUntil {
1901                task,
1902                name,
1903                wake_at,
1904            });
1905            self.add_current_task_to_delayed_list(ticks, wait_indefinitely)?;
1906        }
1907        self.unlock_queue(queue)?;
1908        Ok(())
1909    }
1910
1911    /// `prvIsQueueEmpty`, critical section and all — it is one of the
1912    /// four the blocking path spends, and therefore a sixteenth of a tick
1913    /// every four calls.
1914    // In line at both callers (`rusty-compiler-leverage` A3): win 23, which
1915    // took this and `is_queue_empty`/`lock_queue`'s siblings together for
1916    // -48 B of flash. A two-caller body this small pays two frames for a
1917    // resolve and a critical section; `#[inline]` alone was declined.
1918    #[inline(always)]
1919    fn is_queue_empty(&mut self, queue: QueueHandle) -> bool {
1920        self.enter_critical();
1921        let empty = self
1922            .queues
1923            .resolve(queue)
1924            .map(|q| q.waiting == 0)
1925            .unwrap_or(true);
1926        self.exit_critical();
1927        empty
1928    }
1929
1930    /// `prvIsQueueFull`, likewise.
1931    fn is_queue_full(&mut self, queue: QueueHandle) -> bool {
1932        self.enter_critical();
1933        let full = self
1934            .queues
1935            .resolve(queue)
1936            .map(|q| q.waiting >= q.length)
1937            .unwrap_or(false);
1938        self.exit_critical();
1939        full
1940    }
1941
1942    /// `prvGetHighestPriorityOfWaitToReceiveList`.
1943    fn highest_waiting_priority(&self, queue: QueueHandle) -> Result<u8> {
1944        let list = Self::queue_receive_list(queue);
1945        match self.lists.head(list)? {
1946            // The event item's value is `configMAX_PRIORITIES - priority`.
1947            Some(item) => {
1948                let value = self.lists.value(item)?;
1949                let max = u64::from(C::MAX_PRIORITIES);
1950                Ok(u8::try_from(max.saturating_sub(value)).unwrap_or(0))
1951            }
1952            None => Ok(0),
1953        }
1954    }
1955}
1956
1957/// The next index in a ring of `length`.
1958///
1959/// Five instructions behind a call at six sites. `#[inline]` lets the ring
1960/// bound fold into the caller, which already knows it.
1961#[inline]
1962const fn wrap_next(index: usize, length: usize) -> usize {
1963    // `wrapping_add`, and it is the SAME function, not an approximation
1964    // of it: at `usize::MAX` saturating yields `usize::MAX`, which is
1965    // `>= length`, so the caller gets 0 -- and wrapping yields 0, which is
1966    // `< length`, so the caller gets 0 as well. Every other index agrees
1967    // without argument.
1968    let next = index.wrapping_add(1);
1969    if next >= length { 0 } else { next }
1970}
1971
1972/// The previous index in a ring of `length`.
1973#[inline]
1974const fn wrap_prev(index: usize, length: usize) -> usize {
1975    match index.checked_sub(1) {
1976        Some(prev) => prev,
1977        None => length.saturating_sub(1),
1978    }
1979}
1980
1981/// The Rust face's raw surface: see [`crate::typed`].
1982///
1983/// Every method here is one of the kernel's own calls under another name.
1984/// The one that is not — [`Raw::raw_queue_has_room`] — is the kernel
1985/// reading its own queue rather than a task asking, so it takes no
1986/// critical section and costs no sim time, which is what lets the typed
1987/// face be free.
1988impl<
1989    C: Config,
1990    P: Port,
1991    T: Trace,
1992    H,
1993    const TASKS: usize,
1994    const ITEMS: usize,
1995    const LISTS: usize,
1996    const QUEUES: usize,
1997    const SLOTS: usize,
1998    const BUFFERS: usize,
1999    const BYTES: usize,
2000    const TIMERS: usize,
2001    const GROUPS: usize,
2002    const TIMER_CMDS: usize,
2003> crate::typed::Raw
2004    for Kernel<
2005        C,
2006        P,
2007        T,
2008        H,
2009        TASKS,
2010        ITEMS,
2011        LISTS,
2012        QUEUES,
2013        SLOTS,
2014        BUFFERS,
2015        BYTES,
2016        TIMERS,
2017        GROUPS,
2018        TIMER_CMDS,
2019    >
2020where
2021    H: TickHook<Self>,
2022{
2023    fn raw_queue_create(&mut self, length: usize) -> Result<QueueHandle> {
2024        self.queue_create(length)
2025    }
2026
2027    fn raw_queue_send(&mut self, queue: QueueHandle, value: u64, ticks: u64) -> Result<Wait<()>> {
2028        self.queue_send(queue, value, ticks)
2029    }
2030
2031    fn raw_queue_receive(&mut self, queue: QueueHandle, ticks: u64) -> Result<Wait<u64>> {
2032        self.queue_receive(queue, ticks)
2033    }
2034
2035    fn raw_queue_messages_waiting(&mut self, queue: QueueHandle) -> Result<usize> {
2036        self.queue_messages_waiting(queue)
2037    }
2038
2039    fn raw_queue_has_room(&self, queue: QueueHandle) -> bool {
2040        self.queues
2041            .resolve(queue)
2042            .is_ok_and(|q| q.waiting < q.length)
2043    }
2044
2045    fn raw_in_isr(&self) -> bool {
2046        self.port.in_isr()
2047    }
2048
2049    fn raw_queue_send_from_isr(&mut self, queue: QueueHandle, value: u64) -> Result<Woken> {
2050        self.queue_send_from_isr(queue, value)
2051    }
2052
2053    fn raw_mutex_create(&mut self) -> Result<QueueHandle> {
2054        self.mutex_create()
2055    }
2056
2057    fn raw_mutex_take(&mut self, mutex: QueueHandle, ticks: u64) -> Result<Wait<()>> {
2058        self.semaphore_take(mutex, ticks)
2059    }
2060
2061    fn raw_mutex_give(&mut self, mutex: QueueHandle) -> Result<()> {
2062        self.semaphore_give(mutex).map(|_| ())
2063    }
2064}
2065
2066/// Plan P5: what the mutation survey found no oracle judging in this file --
2067/// the slot arena (Kairos storage; the C mallocs), the queue LOCK (which
2068/// only an interrupt landing while a task holds the queue can reach, and
2069/// the oracles deliver interrupts between calls), and a receive or peek
2070/// made on a semaphore through the queue API.
2071#[cfg(test)]
2072#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2073mod tests {
2074    use rusty_rtos_core::config::Config;
2075    use rusty_rtos_core::error::Error;
2076    use rusty_rtos_core::handle::{QueueHandle, TaskHandle};
2077    use rusty_rtos_core::hooks::NoTickHook;
2078
2079    use super::{UNLOCKED, Wait};
2080    use crate::TaskState;
2081    use crate::system::tests::{NoTrace, TestConfig, TestPort};
2082
2083    /// Room for the timer daemon's queue, a set and three queues, over
2084    /// thirty-two slots.
2085    type K = crate::Kernel<
2086        TestConfig,
2087        TestPort,
2088        NoTrace,
2089        NoTickHook,
2090        4,
2091        { crate::list_slots_for(4, 0, crate::lists_for(TestConfig::MAX_PRIORITIES, 5, 0)) },
2092        { crate::lists_for(TestConfig::MAX_PRIORITIES, 5, 0) },
2093        5,
2094        32,
2095        0,
2096        0,
2097        0,
2098        0,
2099        { <TestConfig as ::rusty_rtos_core::config::Config>::TIMER_QUEUE_LENGTH },
2100    >;
2101
2102    /// Eight slots and nothing else, to fill.
2103    type K8 = crate::Kernel<
2104        TestConfig,
2105        TestPort,
2106        NoTrace,
2107        NoTickHook,
2108        4,
2109        { crate::list_slots_for(4, 0, crate::lists_for(TestConfig::MAX_PRIORITIES, 3, 0)) },
2110        { crate::lists_for(TestConfig::MAX_PRIORITIES, 3, 0) },
2111        3,
2112        8,
2113        0,
2114        0,
2115        0,
2116        0,
2117        { <TestConfig as ::rusty_rtos_core::config::Config>::TIMER_QUEUE_LENGTH },
2118    >;
2119
2120    /// A started kernel in which `rx` (priority 2) is blocked receiving on
2121    /// an empty queue of sixteen, and `tx` (priority 1) runs.
2122    fn with_receiver() -> (K, QueueHandle, TaskHandle) {
2123        let mut k = K::new(TestPort::default(), NoTrace).expect("the declared geometry adds up");
2124        let q = k.queue_create(16).expect("a queue");
2125        let rx = k.create_task("rx", 2).expect("rx");
2126        k.create_task("tx", 1).expect("tx");
2127        let h = k.start_scheduler().expect("start");
2128        k.suspend(Some(h.timer)).expect("park the daemon");
2129        assert_eq!(k.current(), rx);
2130        assert_eq!(k.queue_receive(q, 10), Ok(Wait::Blocked));
2131        (k, q, rx)
2132    }
2133
2134    fn tx_lock(k: &K, q: QueueHandle) -> i8 {
2135        k.queues.resolve(q).expect("the queue").tx_lock
2136    }
2137
2138    /// `prvLockQueue` holds an interrupt's wake back until `prvUnlockQueue`:
2139    /// the interrupt only counts what it did. Two sends, because the first
2140    /// alone cannot tell the count from the `queueUNLOCKED` sentinel.
2141    #[test]
2142    fn an_interrupt_send_to_a_locked_queue_waits_for_the_unlock() {
2143        let (mut k, q, rx) = with_receiver();
2144        k.lock_queue(q);
2145        let _ = k.queue_send_from_isr(q, 7).expect("send");
2146        let _ = k.queue_send_from_isr(q, 8).expect("send");
2147        assert_eq!(
2148            k.task_state_get(rx),
2149            Ok(TaskState::Blocked),
2150            "the lock held the wake back"
2151        );
2152        assert_eq!(tx_lock(&k, q), 2, "and counted both sends");
2153        k.unlock_queue(q).expect("unlock");
2154        assert_eq!(
2155            k.task_state_get(rx),
2156            Ok(TaskState::Ready),
2157            "the unlock woke it"
2158        );
2159        assert_eq!(tx_lock(&k, q), UNLOCKED);
2160    }
2161
2162    /// `prvUnlockQueue` wakes ONE waiter per send the lock held back, not
2163    /// every waiter: one interrupt send, two receivers blocked, one woken.
2164    ///
2165    /// The drain's `while lock > LOCKED_UNMODIFIED` as `>=` survived every
2166    /// oracle once the drain moved out of line (kernel 1ce2ac3): the guard
2167    /// before it means the loop is entered only above zero, so the mutant
2168    /// differs only at the tail -- one wake too many when a waiter is left.
2169    /// No test held back fewer sends than there were waiters.
2170    #[test]
2171    fn an_unlock_wakes_one_waiter_per_held_back_send() {
2172        let mut k = K::new(TestPort::default(), NoTrace).expect("the declared geometry adds up");
2173        let q = k.queue_create(16).expect("a queue");
2174        let first = k.create_task("rx1", 3).expect("rx1");
2175        let second = k.create_task("rx2", 2).expect("rx2");
2176        let h = k.start_scheduler().expect("start");
2177        k.suspend(Some(h.timer)).expect("park the daemon");
2178        assert_eq!(k.current(), first);
2179        assert_eq!(k.queue_receive(q, 10), Ok(Wait::Blocked));
2180        k.switch_context();
2181        assert_eq!(k.current(), second);
2182        assert_eq!(k.queue_receive(q, 10), Ok(Wait::Blocked));
2183        k.switch_context();
2184        k.lock_queue(q);
2185        let _ = k.queue_send_from_isr(q, 7).expect("send");
2186        assert_eq!(tx_lock(&k, q), 1, "one send held back");
2187        k.unlock_queue(q).expect("unlock");
2188        assert_eq!(
2189            k.task_state_get(first),
2190            Ok(TaskState::Ready),
2191            "the first waiter woke"
2192        );
2193        assert_eq!(
2194            k.task_state_get(second),
2195            Ok(TaskState::Blocked),
2196            "and only the first: one send, one wake"
2197        );
2198        assert_eq!(tx_lock(&k, q), UNLOCKED);
2199    }
2200
2201    /// `prvIncrementQueueTxLock` / `RxLock` stop at the task count: there
2202    /// is no point waking more tasks than exist.
2203    #[test]
2204    fn the_lock_counts_stop_at_the_task_count() {
2205        let (mut k, q, _) = with_receiver();
2206        let tasks = k.task_count();
2207        k.lock_queue(q);
2208        for i in 0..tasks.saturating_add(3) {
2209            let _ = k.queue_send_from_isr(q, i as u64).expect("send");
2210        }
2211        assert_eq!(tx_lock(&k, q) as usize, tasks, "the send count is capped");
2212        k.unlock_queue(q).expect("unlock");
2213
2214        k.lock_queue(q);
2215        for _ in 0..tasks.saturating_add(3) {
2216            let _ = k.queue_receive_from_isr(q).expect("receive");
2217        }
2218        let rx_lock = k.queues.resolve(q).expect("the queue").rx_lock;
2219        assert_eq!(rx_lock as usize, tasks, "and so is the receive count");
2220        k.unlock_queue(q).expect("unlock");
2221    }
2222
2223    /// A send an interrupt made while the queue was locked is announced to
2224    /// its set by the unlock, not lost (`prvUnlockQueue`'s
2225    /// `prvNotifyQueueSetContainer`).
2226    #[test]
2227    fn an_unlock_announces_to_the_set_what_the_lock_held_back() {
2228        let mut k = K::new(TestPort::default(), NoTrace).expect("the declared geometry adds up");
2229        let set = k.queue_create_set(4).expect("a set");
2230        let q = k.queue_create(4).expect("a member");
2231        assert_eq!(k.queue_add_to_set(q, set), Ok(true));
2232        k.create_task("t", 1).expect("t");
2233        let h = k.start_scheduler().expect("start");
2234        k.suspend(Some(h.timer)).expect("park the daemon");
2235
2236        k.lock_queue(q);
2237        let _ = k.queue_send_from_isr(q, 5).expect("send");
2238        assert_eq!(
2239            k.queue_select_from_set(set, 0),
2240            Ok(Wait::Ready(None)),
2241            "nothing announced while the lock holds"
2242        );
2243        k.unlock_queue(q).expect("unlock");
2244        assert_eq!(
2245            k.queue_select_from_set(set, 0),
2246            Ok(Wait::Ready(Some(q))),
2247            "the unlock announced the member"
2248        );
2249    }
2250
2251    /// The slot arena refits a freed queue's slots exactly, so a
2252    /// create/delete loop of one length never runs out.
2253    #[test]
2254    fn the_slot_arena_refits_a_hole_of_the_same_length() {
2255        let mut k = K8::new(TestPort::default(), NoTrace).expect("the declared geometry adds up");
2256        let a = k.queue_create(4).expect("a");
2257        let _b = k.queue_create(4).expect("b fills the slots");
2258        assert_eq!(k.queue_create(1), Err(Error::Full));
2259        k.queue_delete(a).expect("delete");
2260        k.queue_create(4).expect("the hole refits exactly");
2261        assert_eq!(k.queue_create(1), Err(Error::Full));
2262    }
2263
2264    /// `xQueueReceive` on a semaphore takes it and `xQueuePeek` does not:
2265    /// a semaphore carries no data, so the count is the item.
2266    #[test]
2267    fn receive_takes_a_semaphore_and_peek_leaves_it() {
2268        let (mut k, _, _) = with_receiver();
2269        let s = k.semaphore_create_binary().expect("a semaphore");
2270        k.semaphore_give(s).expect("give");
2271        assert_eq!(k.queue_peek(s, 0), Ok(Wait::Ready(0)));
2272        assert_eq!(k.semaphore_count(s), Ok(1), "a peek leaves it");
2273        assert_eq!(k.queue_receive(s, 0), Ok(Wait::Ready(0)));
2274        assert_eq!(k.semaphore_count(s), Ok(0), "a receive takes it");
2275
2276        // And from an interrupt -- `xSemaphoreTakeFromISR` is exactly
2277        // `xQueueReceiveFromISR` on a semaphore.
2278        k.semaphore_give(s).expect("give");
2279        assert_eq!(k.queue_peek_from_isr(s), Ok(0));
2280        assert_eq!(k.semaphore_count(s), Ok(1), "an ISR peek leaves it");
2281        let _ = k.queue_receive_from_isr(s).expect("receive");
2282        assert_eq!(k.semaphore_count(s), Ok(0), "an ISR receive takes it");
2283    }
2284
2285    /// `xTaskResumeAll` yields for a task the pending-ready list hands back
2286    /// only when it outranks the running one: an interrupt that wakes a
2287    /// LOWER-priority waiter under suspension costs no yield.
2288    #[test]
2289    fn resuming_with_a_lower_priority_task_pending_does_not_yield() {
2290        let mut k = K::new(TestPort::default(), NoTrace).expect("the declared geometry adds up");
2291        let sem = k.semaphore_create_binary().expect("a semaphore");
2292        let lo = k.create_task("lo", 1).expect("lo");
2293        let hi = k.create_task("hi", 2).expect("hi");
2294        let h = k.start_scheduler().expect("start");
2295        k.suspend(Some(h.timer)).expect("park the daemon");
2296        assert_eq!(k.current(), hi);
2297        k.delay(2).expect("hi steps aside");
2298        assert_eq!(k.current(), lo);
2299        assert_eq!(k.semaphore_take(sem, 100), Ok(Wait::Blocked));
2300        k.tick_from_isr();
2301        k.tick_from_isr();
2302        assert_eq!(k.current(), hi, "hi is back");
2303
2304        k.suspend_all();
2305        let _ = k.semaphore_give_from_isr(sem).expect("give");
2306        assert!(!k.resume_all(), "lo does not outrank hi: no yield");
2307        assert_eq!(k.current(), hi);
2308    }
2309}