Skip to main content

rusty_rtos_port_riscv/
lib.rs

1#![no_std]
2//! `rusty_rtos_port-riscv` — the Kairos RISC-V port.
3//!
4//! RV32, for QEMU `virt` and for the ESP32-C6/C61/H2/P4 family.
5//!
6//! # Simpler than Xtensa, and not for the reason you would guess
7//!
8//! Xtensa's difficulty is the register windows: a task several calls deep
9//! has live frames in a 64-entry file that must reach its own stack before
10//! anything else runs. RISC-V has no windows, so there is nothing to spill.
11//!
12//! But it is NOT simply "Cortex-M with different names", and the difference
13//! is worth stating because it decides the design:
14//!
15//! | | who saves the callee-saved registers |
16//! |---|---|
17//! | Cortex-M | the hardware stacks `r0-r3, r12, lr, pc, xPSR`; the port adds `r4-r11` |
18//! | Xtensa | the window spill, driven by `xtensa-lx-rt`'s exception entry |
19//! | **RISC-V** | **nobody — the port must do all of it** |
20//!
21//! A RISC-V trap saves *nothing* to the stack. `riscv-rt`'s entry saves the
22//! caller-saved registers into a `TrapFrame` so a normal Rust handler can
23//! run, and the callee-saved ones survive only because that handler is an
24//! ordinary function that preserves them. For a context switch they belong
25//! to the outgoing task and have to be written down, so this port keeps its
26//! own frame: `ra`, `sp` and `s0`–`s11`.
27//!
28//! # The switch still happens in the trap
29//!
30//! The lesson from the Xtensa port holds: the decision and the register
31//! swap are one step, taken where the context is already captured. A yield
32//! raises the machine software interrupt through the CLINT; the handler
33//! swaps and returns. [`Port::COMMITS_SWITCH`] is `true` for exactly that
34//! reason — the kernel must not move `current` at the point of the yield,
35//! because the registers do not move until the trap is taken.
36//!
37//! # Unsafe
38//!
39//! Every use is fenced with `#[expect(unsafe_code)]` and a justification.
40//! See `UNSAFE.md`.
41
42use core::sync::atomic::{AtomicU32, Ordering};
43
44use rusty_rtos_core::isr::Woken;
45use rusty_rtos_core::port::Port;
46
47/// Kani proofs of the stack builder (hardening gate H-30).
48#[cfg(kani)]
49mod proofs;
50
51/// Crate version, for manifests and logs.
52pub const VERSION: &str = env!("CARGO_PKG_VERSION");
53
54/// The CLINT's `msip` register for hart 0 on QEMU `virt`.
55///
56/// Writing 1 raises the machine software interrupt; writing 0 clears it.
57/// The ESP32-C6 family reaches the same interrupt through its own
58/// peripheral, which is why this is a constant a board can override rather
59/// than something baked into the switch.
60pub const CLINT_MSIP: usize = 0x0200_0000;
61
62/// How many registers the port itself saves: `ra`, `sp`, `s0`–`s11`.
63pub const SAVED_REGISTERS: usize = 14;
64
65/// The saved machine state of one task.
66///
67/// Only the registers a trap does NOT already preserve. `riscv-rt` captures
68/// the caller-saved set into its own `TrapFrame` on the way in and restores
69/// it on the way out, so duplicating it here would be two places that must
70/// agree about the same bytes.
71#[repr(C)]
72#[derive(Debug, Clone, Copy, Default)]
73pub struct Context {
74    /// Return address — where the task resumes.
75    pub ra: usize,
76    /// Stack pointer.
77    pub sp: usize,
78    /// Callee-saved `s0`–`s11`.
79    pub s: [usize; 12],
80    /// `mepc` — where the task resumes when the trap exits.
81    ///
82    /// Only the PREEMPTIVE path uses this. A cooperative switch happens at a
83    /// call site and resumes through `ra`; a preemptive one resumes through
84    /// `mret`, and `mret` jumps to `mepc`.
85    ///
86    /// Saving it is **not optional, and that is measured rather than
87    /// asserted**: the `mepc` in the CSR at the moment of a switch belongs to
88    /// the task being switched OUT, so a port that leaves it alone resumes
89    /// each task at the other one's address. Deleting the restore hangs
90    /// `riscv32-qemu-preempt` on the spot.
91    pub mepc: usize,
92    /// `mstatus` — in particular `MPIE`, which `mret` copies into `MIE`.
93    ///
94    /// Saved and restored because a task's interrupt state is its own: a task
95    /// preempted inside a critical section must come back with that section
96    /// still in force.
97    ///
98    /// **It is NOT what makes preemption work, and an earlier version of this
99    /// comment said it was.** Removing the restore and re-running
100    /// `riscv32-qemu-preempt` still passes, because the trap entry has already
101    /// copied the live `MIE` into `MPIE`, so `mret` re-enables interrupts
102    /// without our help. The two lines that ARE load bearing are [`mepc`]
103    /// below and the `mret` in `kairos_riscv_trampoline_trap`; removing
104    /// either hangs the cell. Poison-proved, all three.
105    ///
106    /// [`mepc`]: Context::mepc
107    pub mstatus: usize,
108}
109
110impl Context {
111    /// An empty context, for a slot the first switch will fill in.
112    #[must_use]
113    pub const fn new() -> Self {
114        Self {
115            ra: 0,
116            sp: 0,
117            s: [0; 12],
118            mepc: 0,
119            mstatus: 0,
120        }
121    }
122}
123
124/// Build the saved context of a task that has never run.
125///
126/// `stack_top` is one past the highest usable byte; it is rounded down to
127/// the 16-byte alignment the ABI requires of `sp`. When the task is first
128/// switched to, `wrapper` runs with `task_fn` and `param` as its arguments.
129///
130/// The wrapper exists because a task body must not simply `return`: there is
131/// nothing beneath it to return into.
132///
133/// # Safety
134///
135/// `stack_top` must be one past a writable stack that outlives the task.
136#[expect(unsafe_code, reason = "writing the initial frame")]
137#[must_use]
138pub unsafe fn new_task_context(
139    wrapper: extern "C" fn(task_fn: usize, param: usize) -> !,
140    task_fn: usize,
141    param: usize,
142    stack_top: *mut u8,
143) -> Context {
144    // Pointer-width arithmetic, narrowed nowhere: the Xtensa port shipped a
145    // version that did this in `u32` and wrote through a truncated address
146    // the first time a host test called it.
147    let top = (stack_top as usize) & !0xf;
148    let mut context = Context::new();
149    // A fresh task resumes through a TRAMPOLINE, not straight into the
150    // wrapper, and the reason is the calling convention.
151    //
152    // The switch restores only the callee-saved set — `ra`, `sp`, `s0`-`s11`
153    // — because those are the registers a trap does not preserve. A
154    // `extern "C"` function reads its arguments from `a0`/`a1`, which are
155    // caller-saved and therefore NOT in that set. Putting the arguments in
156    // `s0`/`s1` and jumping straight to the wrapper hands it whatever
157    // `a0`/`a1` happened to hold: the first version of this port did that
158    // and hung on the first switch.
159    //
160    // So the entry point and its two arguments travel in `s0`-`s2`, and
161    // three `mv` instructions put them where the ABI expects.
162    context.ra = (kairos_riscv_trampoline as *const ()) as usize;
163    context.sp = top;
164    context.s[0] = wrapper as *const () as usize;
165    context.s[1] = task_fn;
166    context.s[2] = param;
167    context
168}
169
170#[cfg(target_arch = "riscv32")]
171#[expect(unsafe_code, reason = "the trampoline is defined in this crate's asm")]
172unsafe extern "C" {
173    /// Defined in this crate's assembly; see [`new_task_context`].
174    fn kairos_riscv_trampoline();
175    /// Defined in this crate's assembly; see [`new_task_context_preemptive`].
176    fn kairos_riscv_trampoline_trap();
177}
178
179/// Off RISC-V the trampoline is a plain address for the frame arithmetic.
180#[cfg(not(target_arch = "riscv32"))]
181extern "C" fn kairos_riscv_trampoline() {}
182
183/// Off RISC-V, as above.
184#[cfg(not(target_arch = "riscv32"))]
185extern "C" fn kairos_riscv_trampoline_trap() {}
186
187/// Swap the running task.
188///
189/// Called from inside the switching trap. `current` is where the outgoing
190/// task's registers are written — `None` when it is being discarded — and
191/// `next` is the task to resume.
192///
193/// # Safety
194///
195/// Both pointers must be valid, and `next` must hold a context built by
196/// [`new_task_context`] or written by a previous call to this.
197#[cfg(target_arch = "riscv32")]
198#[expect(unsafe_code, reason = "the context switch itself")]
199#[inline(always)]
200#[allow(
201    clippy::unwrap_or_default,
202    reason = "`Default` for raw pointers is Rust 1.88 and the MSRV is 1.85; `allow`, not `expect`, because only clippys that know the impl fire it"
203)]
204pub unsafe fn switch_context(current: Option<*mut Context>, next: *const Context) {
205    // SAFETY: the contract above. `kairos_riscv_switch` writes the outgoing
206    // registers through `a0` and loads the incoming ones from `a1`; it never
207    // touches memory outside those two structs.
208    unsafe {
209        core::arch::asm!(
210            "call kairos_riscv_switch",
211            in("a0") current.unwrap_or(core::ptr::null_mut()),
212            in("a1") next,
213            clobber_abi("C"),
214        );
215    }
216}
217
218/// Off RISC-V this does nothing, so callers keep type-checking on the host.
219///
220/// # Safety
221/// As the RISC-V version; this one is a no-op.
222#[cfg(not(target_arch = "riscv32"))]
223#[expect(unsafe_code, reason = "signature parity with the RISC-V version")]
224#[inline(always)]
225pub unsafe fn switch_context(_current: Option<*mut Context>, _next: *const Context) {}
226
227/// Swap the running task from inside a trap, so the incoming task resumes
228/// through `mret`.
229///
230/// # Why this is a second function and not a flag on the first
231///
232/// [`switch_context`] resumes a task by RETURNING to its `ra`. That is
233/// exactly right at a call site, and it is what makes a cooperative yield 30
234/// instructions against FreeRTOS's 83.
235///
236/// It is wrong inside a trap. A trap is left with `mret`, which restores
237/// `MIE` from `MPIE` and jumps to `mepc`; a task entered with `ret` instead
238/// keeps running in trap context with interrupts masked, and the hart takes
239/// no further interrupts — the first preemptive switch becomes the last.
240/// That was a real defect, witnessed by
241/// `rusty_rtos_port/firmware/riscv32-qemu-preempt`.
242///
243/// So this one also swaps `mepc` and `mstatus`, and a fresh task leaves its
244/// first trap through `mret` rather than `ret`. Keeping the two switches
245/// apart means a yield still pays only for what a yield needs.
246///
247/// Of those three changes, poison-proving says **`mepc` and the `mret` are
248/// what make it work** — remove either and the cell hangs — while `mstatus`
249/// is correctness for the general case rather than the thing that was
250/// broken. See [`Context::mstatus`].
251///
252/// # Safety
253///
254/// Must be called from inside a trap handler. Both pointers must be valid,
255/// and `next` must hold a context built by [`new_task_context_preemptive`]
256/// or written by a previous call to this.
257#[cfg(target_arch = "riscv32")]
258#[expect(unsafe_code, reason = "the context switch itself")]
259#[inline(always)]
260pub unsafe fn switch_context_trap(current: *mut Context, next: *const Context) {
261    // SAFETY: the contract above.
262    unsafe {
263        core::arch::asm!(
264            "call kairos_riscv_switch_trap",
265            in("a0") current,
266            in("a1") next,
267            clobber_abi("C"),
268        );
269    }
270}
271
272/// Off RISC-V this does nothing, so callers keep type-checking on the host.
273///
274/// # Safety
275/// As the RISC-V version; this one is a no-op.
276#[cfg(not(target_arch = "riscv32"))]
277#[expect(unsafe_code, reason = "signature parity with the RISC-V version")]
278#[inline(always)]
279pub unsafe fn switch_context_trap(_current: *mut Context, _next: *const Context) {}
280
281/// `mstatus` for a task that has never run: `MPIE` set so `mret` enables
282/// interrupts, and `MPP` = M-mode so it returns to the privilege it left.
283///
284/// FreeRTOS computes the same value (`0x188 << 4`) in
285/// `pxPortInitialiseStack`, ORing it into the live `mstatus` to preserve the
286/// FPU and VPU dirty bits. This port has no FPU save, so there is nothing to
287/// preserve and the constant stands alone.
288pub const FRESH_MSTATUS: usize = 0x1880;
289
290/// Build the saved context of a task that has never run, for the PREEMPTIVE
291/// path.
292///
293/// The difference from [`new_task_context`] is where the task resumes from.
294/// There, the first switch `ret`s into a trampoline and falls straight into
295/// the wrapper. Here the first switch happens inside a trap, so the task must
296/// leave through `mret` like any preempted task would — otherwise it runs
297/// with interrupts masked for ever.
298///
299/// So `mepc` is the wrapper and `mstatus` carries `MPIE`, and the trampoline
300/// this context returns to does nothing but move the two arguments into
301/// place and `mret`.
302///
303/// # Safety
304///
305/// `stack_top` must be one past a writable stack that outlives the task.
306#[expect(unsafe_code, reason = "writing the initial frame")]
307#[must_use]
308pub unsafe fn new_task_context_preemptive(
309    wrapper: extern "C" fn(task_fn: usize, param: usize) -> !,
310    task_fn: usize,
311    param: usize,
312    stack_top: *mut u8,
313) -> Context {
314    let mut context = Context::new();
315    context.ra = (kairos_riscv_trampoline_trap as *const ()) as usize;
316    context.sp = (stack_top as usize) & !0xf;
317    // The arguments travel in `s1`/`s2` for the same reason as the
318    // cooperative path: a switch restores only the callee-saved set, and a
319    // `extern "C"` function reads its arguments from the caller-saved one.
320    context.s[1] = task_fn;
321    context.s[2] = param;
322    context.mepc = (wrapper as *const ()) as usize;
323    context.mstatus = FRESH_MSTATUS;
324    context
325}
326
327/// The switch itself.
328#[cfg(target_arch = "riscv32")]
329#[expect(unsafe_code, reason = "the context switch itself")]
330mod asm {
331    core::arch::global_asm!(
332        ".section .text.kairos_riscv_switch,\"ax\"",
333        ".align 2",
334        ".global kairos_riscv_switch",
335        ".type kairos_riscv_switch, @function",
336        "kairos_riscv_switch:",
337        // a0 = *mut Context for the outgoing task, or null.
338        // a1 = *const Context for the incoming task.
339        //
340        // Fourteen words, in the order `Context` declares them: ra, sp,
341        // then s0-s11. `#[repr(C)]` is what makes that order a contract
342        // rather than a coincidence.
343        "  beqz  a0, 2f",
344        "  sw    ra, 0(a0)",
345        "  sw    sp, 4(a0)",
346        "  sw    s0, 8(a0)",
347        "  sw    s1, 12(a0)",
348        "  sw    s2, 16(a0)",
349        "  sw    s3, 20(a0)",
350        "  sw    s4, 24(a0)",
351        "  sw    s5, 28(a0)",
352        "  sw    s6, 32(a0)",
353        "  sw    s7, 36(a0)",
354        "  sw    s8, 40(a0)",
355        "  sw    s9, 44(a0)",
356        "  sw    s10, 48(a0)",
357        "  sw    s11, 52(a0)",
358        "2:",
359        "  lw    ra, 0(a1)",
360        "  lw    sp, 4(a1)",
361        "  lw    s0, 8(a1)",
362        "  lw    s1, 12(a1)",
363        "  lw    s2, 16(a1)",
364        "  lw    s3, 20(a1)",
365        "  lw    s4, 24(a1)",
366        "  lw    s5, 28(a1)",
367        "  lw    s6, 32(a1)",
368        "  lw    s7, 36(a1)",
369        "  lw    s8, 40(a1)",
370        "  lw    s9, 44(a1)",
371        "  lw    s10, 48(a1)",
372        "  lw    s11, 52(a1)",
373        "  ret",
374        ".size kairos_riscv_switch, . - kairos_riscv_switch",
375        // A fresh task lands here rather than in its entry point, because
376        // the switch restores callee-saved registers and the C ABI passes
377        // arguments in caller-saved ones. See `new_task_context`.
378        ".section .text.kairos_riscv_trampoline,\"ax\"",
379        ".align 2",
380        ".global kairos_riscv_trampoline",
381        ".type kairos_riscv_trampoline, @function",
382        "kairos_riscv_trampoline:",
383        "  mv    a0, s1", // task_fn
384        "  mv    a1, s2", // param
385        "  jr    s0",     // the wrapper, which never returns
386        ".size kairos_riscv_trampoline, . - kairos_riscv_trampoline",
387        // ------------------------------------------------------------------
388        // The PREEMPTIVE switch. Same fourteen registers, plus the two CSRs
389        // that decide how the trap is left: `mepc` (where) and `mstatus`
390        // (with what interrupt state). Without those two a task entered from
391        // a trap runs with interrupts masked for ever -- which is exactly
392        // the defect `riscv32-qemu-preempt` witnessed.
393        //
394        // `t0` is scratch. It is caller-saved, so `riscv-rt`'s trap epilogue
395        // reloads it from the INCOMING task's frame on the way out; clobbering
396        // it after `sp` has moved is therefore safe.
397        ".section .text.kairos_riscv_switch_trap,\"ax\"",
398        ".align 2",
399        ".global kairos_riscv_switch_trap",
400        ".type kairos_riscv_switch_trap, @function",
401        "kairos_riscv_switch_trap:",
402        "  sw    ra, 0(a0)",
403        "  sw    sp, 4(a0)",
404        "  sw    s0, 8(a0)",
405        "  sw    s1, 12(a0)",
406        "  sw    s2, 16(a0)",
407        "  sw    s3, 20(a0)",
408        "  sw    s4, 24(a0)",
409        "  sw    s5, 28(a0)",
410        "  sw    s6, 32(a0)",
411        "  sw    s7, 36(a0)",
412        "  sw    s8, 40(a0)",
413        "  sw    s9, 44(a0)",
414        "  sw    s10, 48(a0)",
415        "  sw    s11, 52(a0)",
416        "  csrr  t0, mepc",
417        "  sw    t0, 56(a0)",
418        "  csrr  t0, mstatus",
419        "  sw    t0, 60(a0)",
420        "  lw    ra, 0(a1)",
421        "  lw    sp, 4(a1)",
422        "  lw    s0, 8(a1)",
423        "  lw    s1, 12(a1)",
424        "  lw    s2, 16(a1)",
425        "  lw    s3, 20(a1)",
426        "  lw    s4, 24(a1)",
427        "  lw    s5, 28(a1)",
428        "  lw    s6, 32(a1)",
429        "  lw    s7, 36(a1)",
430        "  lw    s8, 40(a1)",
431        "  lw    s9, 44(a1)",
432        "  lw    s10, 48(a1)",
433        "  lw    s11, 52(a1)",
434        "  lw    t0, 56(a1)",
435        "  csrw  mepc, t0",
436        "  lw    t0, 60(a1)",
437        "  csrw  mstatus, t0",
438        "  ret",
439        ".size kairos_riscv_switch_trap, . - kairos_riscv_switch_trap",
440        // A task that has never run cannot `ret` into `_start_trap_rust`:
441        // there is no frame of that function on its stack to return into.
442        // So it leaves the trap here instead, which is also what lets it
443        // pick up the interrupt state `mstatus` carries.
444        ".section .text.kairos_riscv_trampoline_trap,\"ax\"",
445        ".align 2",
446        ".global kairos_riscv_trampoline_trap",
447        ".type kairos_riscv_trampoline_trap, @function",
448        "kairos_riscv_trampoline_trap:",
449        "  mv    a0, s1", // task_fn
450        "  mv    a1, s2", // param
451        "  mret",         // -> mepc, which is the wrapper
452        ".size kairos_riscv_trampoline_trap, . - kairos_riscv_trampoline_trap",
453    );
454}
455
456// -------------------------------------------------------------- counters --
457
458/// Read `mcycle`, the architectural cycle counter.
459///
460/// This is the register the Cortex-M does not have under QEMU — DWT is
461/// unimplemented there, which is why the mission plan says cycle rows come
462/// from silicon. RISC-V defines `mcycle` in the ISA, so whether it means
463/// anything under an emulator is a question to MEASURE rather than assume,
464/// and the switch cell measures it.
465#[cfg(target_arch = "riscv32")]
466#[must_use]
467pub fn mcycle() -> u64 {
468    let lo: u32;
469    let hi: u32;
470    // SAFETY: reads two CSRs; touches no memory and cannot fault at
471    // machine level.
472    #[expect(unsafe_code, reason = "reading a counter CSR")]
473    unsafe {
474        core::arch::asm!("csrr {0}, mcycle", "csrr {1}, mcycleh", out(reg) lo, out(reg) hi, options(nomem, nostack));
475    }
476    (u64::from(hi) << 32) | u64::from(lo)
477}
478
479/// `minstret`: instructions retired. A *work* count rather than a clock,
480/// which is the instrument this family prefers.
481#[cfg(target_arch = "riscv32")]
482#[must_use]
483pub fn minstret() -> u64 {
484    let lo: u32;
485    let hi: u32;
486    // SAFETY: as `mcycle`.
487    #[expect(unsafe_code, reason = "reading a counter CSR")]
488    unsafe {
489        core::arch::asm!("csrr {0}, minstret", "csrr {1}, minstreth", out(reg) lo, out(reg) hi, options(nomem, nostack));
490    }
491    (u64::from(hi) << 32) | u64::from(lo)
492}
493
494/// Off RISC-V there is no `mcycle`; the signature is kept so callers
495/// type-check on the host.
496#[cfg(not(target_arch = "riscv32"))]
497#[must_use]
498pub fn mcycle() -> u64 {
499    0
500}
501
502/// As [`mcycle`].
503#[cfg(not(target_arch = "riscv32"))]
504#[must_use]
505pub fn minstret() -> u64 {
506    0
507}
508
509// ------------------------------------------------------------- the port --
510
511/// The Kairos RISC-V port.
512#[derive(Debug, Default)]
513pub struct RiscvPort {
514    nesting: AtomicU32,
515    /// Whether `mstatus.MIE` was set when the outermost section began.
516    was_enabled: AtomicU32,
517    yields: AtomicU32,
518    ticks: AtomicU32,
519    exits: AtomicU32,
520}
521
522impl RiscvPort {
523    /// A port with nothing counted yet.
524    #[must_use]
525    pub const fn new() -> Self {
526        Self {
527            nesting: AtomicU32::new(0),
528            was_enabled: AtomicU32::new(0),
529            yields: AtomicU32::new(0),
530            ticks: AtomicU32::new(0),
531            exits: AtomicU32::new(0),
532        }
533    }
534
535    /// Ticks delivered so far.
536    #[must_use]
537    pub fn tick_count(&self) -> u64 {
538        u64::from(self.ticks.load(Ordering::Relaxed))
539    }
540
541    /// Count one tick. A firmware calls this from its timer trap.
542    pub fn note_tick(&self) {
543        self.ticks.fetch_add(1, Ordering::Relaxed);
544    }
545
546    /// `portYIELD()` calls counted.
547    #[must_use]
548    pub fn yield_count(&self) -> u64 {
549        u64::from(self.yields.load(Ordering::Relaxed))
550    }
551}
552
553/// Clear `mstatus.MIE`, answering the OLD value of the bit -- `8` if it had been
554/// set, `0` if not.
555///
556/// The raw bit rather than a `bool`, because both callers only ever test it
557/// against zero and the only place it is stored is a `u32`. As a `bool` the
558/// round trip cost an extraction: `csrrci` leaves the bit in position 3, so
559/// `old & 8 != 0` followed by `u32::from` compiled to `slli 0x1c` + `srli 0x1f`.
560/// Masking in place is one `andi 8`.
561#[cfg(target_arch = "riscv32")]
562#[inline(always)]
563fn mask_interrupts() -> u32 {
564    let old: usize;
565    // SAFETY: clears the global interrupt-enable bit and reports its old
566    // value in one instruction; touches no memory.
567    #[expect(unsafe_code, reason = "the critical section is an mstatus write")]
568    unsafe {
569        core::arch::asm!("csrrci {0}, mstatus, 8", out(reg) old, options(nomem, nostack));
570    }
571    (old & 8) as u32
572}
573
574/// Set `mstatus.MIE`.
575#[cfg(target_arch = "riscv32")]
576#[inline(always)]
577fn unmask_interrupts() {
578    // SAFETY: sets the global interrupt-enable bit; touches no memory.
579    #[expect(unsafe_code, reason = "the critical section is an mstatus write")]
580    unsafe {
581        core::arch::asm!("csrsi mstatus, 8", options(nomem, nostack));
582    }
583}
584
585#[cfg(not(target_arch = "riscv32"))]
586#[inline(always)]
587fn mask_interrupts() -> u32 {
588    0
589}
590
591#[cfg(not(target_arch = "riscv32"))]
592#[inline(always)]
593fn unmask_interrupts() {}
594
595/// Raise the machine software interrupt.
596#[cfg(target_arch = "riscv32")]
597#[expect(unsafe_code, reason = "raising an interrupt is a device write")]
598#[inline]
599pub fn raise_switch() {
600    // SAFETY: `CLINT_MSIP` is the CLINT's software-interrupt register for
601    // hart 0 on this machine; writing 1 raises it and nothing else.
602    unsafe {
603        (CLINT_MSIP as *mut u32).write_volatile(1);
604    }
605}
606
607/// Clear the machine software interrupt. The handler must do this first.
608#[cfg(target_arch = "riscv32")]
609#[expect(unsafe_code, reason = "clearing an interrupt is a device write")]
610#[inline]
611pub fn clear_switch_request() {
612    // SAFETY: as `raise_switch`.
613    unsafe {
614        (CLINT_MSIP as *mut u32).write_volatile(0);
615    }
616}
617
618/// Off RISC-V there is no CLINT and nothing to raise.
619#[cfg(not(target_arch = "riscv32"))]
620#[inline]
621pub fn raise_switch() {}
622
623/// As [`raise_switch`].
624#[cfg(not(target_arch = "riscv32"))]
625#[inline]
626pub fn clear_switch_request() {}
627
628impl Port for RiscvPort {
629    /// This port SWITCHES STACKS, so the kernel must not commit a switch at
630    /// the point of the yield — the registers do not move until the trap is
631    /// taken. See the Xtensa port for what happens when it does.
632    const COMMITS_SWITCH: bool = true;
633
634    #[inline]
635    fn yield_now(&self) {
636        // No counter here: `Kernel::port_yield` calls `count_yield` just
637        // before this, and counting in both places doubles every yield.
638        raise_switch();
639    }
640
641    #[inline]
642    fn yield_from_isr(&self, woken: Woken) {
643        if woken == Woken::YES {
644            raise_switch();
645        }
646    }
647
648    /// In line on the SPEED profile, out of line under `small` -- and C's is a
649    /// MACRO, so the asymmetry is priced both ways.
650    ///
651    /// Our arm makes **111 calls** to this pair; the C arm has no
652    /// `vTaskEnterCritical` symbol at all and 237 INLINE `csr` ops, because
653    /// `portENTER_CRITICAL()` is `csrc mstatus, 8` plus a plain non-atomic
654    /// `xCriticalNesting++`. Ours additionally saves and restores the previous
655    /// interrupt state, which C never does (C unconditionally re-enables).
656    ///
657    /// `#[inline]` on the pair was measured on 2026-09-25: **+1,406 B** as
658    /// written, and **+2,164 B** even cut down to C's exact semantics, and
659    /// declined -- one body plus 111 `jal` beat 111 copies while there was ONE
660    /// profile and the K3 flash row rode on it. Since 2026-10-02 the flash row
661    /// is carried by the kernel's `small` profile (1.24x, inside the target),
662    /// and the default is the speed profile. Re-priced 2026-10-03 on the
663    /// shipped port (`bench/tick-work`, `--features real-port`): the opaque
664    /// call is not just `mv`/`jal`/`ret` -- every queue and TCB field read
665    /// before it is re-read and re-checked after it, because LLVM cannot see
666    /// that the port touches only itself. In line, 13 rows fall by 284
667    /// instructions: `peek_ok` 73 -> 41, `queue_roundtrip` 152 -> 120,
668    /// `block_cycle` 947 -> 855, `scaffolding` 22 -> 13. Flash +1,934 B on the
669    /// speed profile; under `small` (this crate's feature, which the flash
670    /// profile turns on) nothing changes.
671    #[cfg_attr(not(feature = "small"), inline)]
672    fn enter_critical(&self) {
673        let was = mask_interrupts();
674        // Only the OUTERMOST section's state is kept: an inner mask reports
675        // interrupts already off, and restoring that on the way out would
676        // leave them off for good.
677        if self.nesting.fetch_add(1, Ordering::Relaxed) == 0 {
678            self.was_enabled.store(was, Ordering::Relaxed);
679        }
680    }
681
682    /// See [`RiscvPort::enter_critical`] for when this is in line.
683    ///
684    /// The `exits` counter is read only by a trace sink, and removing it
685    /// entirely measured **8 B** out of line, where the body costs ONCE. In
686    /// line it is an `amoadd` per site; unpriced there.
687    #[cfg_attr(not(feature = "small"), inline)]
688    fn exit_critical(&self) {
689        let n = self.nesting.load(Ordering::Relaxed).saturating_sub(1);
690        self.nesting.store(n, Ordering::Relaxed);
691        if n == 0 {
692            self.exits.fetch_add(1, Ordering::Relaxed);
693            if self.was_enabled.load(Ordering::Relaxed) != 0 {
694                unmask_interrupts();
695            }
696        }
697    }
698
699    #[inline]
700    fn set_interrupt_mask_from_isr(&self) -> u32 {
701        mask_interrupts()
702    }
703
704    #[inline]
705    fn clear_interrupt_mask_from_isr(&self, saved: u32) {
706        if saved != 0 {
707            unmask_interrupts();
708        }
709    }
710
711    #[inline]
712    fn in_isr(&self) -> bool {
713        // RISC-V has no equivalent of ARM's `IPSR`: a hart in a trap looks
714        // like a hart with interrupts masked. This is therefore weaker than
715        // the Cortex-M answer -- a task inside a critical section reads
716        // `true` -- and it is safe in the direction that matters, because a
717        // task taking the from-ISR half of an API is sound where the
718        // reverse would not be.
719        self.nesting.load(Ordering::Relaxed) != 0
720    }
721
722    #[inline]
723    fn count_tick(&self) {
724        self.note_tick();
725    }
726
727    #[inline]
728    fn count_yield(&self) {
729        self.yields.fetch_add(1, Ordering::Relaxed);
730    }
731
732    // Three instructions (`lw`, a high-word zero, `ret`) behind a call the
733    // kernel makes from 45 sites. The profile is `lto = false`, so without
734    // this attribute the body is not even a CANDIDATE for cross-crate
735    // inlining, and each site pays `mv a0, sN` + `jal` to run it.
736    #[inline]
737    fn exits(&self) -> u64 {
738        u64::from(self.exits.load(Ordering::Relaxed))
739    }
740
741    #[inline]
742    fn idle(&self) {
743        idle_wait();
744    }
745}
746
747/// `wfi`: sleep until the next interrupt.
748#[cfg(target_arch = "riscv32")]
749#[expect(unsafe_code, reason = "the idle instruction")]
750#[inline]
751fn idle_wait() {
752    // SAFETY: `wfi` is a hint; it cannot fault.
753    unsafe {
754        core::arch::asm!("wfi", options(nomem, nostack));
755    }
756}
757
758#[cfg(not(target_arch = "riscv32"))]
759#[inline]
760fn idle_wait() {}