rusty_rtos_port_riscv/lib.rs
1#![no_std]
2//! `rusty_rtos_port-riscv` — the Kairos RISC-V port.
3//!
4//! RV32, for QEMU `virt` and for the ESP32-C6/C61/H2/P4 family.
5//!
6//! # Simpler than Xtensa, and not for the reason you would guess
7//!
8//! Xtensa's difficulty is the register windows: a task several calls deep
9//! has live frames in a 64-entry file that must reach its own stack before
10//! anything else runs. RISC-V has no windows, so there is nothing to spill.
11//!
12//! But it is NOT simply "Cortex-M with different names", and the difference
13//! is worth stating because it decides the design:
14//!
15//! | | who saves the callee-saved registers |
16//! |---|---|
17//! | Cortex-M | the hardware stacks `r0-r3, r12, lr, pc, xPSR`; the port adds `r4-r11` |
18//! | Xtensa | the window spill, driven by `xtensa-lx-rt`'s exception entry |
19//! | **RISC-V** | **nobody — the port must do all of it** |
20//!
21//! A RISC-V trap saves *nothing* to the stack. `riscv-rt`'s entry saves the
22//! caller-saved registers into a `TrapFrame` so a normal Rust handler can
23//! run, and the callee-saved ones survive only because that handler is an
24//! ordinary function that preserves them. For a context switch they belong
25//! to the outgoing task and have to be written down, so this port keeps its
26//! own frame: `ra`, `sp` and `s0`–`s11`.
27//!
28//! # The switch still happens in the trap
29//!
30//! The lesson from the Xtensa port holds: the decision and the register
31//! swap are one step, taken where the context is already captured. A yield
32//! raises the machine software interrupt through the CLINT; the handler
33//! swaps and returns. [`Port::COMMITS_SWITCH`] is `true` for exactly that
34//! reason — the kernel must not move `current` at the point of the yield,
35//! because the registers do not move until the trap is taken.
36//!
37//! # Unsafe
38//!
39//! Every use is fenced with `#[expect(unsafe_code)]` and a justification.
40//! See `UNSAFE.md`.
41
42use core::sync::atomic::{AtomicU32, Ordering};
43
44use rusty_rtos_core::isr::Woken;
45use rusty_rtos_core::port::Port;
46
47/// Kani proofs of the stack builder (hardening gate H-30).
48#[cfg(kani)]
49mod proofs;
50
51/// Crate version, for manifests and logs.
52pub const VERSION: &str = env!("CARGO_PKG_VERSION");
53
54/// The CLINT's `msip` register for hart 0 on QEMU `virt`.
55///
56/// Writing 1 raises the machine software interrupt; writing 0 clears it.
57/// The ESP32-C6 family reaches the same interrupt through its own
58/// peripheral, which is why this is a constant a board can override rather
59/// than something baked into the switch.
60pub const CLINT_MSIP: usize = 0x0200_0000;
61
62/// How many registers the port itself saves: `ra`, `sp`, `s0`–`s11`.
63pub const SAVED_REGISTERS: usize = 14;
64
65/// The saved machine state of one task.
66///
67/// Only the registers a trap does NOT already preserve. `riscv-rt` captures
68/// the caller-saved set into its own `TrapFrame` on the way in and restores
69/// it on the way out, so duplicating it here would be two places that must
70/// agree about the same bytes.
71#[repr(C)]
72#[derive(Debug, Clone, Copy, Default)]
73pub struct Context {
74 /// Return address — where the task resumes.
75 pub ra: usize,
76 /// Stack pointer.
77 pub sp: usize,
78 /// Callee-saved `s0`–`s11`.
79 pub s: [usize; 12],
80 /// `mepc` — where the task resumes when the trap exits.
81 ///
82 /// Only the PREEMPTIVE path uses this. A cooperative switch happens at a
83 /// call site and resumes through `ra`; a preemptive one resumes through
84 /// `mret`, and `mret` jumps to `mepc`.
85 ///
86 /// Saving it is **not optional, and that is measured rather than
87 /// asserted**: the `mepc` in the CSR at the moment of a switch belongs to
88 /// the task being switched OUT, so a port that leaves it alone resumes
89 /// each task at the other one's address. Deleting the restore hangs
90 /// `riscv32-qemu-preempt` on the spot.
91 pub mepc: usize,
92 /// `mstatus` — in particular `MPIE`, which `mret` copies into `MIE`.
93 ///
94 /// Saved and restored because a task's interrupt state is its own: a task
95 /// preempted inside a critical section must come back with that section
96 /// still in force.
97 ///
98 /// **It is NOT what makes preemption work, and an earlier version of this
99 /// comment said it was.** Removing the restore and re-running
100 /// `riscv32-qemu-preempt` still passes, because the trap entry has already
101 /// copied the live `MIE` into `MPIE`, so `mret` re-enables interrupts
102 /// without our help. The two lines that ARE load bearing are [`mepc`]
103 /// below and the `mret` in `kairos_riscv_trampoline_trap`; removing
104 /// either hangs the cell. Poison-proved, all three.
105 ///
106 /// [`mepc`]: Context::mepc
107 pub mstatus: usize,
108}
109
110impl Context {
111 /// An empty context, for a slot the first switch will fill in.
112 #[must_use]
113 pub const fn new() -> Self {
114 Self {
115 ra: 0,
116 sp: 0,
117 s: [0; 12],
118 mepc: 0,
119 mstatus: 0,
120 }
121 }
122}
123
124/// Build the saved context of a task that has never run.
125///
126/// `stack_top` is one past the highest usable byte; it is rounded down to
127/// the 16-byte alignment the ABI requires of `sp`. When the task is first
128/// switched to, `wrapper` runs with `task_fn` and `param` as its arguments.
129///
130/// The wrapper exists because a task body must not simply `return`: there is
131/// nothing beneath it to return into.
132///
133/// # Safety
134///
135/// `stack_top` must be one past a writable stack that outlives the task.
136#[expect(unsafe_code, reason = "writing the initial frame")]
137#[must_use]
138pub unsafe fn new_task_context(
139 wrapper: extern "C" fn(task_fn: usize, param: usize) -> !,
140 task_fn: usize,
141 param: usize,
142 stack_top: *mut u8,
143) -> Context {
144 // Pointer-width arithmetic, narrowed nowhere: the Xtensa port shipped a
145 // version that did this in `u32` and wrote through a truncated address
146 // the first time a host test called it.
147 let top = (stack_top as usize) & !0xf;
148 let mut context = Context::new();
149 // A fresh task resumes through a TRAMPOLINE, not straight into the
150 // wrapper, and the reason is the calling convention.
151 //
152 // The switch restores only the callee-saved set — `ra`, `sp`, `s0`-`s11`
153 // — because those are the registers a trap does not preserve. A
154 // `extern "C"` function reads its arguments from `a0`/`a1`, which are
155 // caller-saved and therefore NOT in that set. Putting the arguments in
156 // `s0`/`s1` and jumping straight to the wrapper hands it whatever
157 // `a0`/`a1` happened to hold: the first version of this port did that
158 // and hung on the first switch.
159 //
160 // So the entry point and its two arguments travel in `s0`-`s2`, and
161 // three `mv` instructions put them where the ABI expects.
162 context.ra = (kairos_riscv_trampoline as *const ()) as usize;
163 context.sp = top;
164 context.s[0] = wrapper as *const () as usize;
165 context.s[1] = task_fn;
166 context.s[2] = param;
167 context
168}
169
170#[cfg(target_arch = "riscv32")]
171#[expect(unsafe_code, reason = "the trampoline is defined in this crate's asm")]
172unsafe extern "C" {
173 /// Defined in this crate's assembly; see [`new_task_context`].
174 fn kairos_riscv_trampoline();
175 /// Defined in this crate's assembly; see [`new_task_context_preemptive`].
176 fn kairos_riscv_trampoline_trap();
177}
178
179/// Off RISC-V the trampoline is a plain address for the frame arithmetic.
180#[cfg(not(target_arch = "riscv32"))]
181extern "C" fn kairos_riscv_trampoline() {}
182
183/// Off RISC-V, as above.
184#[cfg(not(target_arch = "riscv32"))]
185extern "C" fn kairos_riscv_trampoline_trap() {}
186
187/// Swap the running task.
188///
189/// Called from inside the switching trap. `current` is where the outgoing
190/// task's registers are written — `None` when it is being discarded — and
191/// `next` is the task to resume.
192///
193/// # Safety
194///
195/// Both pointers must be valid, and `next` must hold a context built by
196/// [`new_task_context`] or written by a previous call to this.
197#[cfg(target_arch = "riscv32")]
198#[expect(unsafe_code, reason = "the context switch itself")]
199#[inline(always)]
200#[allow(
201 clippy::unwrap_or_default,
202 reason = "`Default` for raw pointers is Rust 1.88 and the MSRV is 1.85; `allow`, not `expect`, because only clippys that know the impl fire it"
203)]
204pub unsafe fn switch_context(current: Option<*mut Context>, next: *const Context) {
205 // SAFETY: the contract above. `kairos_riscv_switch` writes the outgoing
206 // registers through `a0` and loads the incoming ones from `a1`; it never
207 // touches memory outside those two structs.
208 unsafe {
209 core::arch::asm!(
210 "call kairos_riscv_switch",
211 in("a0") current.unwrap_or(core::ptr::null_mut()),
212 in("a1") next,
213 clobber_abi("C"),
214 );
215 }
216}
217
218/// Off RISC-V this does nothing, so callers keep type-checking on the host.
219///
220/// # Safety
221/// As the RISC-V version; this one is a no-op.
222#[cfg(not(target_arch = "riscv32"))]
223#[expect(unsafe_code, reason = "signature parity with the RISC-V version")]
224#[inline(always)]
225pub unsafe fn switch_context(_current: Option<*mut Context>, _next: *const Context) {}
226
227/// Swap the running task from inside a trap, so the incoming task resumes
228/// through `mret`.
229///
230/// # Why this is a second function and not a flag on the first
231///
232/// [`switch_context`] resumes a task by RETURNING to its `ra`. That is
233/// exactly right at a call site, and it is what makes a cooperative yield 30
234/// instructions against FreeRTOS's 83.
235///
236/// It is wrong inside a trap. A trap is left with `mret`, which restores
237/// `MIE` from `MPIE` and jumps to `mepc`; a task entered with `ret` instead
238/// keeps running in trap context with interrupts masked, and the hart takes
239/// no further interrupts — the first preemptive switch becomes the last.
240/// That was a real defect, witnessed by
241/// `rusty_rtos_port/firmware/riscv32-qemu-preempt`.
242///
243/// So this one also swaps `mepc` and `mstatus`, and a fresh task leaves its
244/// first trap through `mret` rather than `ret`. Keeping the two switches
245/// apart means a yield still pays only for what a yield needs.
246///
247/// Of those three changes, poison-proving says **`mepc` and the `mret` are
248/// what make it work** — remove either and the cell hangs — while `mstatus`
249/// is correctness for the general case rather than the thing that was
250/// broken. See [`Context::mstatus`].
251///
252/// # Safety
253///
254/// Must be called from inside a trap handler. Both pointers must be valid,
255/// and `next` must hold a context built by [`new_task_context_preemptive`]
256/// or written by a previous call to this.
257#[cfg(target_arch = "riscv32")]
258#[expect(unsafe_code, reason = "the context switch itself")]
259#[inline(always)]
260pub unsafe fn switch_context_trap(current: *mut Context, next: *const Context) {
261 // SAFETY: the contract above.
262 unsafe {
263 core::arch::asm!(
264 "call kairos_riscv_switch_trap",
265 in("a0") current,
266 in("a1") next,
267 clobber_abi("C"),
268 );
269 }
270}
271
272/// Off RISC-V this does nothing, so callers keep type-checking on the host.
273///
274/// # Safety
275/// As the RISC-V version; this one is a no-op.
276#[cfg(not(target_arch = "riscv32"))]
277#[expect(unsafe_code, reason = "signature parity with the RISC-V version")]
278#[inline(always)]
279pub unsafe fn switch_context_trap(_current: *mut Context, _next: *const Context) {}
280
281/// `mstatus` for a task that has never run: `MPIE` set so `mret` enables
282/// interrupts, and `MPP` = M-mode so it returns to the privilege it left.
283///
284/// FreeRTOS computes the same value (`0x188 << 4`) in
285/// `pxPortInitialiseStack`, ORing it into the live `mstatus` to preserve the
286/// FPU and VPU dirty bits. This port has no FPU save, so there is nothing to
287/// preserve and the constant stands alone.
288pub const FRESH_MSTATUS: usize = 0x1880;
289
290/// Build the saved context of a task that has never run, for the PREEMPTIVE
291/// path.
292///
293/// The difference from [`new_task_context`] is where the task resumes from.
294/// There, the first switch `ret`s into a trampoline and falls straight into
295/// the wrapper. Here the first switch happens inside a trap, so the task must
296/// leave through `mret` like any preempted task would — otherwise it runs
297/// with interrupts masked for ever.
298///
299/// So `mepc` is the wrapper and `mstatus` carries `MPIE`, and the trampoline
300/// this context returns to does nothing but move the two arguments into
301/// place and `mret`.
302///
303/// # Safety
304///
305/// `stack_top` must be one past a writable stack that outlives the task.
306#[expect(unsafe_code, reason = "writing the initial frame")]
307#[must_use]
308pub unsafe fn new_task_context_preemptive(
309 wrapper: extern "C" fn(task_fn: usize, param: usize) -> !,
310 task_fn: usize,
311 param: usize,
312 stack_top: *mut u8,
313) -> Context {
314 let mut context = Context::new();
315 context.ra = (kairos_riscv_trampoline_trap as *const ()) as usize;
316 context.sp = (stack_top as usize) & !0xf;
317 // The arguments travel in `s1`/`s2` for the same reason as the
318 // cooperative path: a switch restores only the callee-saved set, and a
319 // `extern "C"` function reads its arguments from the caller-saved one.
320 context.s[1] = task_fn;
321 context.s[2] = param;
322 context.mepc = (wrapper as *const ()) as usize;
323 context.mstatus = FRESH_MSTATUS;
324 context
325}
326
327/// The switch itself.
328#[cfg(target_arch = "riscv32")]
329#[expect(unsafe_code, reason = "the context switch itself")]
330mod asm {
331 core::arch::global_asm!(
332 ".section .text.kairos_riscv_switch,\"ax\"",
333 ".align 2",
334 ".global kairos_riscv_switch",
335 ".type kairos_riscv_switch, @function",
336 "kairos_riscv_switch:",
337 // a0 = *mut Context for the outgoing task, or null.
338 // a1 = *const Context for the incoming task.
339 //
340 // Fourteen words, in the order `Context` declares them: ra, sp,
341 // then s0-s11. `#[repr(C)]` is what makes that order a contract
342 // rather than a coincidence.
343 " beqz a0, 2f",
344 " sw ra, 0(a0)",
345 " sw sp, 4(a0)",
346 " sw s0, 8(a0)",
347 " sw s1, 12(a0)",
348 " sw s2, 16(a0)",
349 " sw s3, 20(a0)",
350 " sw s4, 24(a0)",
351 " sw s5, 28(a0)",
352 " sw s6, 32(a0)",
353 " sw s7, 36(a0)",
354 " sw s8, 40(a0)",
355 " sw s9, 44(a0)",
356 " sw s10, 48(a0)",
357 " sw s11, 52(a0)",
358 "2:",
359 " lw ra, 0(a1)",
360 " lw sp, 4(a1)",
361 " lw s0, 8(a1)",
362 " lw s1, 12(a1)",
363 " lw s2, 16(a1)",
364 " lw s3, 20(a1)",
365 " lw s4, 24(a1)",
366 " lw s5, 28(a1)",
367 " lw s6, 32(a1)",
368 " lw s7, 36(a1)",
369 " lw s8, 40(a1)",
370 " lw s9, 44(a1)",
371 " lw s10, 48(a1)",
372 " lw s11, 52(a1)",
373 " ret",
374 ".size kairos_riscv_switch, . - kairos_riscv_switch",
375 // A fresh task lands here rather than in its entry point, because
376 // the switch restores callee-saved registers and the C ABI passes
377 // arguments in caller-saved ones. See `new_task_context`.
378 ".section .text.kairos_riscv_trampoline,\"ax\"",
379 ".align 2",
380 ".global kairos_riscv_trampoline",
381 ".type kairos_riscv_trampoline, @function",
382 "kairos_riscv_trampoline:",
383 " mv a0, s1", // task_fn
384 " mv a1, s2", // param
385 " jr s0", // the wrapper, which never returns
386 ".size kairos_riscv_trampoline, . - kairos_riscv_trampoline",
387 // ------------------------------------------------------------------
388 // The PREEMPTIVE switch. Same fourteen registers, plus the two CSRs
389 // that decide how the trap is left: `mepc` (where) and `mstatus`
390 // (with what interrupt state). Without those two a task entered from
391 // a trap runs with interrupts masked for ever -- which is exactly
392 // the defect `riscv32-qemu-preempt` witnessed.
393 //
394 // `t0` is scratch. It is caller-saved, so `riscv-rt`'s trap epilogue
395 // reloads it from the INCOMING task's frame on the way out; clobbering
396 // it after `sp` has moved is therefore safe.
397 ".section .text.kairos_riscv_switch_trap,\"ax\"",
398 ".align 2",
399 ".global kairos_riscv_switch_trap",
400 ".type kairos_riscv_switch_trap, @function",
401 "kairos_riscv_switch_trap:",
402 " sw ra, 0(a0)",
403 " sw sp, 4(a0)",
404 " sw s0, 8(a0)",
405 " sw s1, 12(a0)",
406 " sw s2, 16(a0)",
407 " sw s3, 20(a0)",
408 " sw s4, 24(a0)",
409 " sw s5, 28(a0)",
410 " sw s6, 32(a0)",
411 " sw s7, 36(a0)",
412 " sw s8, 40(a0)",
413 " sw s9, 44(a0)",
414 " sw s10, 48(a0)",
415 " sw s11, 52(a0)",
416 " csrr t0, mepc",
417 " sw t0, 56(a0)",
418 " csrr t0, mstatus",
419 " sw t0, 60(a0)",
420 " lw ra, 0(a1)",
421 " lw sp, 4(a1)",
422 " lw s0, 8(a1)",
423 " lw s1, 12(a1)",
424 " lw s2, 16(a1)",
425 " lw s3, 20(a1)",
426 " lw s4, 24(a1)",
427 " lw s5, 28(a1)",
428 " lw s6, 32(a1)",
429 " lw s7, 36(a1)",
430 " lw s8, 40(a1)",
431 " lw s9, 44(a1)",
432 " lw s10, 48(a1)",
433 " lw s11, 52(a1)",
434 " lw t0, 56(a1)",
435 " csrw mepc, t0",
436 " lw t0, 60(a1)",
437 " csrw mstatus, t0",
438 " ret",
439 ".size kairos_riscv_switch_trap, . - kairos_riscv_switch_trap",
440 // A task that has never run cannot `ret` into `_start_trap_rust`:
441 // there is no frame of that function on its stack to return into.
442 // So it leaves the trap here instead, which is also what lets it
443 // pick up the interrupt state `mstatus` carries.
444 ".section .text.kairos_riscv_trampoline_trap,\"ax\"",
445 ".align 2",
446 ".global kairos_riscv_trampoline_trap",
447 ".type kairos_riscv_trampoline_trap, @function",
448 "kairos_riscv_trampoline_trap:",
449 " mv a0, s1", // task_fn
450 " mv a1, s2", // param
451 " mret", // -> mepc, which is the wrapper
452 ".size kairos_riscv_trampoline_trap, . - kairos_riscv_trampoline_trap",
453 );
454}
455
456// -------------------------------------------------------------- counters --
457
458/// Read `mcycle`, the architectural cycle counter.
459///
460/// This is the register the Cortex-M does not have under QEMU — DWT is
461/// unimplemented there, which is why the mission plan says cycle rows come
462/// from silicon. RISC-V defines `mcycle` in the ISA, so whether it means
463/// anything under an emulator is a question to MEASURE rather than assume,
464/// and the switch cell measures it.
465#[cfg(target_arch = "riscv32")]
466#[must_use]
467pub fn mcycle() -> u64 {
468 let lo: u32;
469 let hi: u32;
470 // SAFETY: reads two CSRs; touches no memory and cannot fault at
471 // machine level.
472 #[expect(unsafe_code, reason = "reading a counter CSR")]
473 unsafe {
474 core::arch::asm!("csrr {0}, mcycle", "csrr {1}, mcycleh", out(reg) lo, out(reg) hi, options(nomem, nostack));
475 }
476 (u64::from(hi) << 32) | u64::from(lo)
477}
478
479/// `minstret`: instructions retired. A *work* count rather than a clock,
480/// which is the instrument this family prefers.
481#[cfg(target_arch = "riscv32")]
482#[must_use]
483pub fn minstret() -> u64 {
484 let lo: u32;
485 let hi: u32;
486 // SAFETY: as `mcycle`.
487 #[expect(unsafe_code, reason = "reading a counter CSR")]
488 unsafe {
489 core::arch::asm!("csrr {0}, minstret", "csrr {1}, minstreth", out(reg) lo, out(reg) hi, options(nomem, nostack));
490 }
491 (u64::from(hi) << 32) | u64::from(lo)
492}
493
494/// Off RISC-V there is no `mcycle`; the signature is kept so callers
495/// type-check on the host.
496#[cfg(not(target_arch = "riscv32"))]
497#[must_use]
498pub fn mcycle() -> u64 {
499 0
500}
501
502/// As [`mcycle`].
503#[cfg(not(target_arch = "riscv32"))]
504#[must_use]
505pub fn minstret() -> u64 {
506 0
507}
508
509// ------------------------------------------------------------- the port --
510
511/// The Kairos RISC-V port.
512#[derive(Debug, Default)]
513pub struct RiscvPort {
514 nesting: AtomicU32,
515 /// Whether `mstatus.MIE` was set when the outermost section began.
516 was_enabled: AtomicU32,
517 yields: AtomicU32,
518 ticks: AtomicU32,
519 exits: AtomicU32,
520}
521
522impl RiscvPort {
523 /// A port with nothing counted yet.
524 #[must_use]
525 pub const fn new() -> Self {
526 Self {
527 nesting: AtomicU32::new(0),
528 was_enabled: AtomicU32::new(0),
529 yields: AtomicU32::new(0),
530 ticks: AtomicU32::new(0),
531 exits: AtomicU32::new(0),
532 }
533 }
534
535 /// Ticks delivered so far.
536 #[must_use]
537 pub fn tick_count(&self) -> u64 {
538 u64::from(self.ticks.load(Ordering::Relaxed))
539 }
540
541 /// Count one tick. A firmware calls this from its timer trap.
542 pub fn note_tick(&self) {
543 self.ticks.fetch_add(1, Ordering::Relaxed);
544 }
545
546 /// `portYIELD()` calls counted.
547 #[must_use]
548 pub fn yield_count(&self) -> u64 {
549 u64::from(self.yields.load(Ordering::Relaxed))
550 }
551}
552
553/// Clear `mstatus.MIE`, answering the OLD value of the bit -- `8` if it had been
554/// set, `0` if not.
555///
556/// The raw bit rather than a `bool`, because both callers only ever test it
557/// against zero and the only place it is stored is a `u32`. As a `bool` the
558/// round trip cost an extraction: `csrrci` leaves the bit in position 3, so
559/// `old & 8 != 0` followed by `u32::from` compiled to `slli 0x1c` + `srli 0x1f`.
560/// Masking in place is one `andi 8`.
561#[cfg(target_arch = "riscv32")]
562#[inline(always)]
563fn mask_interrupts() -> u32 {
564 let old: usize;
565 // SAFETY: clears the global interrupt-enable bit and reports its old
566 // value in one instruction; touches no memory.
567 #[expect(unsafe_code, reason = "the critical section is an mstatus write")]
568 unsafe {
569 core::arch::asm!("csrrci {0}, mstatus, 8", out(reg) old, options(nomem, nostack));
570 }
571 (old & 8) as u32
572}
573
574/// Set `mstatus.MIE`.
575#[cfg(target_arch = "riscv32")]
576#[inline(always)]
577fn unmask_interrupts() {
578 // SAFETY: sets the global interrupt-enable bit; touches no memory.
579 #[expect(unsafe_code, reason = "the critical section is an mstatus write")]
580 unsafe {
581 core::arch::asm!("csrsi mstatus, 8", options(nomem, nostack));
582 }
583}
584
585#[cfg(not(target_arch = "riscv32"))]
586#[inline(always)]
587fn mask_interrupts() -> u32 {
588 0
589}
590
591#[cfg(not(target_arch = "riscv32"))]
592#[inline(always)]
593fn unmask_interrupts() {}
594
595/// Raise the machine software interrupt.
596#[cfg(target_arch = "riscv32")]
597#[expect(unsafe_code, reason = "raising an interrupt is a device write")]
598#[inline]
599pub fn raise_switch() {
600 // SAFETY: `CLINT_MSIP` is the CLINT's software-interrupt register for
601 // hart 0 on this machine; writing 1 raises it and nothing else.
602 unsafe {
603 (CLINT_MSIP as *mut u32).write_volatile(1);
604 }
605}
606
607/// Clear the machine software interrupt. The handler must do this first.
608#[cfg(target_arch = "riscv32")]
609#[expect(unsafe_code, reason = "clearing an interrupt is a device write")]
610#[inline]
611pub fn clear_switch_request() {
612 // SAFETY: as `raise_switch`.
613 unsafe {
614 (CLINT_MSIP as *mut u32).write_volatile(0);
615 }
616}
617
618/// Off RISC-V there is no CLINT and nothing to raise.
619#[cfg(not(target_arch = "riscv32"))]
620#[inline]
621pub fn raise_switch() {}
622
623/// As [`raise_switch`].
624#[cfg(not(target_arch = "riscv32"))]
625#[inline]
626pub fn clear_switch_request() {}
627
628impl Port for RiscvPort {
629 /// This port SWITCHES STACKS, so the kernel must not commit a switch at
630 /// the point of the yield — the registers do not move until the trap is
631 /// taken. See the Xtensa port for what happens when it does.
632 const COMMITS_SWITCH: bool = true;
633
634 #[inline]
635 fn yield_now(&self) {
636 // No counter here: `Kernel::port_yield` calls `count_yield` just
637 // before this, and counting in both places doubles every yield.
638 raise_switch();
639 }
640
641 #[inline]
642 fn yield_from_isr(&self, woken: Woken) {
643 if woken == Woken::YES {
644 raise_switch();
645 }
646 }
647
648 /// In line on the SPEED profile, out of line under `small` -- and C's is a
649 /// MACRO, so the asymmetry is priced both ways.
650 ///
651 /// Our arm makes **111 calls** to this pair; the C arm has no
652 /// `vTaskEnterCritical` symbol at all and 237 INLINE `csr` ops, because
653 /// `portENTER_CRITICAL()` is `csrc mstatus, 8` plus a plain non-atomic
654 /// `xCriticalNesting++`. Ours additionally saves and restores the previous
655 /// interrupt state, which C never does (C unconditionally re-enables).
656 ///
657 /// `#[inline]` on the pair was measured on 2026-09-25: **+1,406 B** as
658 /// written, and **+2,164 B** even cut down to C's exact semantics, and
659 /// declined -- one body plus 111 `jal` beat 111 copies while there was ONE
660 /// profile and the K3 flash row rode on it. Since 2026-10-02 the flash row
661 /// is carried by the kernel's `small` profile (1.24x, inside the target),
662 /// and the default is the speed profile. Re-priced 2026-10-03 on the
663 /// shipped port (`bench/tick-work`, `--features real-port`): the opaque
664 /// call is not just `mv`/`jal`/`ret` -- every queue and TCB field read
665 /// before it is re-read and re-checked after it, because LLVM cannot see
666 /// that the port touches only itself. In line, 13 rows fall by 284
667 /// instructions: `peek_ok` 73 -> 41, `queue_roundtrip` 152 -> 120,
668 /// `block_cycle` 947 -> 855, `scaffolding` 22 -> 13. Flash +1,934 B on the
669 /// speed profile; under `small` (this crate's feature, which the flash
670 /// profile turns on) nothing changes.
671 #[cfg_attr(not(feature = "small"), inline)]
672 fn enter_critical(&self) {
673 let was = mask_interrupts();
674 // Only the OUTERMOST section's state is kept: an inner mask reports
675 // interrupts already off, and restoring that on the way out would
676 // leave them off for good.
677 if self.nesting.fetch_add(1, Ordering::Relaxed) == 0 {
678 self.was_enabled.store(was, Ordering::Relaxed);
679 }
680 }
681
682 /// See [`RiscvPort::enter_critical`] for when this is in line.
683 ///
684 /// The `exits` counter is read only by a trace sink, and removing it
685 /// entirely measured **8 B** out of line, where the body costs ONCE. In
686 /// line it is an `amoadd` per site; unpriced there.
687 #[cfg_attr(not(feature = "small"), inline)]
688 fn exit_critical(&self) {
689 let n = self.nesting.load(Ordering::Relaxed).saturating_sub(1);
690 self.nesting.store(n, Ordering::Relaxed);
691 if n == 0 {
692 self.exits.fetch_add(1, Ordering::Relaxed);
693 if self.was_enabled.load(Ordering::Relaxed) != 0 {
694 unmask_interrupts();
695 }
696 }
697 }
698
699 #[inline]
700 fn set_interrupt_mask_from_isr(&self) -> u32 {
701 mask_interrupts()
702 }
703
704 #[inline]
705 fn clear_interrupt_mask_from_isr(&self, saved: u32) {
706 if saved != 0 {
707 unmask_interrupts();
708 }
709 }
710
711 #[inline]
712 fn in_isr(&self) -> bool {
713 // RISC-V has no equivalent of ARM's `IPSR`: a hart in a trap looks
714 // like a hart with interrupts masked. This is therefore weaker than
715 // the Cortex-M answer -- a task inside a critical section reads
716 // `true` -- and it is safe in the direction that matters, because a
717 // task taking the from-ISR half of an API is sound where the
718 // reverse would not be.
719 self.nesting.load(Ordering::Relaxed) != 0
720 }
721
722 #[inline]
723 fn count_tick(&self) {
724 self.note_tick();
725 }
726
727 #[inline]
728 fn count_yield(&self) {
729 self.yields.fetch_add(1, Ordering::Relaxed);
730 }
731
732 // Three instructions (`lw`, a high-word zero, `ret`) behind a call the
733 // kernel makes from 45 sites. The profile is `lto = false`, so without
734 // this attribute the body is not even a CANDIDATE for cross-crate
735 // inlining, and each site pays `mv a0, sN` + `jal` to run it.
736 #[inline]
737 fn exits(&self) -> u64 {
738 u64::from(self.exits.load(Ordering::Relaxed))
739 }
740
741 #[inline]
742 fn idle(&self) {
743 idle_wait();
744 }
745}
746
747/// `wfi`: sleep until the next interrupt.
748#[cfg(target_arch = "riscv32")]
749#[expect(unsafe_code, reason = "the idle instruction")]
750#[inline]
751fn idle_wait() {
752 // SAFETY: `wfi` is a hint; it cannot fault.
753 unsafe {
754 core::arch::asm!("wfi", options(nomem, nostack));
755 }
756}
757
758#[cfg(not(target_arch = "riscv32"))]
759#[inline]
760fn idle_wait() {}