use crate::config::{ConfigError, PlatformLimits, SchedulerConfig, Slice};
use crate::tcb::{TaskControlBlock, KERNEL};
use core::arch::asm;
use core::ptr::{read_volatile, write_volatile};
pub const FRAME_SIZE: usize = 128;
macro_rules! frame_off {
($($name:ident = $offset:expr),* $(,)?) => {
$( pub const $name: usize = $offset; )*
pub const FRAME_OFFSETS: &[usize] = &[$($offset),*];
};
}
frame_off! {
OFF_RA = 0,
OFF_S0 = 4,
OFF_S1 = 8,
OFF_S2 = 12,
OFF_S3 = 16,
OFF_S4 = 20,
OFF_S5 = 24,
OFF_S6 = 28,
OFF_S7 = 32,
OFF_S8 = 36,
OFF_S9 = 40,
OFF_S10 = 44,
OFF_S11 = 48,
OFF_A0 = 52,
OFF_A1 = 56,
OFF_A2 = 60,
OFF_A3 = 64,
OFF_A4 = 68,
OFF_A5 = 72,
OFF_A6 = 76,
OFF_A7 = 80,
OFF_T0 = 84,
OFF_T1 = 88,
OFF_T2 = 92,
OFF_T3 = 96,
OFF_T4 = 100,
OFF_T5 = 104,
OFF_T6 = 108,
OFF_MEPC = 112,
OFF_MSTATUS = 116,
}
const MSTATUS_TASK_ENTRY: usize = (3 << 11) | (1 << 7);
const MSTATUS_MIE: usize = 1 << 3;
const MIE_MTIE: usize = 1 << 7;
pub const MCAUSE_TIMER: usize = 0x8000_0007;
pub const MCAUSE_SOFT: usize = 0x8000_0003;
pub const DEFAULT_CLINT_BASE: usize = 0x0200_0000;
#[derive(Clone, Copy)]
pub struct TimerRegs {
pub mtime: *mut u64,
pub mtimecmp: *mut u64,
pub msip: *mut u32,
}
unsafe impl Sync for TimerRegs {}
static mut TIMER_REGS: TimerRegs = TimerRegs {
mtime: (DEFAULT_CLINT_BASE + 0xBFF8) as *mut u64,
mtimecmp: (DEFAULT_CLINT_BASE + 0x4000) as *mut u64,
msip: DEFAULT_CLINT_BASE as *mut u32,
};
pub unsafe fn configure_timer(regs: TimerRegs) {
write_volatile(core::ptr::addr_of_mut!(TIMER_REGS), regs);
}
#[inline]
unsafe fn regs() -> TimerRegs {
read_volatile(core::ptr::addr_of!(TIMER_REGS))
}
#[inline]
pub unsafe fn mtime_now() -> u64 {
let r = regs();
let p = r.mtime as *const u32;
loop {
let hi1 = read_volatile(p.add(1)) as u64;
let lo = read_volatile(p) as u64;
let hi2 = read_volatile(p.add(1)) as u64;
if hi1 == hi2 {
return (hi2 << 32) | lo;
}
}
}
unsafe fn mtimecmp_write(value: u64) {
let r = regs();
let p = r.mtimecmp as *mut u32;
asm!("csrc mie, {}", in(reg) MIE_MTIE, options(nomem, nostack, preserves_flags));
write_volatile(p.add(1), 0xFFFF_FFFF);
write_volatile(p, value as u32);
write_volatile(p.add(1), (value >> 32) as u32);
asm!("csrs mie, {}", in(reg) MIE_MTIE, options(nomem, nostack, preserves_flags));
}
#[inline]
unsafe fn rearm_timer() {
let slice = (*KERNEL.config.get()).slice_cycles as u64;
mtimecmp_write(mtime_now().wrapping_add(slice));
}
#[inline]
pub fn request_switch() {
unsafe {
let r = regs();
write_volatile(r.msip, 1);
}
}
pub type CriticalToken = usize;
pub unsafe fn critical_enter() -> CriticalToken {
let old: usize;
asm!("csrr {}, mstatus", out(reg) old, options(nomem, nostack, preserves_flags));
asm!("csrci mstatus, 8", options(nomem, nostack, preserves_flags)); old & MSTATUS_MIE
}
pub unsafe fn critical_exit(token: CriticalToken) {
if token != 0 {
asm!("csrsi mstatus, 8", options(nomem, nostack, preserves_flags));
}
}
static LAST_TICK_LOW: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0);
#[derive(Clone, Copy)]
pub struct TrapEvent {
pub mcause: usize,
pub mepc: usize,
pub current: *mut TaskControlBlock,
pub next: *mut TaskControlBlock,
pub after_switch: bool,
}
static TRAP_HOOK: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0);
pub fn set_trap_hook(hook: fn(TrapEvent)) {
TRAP_HOOK.store(
hook as *const () as usize,
core::sync::atomic::Ordering::Relaxed,
);
}
pub fn clear_trap_hook() {
TRAP_HOOK.store(0, core::sync::atomic::Ordering::Relaxed);
}
#[inline]
fn report_trap(ev: TrapEvent) {
let raw = TRAP_HOOK.load(core::sync::atomic::Ordering::Relaxed);
if raw != 0 {
let hook: fn(TrapEvent) = unsafe { core::mem::transmute::<usize, fn(TrapEvent)>(raw) };
hook(ev);
}
}
#[inline]
pub fn current_mepc() -> usize {
let v: usize;
unsafe {
asm!("csrr {}, mepc", out(reg) v, options(nomem, nostack, preserves_flags));
}
v
}
#[no_mangle]
unsafe extern "C" fn rrkernel_trap_switch(mcause: usize) -> *mut TaskControlBlock {
let mepc = current_mepc();
if mcause == MCAUSE_SOFT {
let r = regs();
write_volatile(r.msip, 0);
} else if mcause == MCAUSE_TIMER {
crate::scheduler::on_tick();
if (*KERNEL.config.get()).measure != crate::tcb::Measure::Off {
let now = mtime_now() as usize;
let last = LAST_TICK_LOW.swap(now, core::sync::atomic::Ordering::Relaxed);
if last != 0 {
let period = now.wrapping_sub(last) as u64;
let want = (*KERNEL.config.get()).slice_cycles as u64;
let err = period.abs_diff(want);
crate::scheduler::record_period_error(err.min(u32::MAX as u64) as u32);
}
}
}
let next = crate::scheduler::schedule_next();
report_trap(TrapEvent {
mcause,
mepc,
current: KERNEL.current(),
next,
after_switch: false,
});
next
}
#[no_mangle]
unsafe extern "C" fn rrkernel_after_switch() {
let measuring = (*KERNEL.config.get()).measure != crate::tcb::Measure::Off;
let t0 = if measuring { mtime_now() } else { 0 };
report_trap(TrapEvent {
mcause: 0,
mepc: 0,
current: KERNEL.current(),
next: KERNEL.current(),
after_switch: true,
});
crate::scheduler::reclaim_finished_tasks();
rearm_timer();
if measuring {
let dt = mtime_now().saturating_sub(t0);
crate::scheduler::record_latency(dt.min(u32::MAX as u64) as u32);
}
}
#[no_mangle]
#[unsafe(naked)]
pub unsafe extern "C" fn trap_entry() {
core::arch::naked_asm!(
"addi sp, sp, -{frame}",
"sw ra, {off_ra}(sp)",
"sw s0, {off_s0}(sp)",
"sw s1, {off_s1}(sp)",
"sw s2, {off_s2}(sp)",
"sw s3, {off_s3}(sp)",
"sw s4, {off_s4}(sp)",
"sw s5, {off_s5}(sp)",
"sw s6, {off_s6}(sp)",
"sw s7, {off_s7}(sp)",
"sw s8, {off_s8}(sp)",
"sw s9, {off_s9}(sp)",
"sw s10, {off_s10}(sp)",
"sw s11, {off_s11}(sp)",
"sw a0, {off_a0}(sp)",
"sw a1, {off_a1}(sp)",
"sw a2, {off_a2}(sp)",
"sw a3, {off_a3}(sp)",
"sw a4, {off_a4}(sp)",
"sw a5, {off_a5}(sp)",
"sw a6, {off_a6}(sp)",
"sw a7, {off_a7}(sp)",
"sw t0, {off_t0}(sp)",
"sw t1, {off_t1}(sp)",
"sw t2, {off_t2}(sp)",
"sw t3, {off_t3}(sp)",
"sw t4, {off_t4}(sp)",
"sw t5, {off_t5}(sp)",
"sw t6, {off_t6}(sp)",
"csrr t0, mepc",
"sw t0, {off_mepc}(sp)",
"csrr t0, mstatus",
"sw t0, {off_mstatus}(sp)",
"la t0, {kernel}",
"lw t1, 0(t0)",
"beqz t1, 1f",
"sw sp, 0(t1)",
"1:",
"csrr a0, mcause",
"call {trap_switch}",
"beqz a0, 4f",
"lw sp, 0(a0)",
"call {after_switch}",
"lw t0, {off_mepc}(sp)",
"csrw mepc, t0",
"lw t0, {off_mstatus}(sp)",
"csrw mstatus, t0",
"lw ra, {off_ra}(sp)",
"lw s0, {off_s0}(sp)",
"lw s1, {off_s1}(sp)",
"lw s2, {off_s2}(sp)",
"lw s3, {off_s3}(sp)",
"lw s4, {off_s4}(sp)",
"lw s5, {off_s5}(sp)",
"lw s6, {off_s6}(sp)",
"lw s7, {off_s7}(sp)",
"lw s8, {off_s8}(sp)",
"lw s9, {off_s9}(sp)",
"lw s10, {off_s10}(sp)",
"lw s11, {off_s11}(sp)",
"lw a0, {off_a0}(sp)",
"lw a1, {off_a1}(sp)",
"lw a2, {off_a2}(sp)",
"lw a3, {off_a3}(sp)",
"lw a4, {off_a4}(sp)",
"lw a5, {off_a5}(sp)",
"lw a6, {off_a6}(sp)",
"lw a7, {off_a7}(sp)",
"lw t0, {off_t0}(sp)",
"lw t1, {off_t1}(sp)",
"lw t2, {off_t2}(sp)",
"lw t3, {off_t3}(sp)",
"lw t4, {off_t4}(sp)",
"lw t5, {off_t5}(sp)",
"lw t6, {off_t6}(sp)",
"addi sp, sp, {frame}",
"mret",
"4:",
"wfi",
"j 1b",
frame = const FRAME_SIZE,
off_ra = const OFF_RA,
off_s0 = const OFF_S0,
off_s1 = const OFF_S1,
off_s2 = const OFF_S2,
off_s3 = const OFF_S3,
off_s4 = const OFF_S4,
off_s5 = const OFF_S5,
off_s6 = const OFF_S6,
off_s7 = const OFF_S7,
off_s8 = const OFF_S8,
off_s9 = const OFF_S9,
off_s10 = const OFF_S10,
off_s11 = const OFF_S11,
off_a0 = const OFF_A0,
off_a1 = const OFF_A1,
off_a2 = const OFF_A2,
off_a3 = const OFF_A3,
off_a4 = const OFF_A4,
off_a5 = const OFF_A5,
off_a6 = const OFF_A6,
off_a7 = const OFF_A7,
off_t0 = const OFF_T0,
off_t1 = const OFF_T1,
off_t2 = const OFF_T2,
off_t3 = const OFF_T3,
off_t4 = const OFF_T4,
off_t5 = const OFF_T5,
off_t6 = const OFF_T6,
off_mepc = const OFF_MEPC,
off_mstatus = const OFF_MSTATUS,
kernel = sym KERNEL,
trap_switch = sym rrkernel_trap_switch,
after_switch = sym rrkernel_after_switch,
);
}
const MIE_MSIE: usize = 1 << 3;
const MIN_SLICE_CYCLES: u64 = 100;
const NOMINAL_MTIME_HZ: u32 = 10_000_000;
fn configured_hz() -> u32 {
unsafe { *KERNEL.config.get() }.timer_hz
}
fn cycles_to_ns_with(hz: u32, cycles: u32) -> u64 {
if hz == 0 {
return 0;
}
((cycles as u128) * 1_000_000_000u128 / hz as u128) as u64
}
pub fn cycles_to_ns(cycles: u32) -> u64 {
let hz = configured_hz();
cycles_to_ns_with(if hz == 0 { NOMINAL_MTIME_HZ } else { hz }, cycles)
}
pub fn platform_limits() -> PlatformLimits {
let hz = configured_hz();
let hz = if hz == 0 { NOMINAL_MTIME_HZ } else { hz };
PlatformLimits {
min_slice_ns: cycles_to_ns_with(hz, MIN_SLICE_CYCLES as u32),
max_slice_ns: Some(cycles_to_ns_with(hz, u32::MAX)),
timer_hz: hz,
timer_note: "RISC-V machine timer (CLINT mtime/mtimecmp), trap-based switch",
}
}
pub fn plan_timer(cfg: &SchedulerConfig) -> Result<crate::arch::TimerPlan, ConfigError> {
let hz = cfg.timer_hz;
if hz == 0 {
return Err(ConfigError::TimerClockRequired);
}
let cycles = match cfg.slice {
Slice::Cycles(c) => c as u64,
other => other.to_cycles(hz),
};
if cycles == 0 {
return Err(ConfigError::ZeroSlice);
}
if cycles < MIN_SLICE_CYCLES {
return Err(ConfigError::SliceBelowPlatformMinimum {
requested_ns: cycles_to_ns_with(hz, cycles as u32),
minimum_ns: cycles_to_ns_with(hz, MIN_SLICE_CYCLES as u32),
});
}
if cycles > u32::MAX as u64 {
return Err(ConfigError::SliceAboveTimerRange {
requested_ns: cycles_to_ns_with(hz, u32::MAX),
maximum_ns: cycles_to_ns_with(hz, u32::MAX),
});
}
Ok(crate::arch::TimerPlan {
slice_ns: cycles_to_ns_with(hz, cycles as u32),
slice_cycles: cycles as u32,
timer_hz: hz,
})
}
pub fn start_timer(_plan: crate::arch::TimerPlan) -> Result<(), ConfigError> {
unsafe {
let handler = trap_entry as *const () as usize;
asm!("csrw mtvec, {}", in(reg) handler, options(nomem, nostack));
let r = regs();
write_volatile(r.msip, 0);
asm!(
"csrs mie, {}",
in(reg) MIE_MTIE | MIE_MSIE,
options(nomem, nostack)
);
rearm_timer();
asm!("csrsi mstatus, 8", options(nomem, nostack));
}
Ok(())
}
pub fn retune_timer(_plan: crate::arch::TimerPlan) -> Result<(), ConfigError> {
unsafe {
rearm_timer();
}
Ok(())
}
pub unsafe fn adopt_current_task(tcb: *mut TaskControlBlock) -> Result<(), ConfigError> {
unsafe {
(*tcb).sp = core::ptr::null_mut();
(*tcb).stack_base = core::ptr::null_mut();
(*tcb).stack_size = 0;
(*tcb).flags = 0;
}
Ok(())
}
pub unsafe fn create_task(
tcb: *mut TaskControlBlock,
stack_size: usize,
) -> Result<(), crate::SpawnError> {
let size = stack_size.max(256) + 16;
let block = {
let guard = crate::critical::enter();
let p = unsafe { (*crate::scheduler::arena()).alloc(size, 16) };
drop(guard);
p
};
let block = match block {
Some(p) => p,
None => return Err(crate::SpawnError::ArenaExhausted),
};
unsafe {
(*tcb).stack_base = block;
(*tcb).stack_size = size - 16;
let top = (block as usize + size) & !15;
let frame = (top - FRAME_SIZE) as *mut u8;
core::ptr::write_bytes(frame, 0, FRAME_SIZE);
let w = |off: usize, v: usize| {
core::ptr::write_volatile(frame.add(off) as *mut usize, v);
};
w(OFF_A0, tcb as usize);
w(
OFF_MEPC,
crate::trampoline::task_trampoline as *const () as usize,
);
w(OFF_MSTATUS, MSTATUS_TASK_ENTRY);
(*tcb).sp = frame;
(*tcb).flags = 0;
}
Ok(())
}
pub fn mark_running() {}
pub fn exit_current_task_forever() -> ! {
idle_forever()
}
pub fn idle_forever() -> ! {
loop {
unsafe {
asm!("wfi", options(nomem, nostack, preserves_flags));
}
}
}
pub fn shutdown(_code: i32) -> ! {
unsafe {
let _ = critical_enter();
}
loop {
core::hint::spin_loop();
}
}
pub unsafe fn on_task_reclaimed(_tcb: *mut TaskControlBlock) {}
pub unsafe fn is_pinned(_tcb: *mut TaskControlBlock) -> bool {
false
}
pub fn parks_synchronously() -> bool {
true
}
pub fn cycle_counter() -> Option<u32> {
let c: u32;
unsafe { core::arch::asm!("rdcycle {}", out(reg) c, options(nomem, nostack)) };
Some(c)
}
pub fn cycle_counter_hz() -> u32 {
let hz = crate::scheduler::config().timer_hz;
if hz == 0 {
1
} else {
hz
}
}