use crate::config::{ConfigError, PlatformLimits, SchedulerConfig, Slice};
use crate::tcb::{TaskControlBlock, KERNEL};
use core::arch::asm;
use core::ptr::{read_volatile, write_volatile};
pub const FRAME_SIZE: usize = 128;
macro_rules! frame_off {
($($name:ident = $offset:expr),* $(,)?) => {
$( pub const $name: usize = $offset; )*
pub const FRAME_OFFSETS: &[usize] = &[$($offset),*];
};
}
frame_off! {
OFF_PC = 0,
OFF_PS = 4,
OFF_A0 = 8,
OFF_A1 = 12,
OFF_A2 = 16,
OFF_A3 = 20,
OFF_A4 = 24,
OFF_A5 = 28,
OFF_A6 = 32,
OFF_A7 = 36,
OFF_A8 = 40,
OFF_A9 = 44,
OFF_A10 = 48,
OFF_A11 = 52,
OFF_A12 = 56,
OFF_A13 = 60,
OFF_A14 = 64,
OFF_A15 = 68,
OFF_SAR = 72,
OFF_EXCCAUSE = 76,
OFF_EXCVADDR = 80,
OFF_LBEG = 84,
OFF_LEND = 88,
OFF_LCOUNT = 92,
OFF_THREADPTR = 96,
OFF_SCOMPARE1 = 100,
}
pub const VECBASE: usize = 0x4008_0000;
pub const VEC_WINDOW_OVERFLOW4: usize = VECBASE + 0x000;
pub const VEC_WINDOW_UNDERFLOW4: usize = VECBASE + 0x040;
pub const VEC_WINDOW_OVERFLOW8: usize = VECBASE + 0x080;
pub const VEC_WINDOW_UNDERFLOW8: usize = VECBASE + 0x0C0;
pub const VEC_WINDOW_OVERFLOW12: usize = VECBASE + 0x100;
pub const VEC_WINDOW_UNDERFLOW12: usize = VECBASE + 0x140;
pub const VEC_KERNEL: usize = VECBASE + 0x300;
pub const VEC_DOUBLE: usize = VECBASE + 0x3C0;
pub const PS_INTLEVEL_MASK: u32 = 0x0000_000F;
pub const PS_EXCM: u32 = 0x0000_0010;
pub const PS_UM: u32 = 0x0000_0020;
pub const PS_OWB_SHIFT: u32 = 8;
pub const PS_WOE: u32 = 0x0004_0000;
pub const PS_TASK_ENTRY: u32 = PS_WOE;
pub const EXCCAUSE_LEVEL1_INTERRUPT: u32 = 4;
pub const INT_TIMER0: u32 = 6;
pub const INT_SW0: u32 = 0;
pub const USE_SOFTWARE_KICK: bool = true;
pub const EXCCAUSE_ALLOCA: u32 = 5;
pub const EXCCAUSE_OVERFLOW4: u32 = 12;
pub const EXCCAUSE_OVERFLOW8: u32 = 13;
pub const EXCCAUSE_OVERFLOW12: u32 = 14;
pub const EXCCAUSE_UNDERFLOW4: u32 = 16;
pub const EXCCAUSE_UNDERFLOW8: u32 = 17;
pub const EXCCAUSE_UNDERFLOW12: u32 = 18;
pub const NOMINAL_CCOUNT_HZ: u32 = 240_000_000;
const MIN_SLICE_CYCLES: u64 = 300;
const DPORT_CPU_INTR_FROM_CPU_0: usize = 0x3FF0_0000;
macro_rules! rsr {
($name:ident, $reg:literal) => {
#[inline]
pub fn $name() -> u32 {
let v: u32;
unsafe {
asm!(concat!("rsr {0}, ", $reg), out(reg) v, options(nomem, nostack));
}
v
}
};
}
rsr!(read_ccount, "CCOUNT");
rsr!(read_sar, "SAR");
rsr!(read_exccause, "EXCCAUSE");
rsr!(read_epc1, "EPC1");
rsr!(read_ps, "PS");
rsr!(read_interrupt, "INTERRUPT");
rsr!(read_intenable, "INTENABLE");
rsr!(read_vecbase, "VECBASE");
rsr!(read_prid, "PRID");
#[inline]
pub fn core_id() -> u32 {
read_prid()
}
fn configured_hz() -> u32 {
unsafe { *KERNEL.config.get() }.timer_hz
}
fn cycles_to_ns_with(hz: u32, cycles: u32) -> u64 {
if hz == 0 {
return 0;
}
((cycles as u128) * 1_000_000_000u128 / hz as u128) as u64
}
pub fn cycles_to_ns(cycles: u32) -> u64 {
let hz = configured_hz();
cycles_to_ns_with(if hz == 0 { NOMINAL_CCOUNT_HZ } else { hz }, cycles)
}
#[inline]
unsafe fn ccompare0_write(value: u32) {
asm!("wsr {0}, CCOMPARE0", in(reg) value, options(nomem, nostack));
}
#[inline]
unsafe fn enable_kernel_interrupts() {
let mut mask = 1u32 << INT_TIMER0;
if USE_SOFTWARE_KICK {
mask |= 1u32 << INT_SW0;
}
asm!("wsr {0}, INTENABLE", in(reg) mask, options(nomem, nostack));
}
#[inline]
unsafe fn rearm_timer() {
let slice = (*KERNEL.config.get()).slice_cycles;
ccompare0_write(read_ccount().wrapping_add(slice));
}
#[inline]
pub fn request_switch() {
if !USE_SOFTWARE_KICK {
return;
}
unsafe {
write_volatile(DPORT_CPU_INTR_FROM_CPU_0 as *mut u32, 1);
}
}
#[inline]
unsafe fn clear_software_interrupt() {
asm!("wsr {0}, INTCLEAR", in(reg) 1u32 << INT_SW0, options(nomem, nostack));
}
pub type CriticalToken = usize;
#[inline]
pub unsafe fn critical_enter() -> CriticalToken {
let ps = read_ps();
let old_level = ps & PS_INTLEVEL_MASK;
if old_level < 1 {
asm!(
"wsr {0}, PS",
"rsync",
in(reg) (ps & !PS_INTLEVEL_MASK) | 1,
options(nomem, nostack, preserves_flags)
);
}
old_level as CriticalToken
}
#[inline]
pub unsafe fn critical_exit(token: CriticalToken) {
let cur: u32;
asm!("rsr {0}, PS", out(reg) cur, options(nomem, nostack, preserves_flags));
let new = (cur & !PS_INTLEVEL_MASK) | (token as u32 & PS_INTLEVEL_MASK);
if new != cur {
asm!(
"wsr {0}, PS",
"rsync",
in(reg) new,
options(nomem, nostack, preserves_flags)
);
}
}
#[derive(Clone, Copy)]
pub struct TrapEvent {
pub exccause: u32,
pub epc: u32,
pub current: *mut TaskControlBlock,
pub next: *mut TaskControlBlock,
pub after_switch: bool,
}
static TRAP_HOOK: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0);
pub fn set_trap_hook(hook: fn(TrapEvent)) {
TRAP_HOOK.store(
hook as *const () as usize,
core::sync::atomic::Ordering::Relaxed,
);
}
pub fn clear_trap_hook() {
TRAP_HOOK.store(0, core::sync::atomic::Ordering::Relaxed);
}
#[inline]
fn report_trap(ev: TrapEvent) {
let raw = TRAP_HOOK.load(core::sync::atomic::Ordering::Relaxed);
if raw != 0 {
let hook: fn(TrapEvent) = unsafe { core::mem::transmute::<usize, fn(TrapEvent)>(raw) };
hook(ev);
}
}
#[inline]
pub fn current_epc() -> u32 {
read_epc1()
}
#[inline]
pub fn current_vecbase() -> u32 {
read_vecbase()
}
const UART0_FIFO: usize = 0x3FF4_0000;
#[inline]
unsafe fn diag_putc(c: u8) {
write_volatile(UART0_FIFO as *mut u32, c as u32);
}
unsafe fn diag_hex(v: u32) {
diag_putc(b'0');
diag_putc(b'x');
let mut shift = 28i32;
loop {
let nib = ((v >> shift) & 0xF) as u8;
diag_putc(if nib < 10 {
b'0' + nib
} else {
b'a' + nib - 10
});
if shift == 0 {
break;
}
shift -= 4;
}
}
unsafe fn diag_field(label: &str, v: u32) {
for b in label.bytes() {
diag_putc(b);
}
diag_hex(v);
diag_putc(b'\r');
diag_putc(b'\n');
}
pub fn diag_str(s: &str) {
for b in s.bytes() {
unsafe { diag_putc(b) };
}
}
pub fn diag_num(v: u32) {
if v == 0 {
unsafe { diag_putc(b'0') };
return;
}
let mut buf = [0u8; 10];
let mut n = v;
let mut i = buf.len();
while n > 0 {
i -= 1;
buf[i] = b'0' + (n % 10) as u8;
n /= 10;
}
for b in &buf[i..] {
unsafe { diag_putc(*b) };
}
}
#[inline]
pub fn diag_marker(marker: u8) {
if TRACE_SWITCH {
unsafe { diag_putc(marker) };
}
}
#[no_mangle]
unsafe extern "C" fn rrkernel_trace(marker: u32) {
diag_putc(marker as u8);
}
pub const TRACE_SWITCH: bool = false;
#[no_mangle]
unsafe extern "C" fn rrkernel_slot_trap(slot: u32) -> ! {
static REPORTED: SingleWriterU32 = SingleWriterU32::new(0);
if REPORTED.replace(1) == 0 {
let name = match slot {
0 => "window-overflow4",
1 => "window-underflow4",
2 => "window-overflow8",
3 => "window-underflow8",
4 => "window-overflow12",
5 => "window-underflow12",
6 => "level2",
7 => "level3",
8 => "level4",
9 => "level5",
10 => "level6/debug",
11 => "level7/nmi",
12 => "kernel",
13 => "user-exception",
14 => "double-exception",
_ => "unknown",
};
diag_putc(b'\r');
diag_putc(b'\n');
diag_field("[trap] slot=", slot);
for b in name.bytes() {
diag_putc(b);
}
diag_putc(b'\r');
diag_putc(b'\n');
let mut exccause: u32;
asm!("rsr {0}, EXCCAUSE", out(reg) exccause, options(nomem, nostack));
diag_field("[trap] cause=", exccause);
let mut v: u32;
asm!("rsr {0}, EXCVADDR", out(reg) v, options(nomem, nostack));
diag_field("[trap] vaddr=", v);
asm!("rsr {0}, EPC1", out(reg) v, options(nomem, nostack));
diag_field("[trap] epc1 =", v);
asm!("rsr {0}, PS", out(reg) v, options(nomem, nostack));
diag_field("[trap] ps =", v);
asm!("rsr {0}, EXCSAVE1", out(reg) v, options(nomem, nostack));
diag_field("[trap] a0 =", v);
#[cfg(target_feature = "windowed")]
{
asm!("rsr {0}, WINDOWBASE", out(reg) v, options(nomem, nostack));
diag_field("[trap] wbase=", v);
asm!("rsr {0}, WINDOWSTART", out(reg) v, options(nomem, nostack));
diag_field("[trap] wstart=", v);
}
asm!("rsr {0}, INTENABLE", out(reg) v, options(nomem, nostack));
diag_field("[trap] inten=", v);
asm!("rsr {0}, INTERRUPT", out(reg) v, options(nomem, nostack));
diag_field("[trap] intset=", v);
}
loop {
core::hint::spin_loop();
}
}
macro_rules! trap_slot {
($name:ident, $section:literal, $slot:expr, $label:literal, $jump:literal, $doc:literal) => {
#[doc = $doc]
#[no_mangle]
#[unsafe(naked)]
#[link_section = $section]
pub unsafe extern "C" fn $name() {
core::arch::naked_asm!(
"wsr a0, EXCSAVE1",
"movi a2, {slot}",
"call0 {report}",
$label,
$jump,
slot = const $slot,
report = sym rrkernel_slot_trap,
);
}
};
}
trap_slot!(
xtensa_level2_vector,
".Level2Vector.text",
6,
".Lxt_park2:",
"j .Lxt_park2",
"Level-2 interrupt vector (unused: reports and parks)."
);
trap_slot!(
xtensa_level3_vector,
".Level3Vector.text",
7,
".Lxt_park3:",
"j .Lxt_park3",
"Level-3 interrupt vector (unused: reports and parks)."
);
trap_slot!(
xtensa_level4_vector,
".Level4Vector.text",
8,
".Lxt_park4:",
"j .Lxt_park4",
"Level-4 interrupt vector (unused: reports and parks)."
);
trap_slot!(
xtensa_level5_vector,
".Level5Vector.text",
9,
".Lxt_park5:",
"j .Lxt_park5",
"Level-5 interrupt vector (unused: reports and parks)."
);
trap_slot!(
xtensa_level6_vector,
".Level6Vector.text",
10,
".Lxt_park6:",
"j .Lxt_park6",
"Level-6 (debug) interrupt vector: reports and parks."
);
trap_slot!(
xtensa_level7_vector,
".Level7Vector.text",
11,
".Lxt_park7:",
"j .Lxt_park7",
"Level-7 (NMI) interrupt vector: reports and parks."
);
trap_slot!(
xtensa_user_exception_vector,
".UserExceptionVector.text",
13,
".Lxt_parku:",
"j .Lxt_parku",
"User exception vector (unused: this kernel stays in kernel mode)."
);
trap_slot!(
xtensa_double_exception_vector,
".DoubleExceptionVector.text",
14,
".Lxt_parkd:",
"j .Lxt_parkd",
"Double exception vector: reports and parks."
);
#[no_mangle]
#[unsafe(naked)]
#[link_section = ".KernelExceptionVector.text"]
pub unsafe extern "C" fn xtensa_kernel_vector() {
core::arch::naked_asm!(
"wsr a0, EXCSAVE1",
"j {handler}",
handler = sym xtensa_exception_handler,
);
}
#[no_mangle]
#[unsafe(naked)]
pub unsafe extern "C" fn xtensa_exception_handler() {
core::arch::naked_asm!(
"rsr a2, EXCCAUSE",
"movi a3, {exc_alloca}",
"beq a2, a3, .Lxt_alloca",
"j .Lxt_frame",
".Lxt_alloca:",
"j {alloca}", ".Lxt_frame:",
"mov a0, sp", "addi sp, sp, -{frame}", "s32i a0, sp, {off_a1}", "rsr a0, EXCSAVE1", "s32i a0, sp, {off_a0}",
"s32i a2, sp, {off_a2}",
"s32i a3, sp, {off_a3}",
"s32i a4, sp, {off_a4}",
"s32i a5, sp, {off_a5}",
"s32i a6, sp, {off_a6}",
"s32i a7, sp, {off_a7}",
"s32i a8, sp, {off_a8}",
"s32i a9, sp, {off_a9}",
"s32i a10, sp, {off_a10}",
"s32i a11, sp, {off_a11}",
"s32i a12, sp, {off_a12}",
"s32i a13, sp, {off_a13}",
"s32i a14, sp, {off_a14}",
"s32i a15, sp, {off_a15}",
"rsr a0, EPC1",
"s32i a0, sp, {off_pc}",
"rsr a0, PS",
"movi a2, {ps_excm_mask}",
"and a0, a0, a2",
"s32i a0, sp, {off_ps}",
"rsr a0, SAR",
"s32i a0, sp, {off_sar}",
"rsr a0, EXCCAUSE",
"s32i a0, sp, {off_exccause}",
"rsr a0, EXCVADDR",
"s32i a0, sp, {off_excvaddr}",
"rsr a0, LBEG",
"s32i a0, sp, {off_lbeg}",
"rsr a0, LEND",
"s32i a0, sp, {off_lend}",
"rsr a0, LCOUNT",
"s32i a0, sp, {off_lcount}",
"rur a0, THREADPTR",
"s32i a0, sp, {off_threadptr}",
"rsr a0, SCOMPARE1",
"s32i a0, sp, {off_scompare1}",
"l32i a2, sp, {off_exccause}",
"movi a3, {exc_level1}",
"beq a2, a3, .Lxt_level1",
"movi a0, {ps_kernel}",
"wsr a0, PS",
"rsync",
"l32i a2, sp, {off_exccause}",
"l32i a3, sp, {off_pc}",
"call0 {unexpected}",
".Lxt_park:",
"j .Lxt_park",
".Lxt_level1:",
"movi a0, {ps_kernel}",
"wsr a0, PS",
"rsync",
"movi a2, 69", "call0 {trace}",
"mov a2, sp",
"call0 {trap_switch}",
"beqz a2, .Lxt_noswitch", "l32i sp, a2, 0", "movi a2, 83", "call0 {trace}",
".Lxt_noswitch:",
"call0 {after_switch}",
"movi a2, 82", "call0 {trace}",
"l32i a2, sp, {off_sar}",
"wsr a2, SAR",
"l32i a2, sp, {off_lbeg}",
"wsr a2, LBEG",
"l32i a2, sp, {off_lend}",
"wsr a2, LEND",
"l32i a2, sp, {off_lcount}",
"wsr a2, LCOUNT",
"l32i a2, sp, {off_threadptr}",
"wur a2, THREADPTR",
"l32i a2, sp, {off_scompare1}",
"wsr a2, SCOMPARE1",
"l32i a0, sp, {off_ps}",
"wsr a0, PS", "rsync",
"l32i a0, sp, {off_pc}",
"wsr a0, EPC1",
"l32i a2, sp, {off_a2}",
"l32i a3, sp, {off_a3}",
"l32i a4, sp, {off_a4}",
"l32i a5, sp, {off_a5}",
"l32i a6, sp, {off_a6}",
"l32i a7, sp, {off_a7}",
"l32i a8, sp, {off_a8}",
"l32i a9, sp, {off_a9}",
"l32i a10, sp, {off_a10}",
"l32i a11, sp, {off_a11}",
"l32i a12, sp, {off_a12}",
"l32i a13, sp, {off_a13}",
"l32i a14, sp, {off_a14}",
"l32i a15, sp, {off_a15}",
"l32i a0, sp, {off_a0}",
"l32i sp, sp, {off_a1}",
"rsync",
"rfi 1",
frame = const FRAME_SIZE,
exc_level1 = const EXCCAUSE_LEVEL1_INTERRUPT,
exc_alloca = const EXCCAUSE_ALLOCA,
ps_kernel = const (3 | PS_WOE),
ps_excm_mask = const !PS_EXCM,
alloca = sym xtensa_alloca_exception,
off_pc = const OFF_PC,
off_ps = const OFF_PS,
off_a0 = const OFF_A0,
off_a1 = const OFF_A1,
off_a2 = const OFF_A2,
off_a3 = const OFF_A3,
off_a4 = const OFF_A4,
off_a5 = const OFF_A5,
off_a6 = const OFF_A6,
off_a7 = const OFF_A7,
off_a8 = const OFF_A8,
off_a9 = const OFF_A9,
off_a10 = const OFF_A10,
off_a11 = const OFF_A11,
off_a12 = const OFF_A12,
off_a13 = const OFF_A13,
off_a14 = const OFF_A14,
off_a15 = const OFF_A15,
off_sar = const OFF_SAR,
off_exccause = const OFF_EXCCAUSE,
off_excvaddr = const OFF_EXCVADDR,
off_lbeg = const OFF_LBEG,
off_lend = const OFF_LEND,
off_lcount = const OFF_LCOUNT,
off_threadptr = const OFF_THREADPTR,
off_scompare1 = const OFF_SCOMPARE1,
trap_switch = sym rrkernel_trap_switch,
after_switch = sym rrkernel_after_switch,
unexpected = sym rrkernel_unexpected_trap,
trace = sym rrkernel_trace,
);
}
#[cfg(target_feature = "windowed")]
#[no_mangle]
#[unsafe(naked)]
#[link_section = ".WindowOverflow4.text"]
pub unsafe extern "C" fn _WindowOverflow4() {
core::arch::naked_asm!(
"s32e a0, a5, -16",
"s32e a1, a5, -12",
"s32e a2, a5, -8",
"s32e a3, a5, -4",
"rfwo",
);
}
#[cfg(target_feature = "windowed")]
#[no_mangle]
#[unsafe(naked)]
#[link_section = ".WindowUnderflow4.text"]
pub unsafe extern "C" fn _WindowUnderflow4() {
core::arch::naked_asm!(
"l32e a0, a5, -16",
"l32e a1, a5, -12",
"l32e a2, a5, -8",
"l32e a3, a5, -4",
"rfwu",
);
}
#[cfg(target_feature = "windowed")]
#[no_mangle]
#[unsafe(naked)]
#[link_section = ".WindowUnderflow4.text"]
pub unsafe extern "C" fn xtensa_alloca_exception() {
core::arch::naked_asm!(
"rsr a0, WINDOWBASE", "rotw -1", "rsr a2, PS",
"extui a3, a2, 8, 4", "xor a3, a3, a4", "rsr a4, EXCSAVE1", "slli a3, a3, 8", "xor a2, a2, a3", "wsr a2, PS",
"rsync",
"bbci a4, 31, _WindowUnderflow4",
"rotw -1", "bbci a8, 30, _WindowUnderflow8",
"rotw -1",
"j _WindowUnderflow12",
);
}
#[cfg(not(target_feature = "windowed"))]
#[no_mangle]
#[unsafe(naked)]
pub unsafe extern "C" fn xtensa_alloca_exception() {
core::arch::naked_asm!(".Lxt_alloca_park:", "j .Lxt_alloca_park");
}
#[cfg(target_feature = "windowed")]
#[no_mangle]
#[unsafe(naked)]
#[link_section = ".WindowOverflow8.text"]
pub unsafe extern "C" fn _WindowOverflow8() {
core::arch::naked_asm!(
"s32e a0, a9, -16",
"l32e a0, a1, -12", "s32e a1, a9, -12",
"s32e a2, a9, -8",
"s32e a3, a9, -4",
"s32e a4, a0, -32",
"s32e a5, a0, -28",
"s32e a6, a0, -24",
"s32e a7, a0, -20",
"rfwo",
);
}
#[cfg(target_feature = "windowed")]
#[no_mangle]
#[unsafe(naked)]
#[link_section = ".WindowUnderflow8.text"]
pub unsafe extern "C" fn _WindowUnderflow8() {
core::arch::naked_asm!(
"l32e a0, a9, -16",
"l32e a1, a9, -12",
"l32e a2, a9, -8",
"l32e a7, a1, -12",
"l32e a3, a9, -4",
"l32e a4, a7, -32",
"l32e a5, a7, -28",
"l32e a6, a7, -24",
"l32e a7, a7, -20",
"rfwu",
);
}
#[cfg(target_feature = "windowed")]
#[no_mangle]
#[unsafe(naked)]
#[link_section = ".WindowOverflow12.text"]
pub unsafe extern "C" fn _WindowOverflow12() {
core::arch::naked_asm!(
"s32e a0, a13, -16",
"l32e a0, a1, -12",
"s32e a1, a13, -12",
"s32e a2, a13, -8",
"s32e a3, a13, -4",
"s32e a4, a0, -48",
"s32e a5, a0, -44",
"s32e a6, a0, -40",
"s32e a7, a0, -36",
"s32e a8, a0, -32",
"s32e a9, a0, -28",
"s32e a10, a0, -24",
"s32e a11, a0, -20",
"rfwo",
);
}
#[cfg(target_feature = "windowed")]
#[no_mangle]
#[unsafe(naked)]
#[link_section = ".WindowUnderflow12.text"]
pub unsafe extern "C" fn _WindowUnderflow12() {
core::arch::naked_asm!(
"l32e a0, a13, -16",
"l32e a1, a13, -12",
"l32e a2, a13, -8",
"l32e a11, a1, -12",
"l32e a3, a13, -4",
"l32e a4, a11, -48",
"l32e a5, a11, -44",
"l32e a6, a11, -40",
"l32e a7, a11, -36",
"l32e a8, a11, -32",
"l32e a9, a11, -28",
"l32e a10, a11, -24",
"l32e a11, a11, -20",
"rfwu",
);
}
static LAST_TICK_CC: SingleWriterU32 = SingleWriterU32::new(0);
struct SingleWriterU32(core::cell::UnsafeCell<u32>);
unsafe impl Sync for SingleWriterU32 {}
impl SingleWriterU32 {
const fn new(v: u32) -> Self {
SingleWriterU32(core::cell::UnsafeCell::new(v))
}
#[inline]
fn replace(&self, v: u32) -> u32 {
unsafe {
let old = core::ptr::read_volatile(self.0.get());
core::ptr::write_volatile(self.0.get(), v);
old
}
}
}
#[no_mangle]
unsafe extern "C" fn rrkernel_trap_switch(frame: *mut u8) -> *mut TaskControlBlock {
let cur = KERNEL.current();
if !cur.is_null() {
(*cur).sp = frame;
}
let exccause = read_volatile(frame.add(OFF_EXCCAUSE) as *const u32);
let epc = read_volatile(frame.add(OFF_PC) as *const u32);
diag_marker(b'T');
let pending = read_interrupt();
let mut ticked = false;
if pending & (1 << INT_TIMER0) != 0 {
ticked = true;
crate::scheduler::on_tick();
diag_marker(b'K');
if (*KERNEL.config.get()).measure != crate::tcb::Measure::Off {
let now = read_ccount();
let last = LAST_TICK_CC.replace(now);
if last != 0 {
let period = now.wrapping_sub(last);
let want = (*KERNEL.config.get()).slice_cycles;
crate::scheduler::record_period_error(period.abs_diff(want));
}
}
}
if pending & (1 << INT_SW0) != 0 {
write_volatile(DPORT_CPU_INTR_FROM_CPU_0 as *mut u32, 0);
clear_software_interrupt();
}
let next = crate::scheduler::schedule_next();
diag_marker(b'N');
if next.is_null() && ticked {
rearm_timer();
}
report_trap(TrapEvent {
exccause,
epc,
current: cur,
next,
after_switch: false,
});
next
}
#[no_mangle]
unsafe extern "C" fn rrkernel_after_switch() {
let measuring = (*KERNEL.config.get()).measure != crate::tcb::Measure::Off;
let t0 = if measuring { read_ccount() } else { 0 };
report_trap(TrapEvent {
exccause: 0,
epc: 0,
current: KERNEL.current(),
next: KERNEL.current(),
after_switch: true,
});
crate::scheduler::reclaim_finished_tasks();
rearm_timer();
if measuring {
let dt = read_ccount().wrapping_sub(t0);
crate::scheduler::record_latency(dt);
}
}
#[no_mangle]
unsafe extern "C" fn rrkernel_unexpected_trap(exccause: u32, epc: u32) -> ! {
report_trap(TrapEvent {
exccause,
epc,
current: KERNEL.current(),
next: core::ptr::null_mut(),
after_switch: false,
});
loop {
core::hint::spin_loop();
}
}
pub fn platform_limits() -> PlatformLimits {
let hz = configured_hz();
let hz = if hz == 0 { NOMINAL_CCOUNT_HZ } else { hz };
PlatformLimits {
min_slice_ns: cycles_to_ns_with(hz, MIN_SLICE_CYCLES as u32),
max_slice_ns: Some(cycles_to_ns_with(hz, u32::MAX)),
timer_hz: hz,
timer_note: "Xtensa CCOUNT/CCOMPARE0 (ESP32 LX6), windowed register file",
}
}
pub fn plan_timer(cfg: &SchedulerConfig) -> Result<crate::arch::TimerPlan, ConfigError> {
let hz = cfg.timer_hz;
if hz == 0 {
return Err(ConfigError::TimerClockRequired);
}
let cycles = match cfg.slice {
Slice::Cycles(c) => c as u64,
other => other.to_cycles(hz),
};
if cycles == 0 {
return Err(ConfigError::ZeroSlice);
}
if cycles < MIN_SLICE_CYCLES {
return Err(ConfigError::SliceBelowPlatformMinimum {
requested_ns: cycles_to_ns_with(hz, cycles as u32),
minimum_ns: cycles_to_ns_with(hz, MIN_SLICE_CYCLES as u32),
});
}
if cycles > u32::MAX as u64 {
return Err(ConfigError::SliceAboveTimerRange {
requested_ns: cycles_to_ns_with(hz, u32::MAX),
maximum_ns: cycles_to_ns_with(hz, u32::MAX),
});
}
Ok(crate::arch::TimerPlan {
slice_ns: cycles_to_ns_with(hz, cycles as u32),
slice_cycles: cycles as u32,
timer_hz: hz,
})
}
pub fn start_timer(_plan: crate::arch::TimerPlan) -> Result<(), ConfigError> {
unsafe {
asm!(
"wsr {0}, VECBASE",
"rsync",
in(reg) VECBASE,
options(nomem, nostack)
);
let vbase = read_vecbase();
if vbase as usize != VECBASE {
return Err(ConfigError::Platform(
"the core did not accept VECBASE = 0x4008_0000; the ESP32 port links \
its vector table at the bottom of IRAM",
));
}
asm!("wsr {0}, PS", "rsync", in(reg) PS_TASK_ENTRY, options(nomem, nostack));
write_volatile(DPORT_CPU_INTR_FROM_CPU_0 as *mut u32, 0);
clear_software_interrupt();
enable_kernel_interrupts();
rearm_timer();
}
Ok(())
}
pub fn retune_timer(_plan: crate::arch::TimerPlan) -> Result<(), ConfigError> {
unsafe {
rearm_timer();
}
Ok(())
}
pub unsafe fn adopt_current_task(tcb: *mut TaskControlBlock) -> Result<(), ConfigError> {
unsafe {
(*tcb).sp = core::ptr::null_mut();
(*tcb).stack_base = core::ptr::null_mut();
(*tcb).stack_size = 0;
(*tcb).flags = 0;
}
Ok(())
}
pub unsafe fn create_task(
tcb: *mut TaskControlBlock,
stack_size: usize,
) -> Result<(), crate::SpawnError> {
let size = stack_size.max(512) + FRAME_SIZE + 16;
let block = {
let guard = crate::critical::enter();
let p = unsafe { (*crate::scheduler::arena()).alloc(size, 16) };
drop(guard);
p
};
let block = match block {
Some(p) => p,
None => return Err(crate::SpawnError::ArenaExhausted),
};
unsafe {
(*tcb).stack_base = block;
(*tcb).stack_size = size - FRAME_SIZE - 16;
let top = (block as usize + size) & !15;
let frame = (top - FRAME_SIZE) as *mut u8;
core::ptr::write_bytes(frame, 0, FRAME_SIZE);
let w = |off: usize, v: usize| {
core::ptr::write_volatile(frame.add(off) as *mut usize, v);
};
w(
OFF_PC,
crate::trampoline::task_trampoline as *const () as usize,
);
w(OFF_PS, PS_TASK_ENTRY as usize);
w(OFF_A1, frame as usize);
w(OFF_A2, tcb as usize);
(*tcb).sp = frame;
(*tcb).flags = 0;
}
Ok(())
}
pub fn mark_running() {}
pub fn exit_current_task_forever() -> ! {
idle_forever()
}
pub fn idle_forever() -> ! {
loop {
unsafe {
asm!("waiti 0", options(nomem, nostack));
}
}
}
pub fn shutdown(_code: i32) -> ! {
unsafe {
asm!("wsr {0}, INTENABLE", in(reg) 0u32, options(nomem, nostack));
}
loop {
core::hint::spin_loop();
}
}
pub unsafe fn on_task_reclaimed(_tcb: *mut TaskControlBlock) {}
pub unsafe fn is_pinned(_tcb: *mut TaskControlBlock) -> bool {
false
}
pub fn parks_synchronously() -> bool {
true
}
pub fn cycle_counter() -> Option<u32> {
None
}
pub fn cycle_counter_hz() -> u32 {
0
}