pub use esp_riscv_rt::TrapFrame;
use riscv::register::{mcause, mtvec};
#[cfg(not(plic))]
pub use self::classic::*;
#[cfg(plic)]
pub use self::plic::*;
pub use self::vectored::*;
use super::InterruptStatus;
use crate::{
peripherals::{self, Interrupt},
Cpu,
};
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
pub enum Error {
InvalidInterruptPriority,
CpuInterruptReserved,
}
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
pub enum InterruptKind {
Level,
Edge,
}
#[repr(u32)]
#[derive(Debug, Copy, Clone)]
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
pub enum CpuInterrupt {
Interrupt1 = 1,
Interrupt2,
Interrupt3,
Interrupt4,
Interrupt5,
Interrupt6,
Interrupt7,
Interrupt8,
Interrupt9,
Interrupt10,
Interrupt11,
Interrupt12,
Interrupt13,
Interrupt14,
Interrupt15,
Interrupt16,
Interrupt17,
Interrupt18,
Interrupt19,
Interrupt20,
Interrupt21,
Interrupt22,
Interrupt23,
Interrupt24,
Interrupt25,
Interrupt26,
Interrupt27,
Interrupt28,
Interrupt29,
Interrupt30,
Interrupt31,
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
#[repr(u8)]
pub enum Priority {
None = 0,
Priority1,
Priority2,
Priority3,
Priority4,
Priority5,
Priority6,
Priority7,
Priority8,
Priority9,
Priority10,
Priority11,
Priority12,
Priority13,
Priority14,
Priority15,
}
impl Priority {
pub const fn max() -> Priority {
Priority::Priority15
}
pub const fn min() -> Priority {
Priority::Priority1
}
}
#[cfg_attr(place_switch_tables_in_ram, link_section = ".rwtext")]
pub static RESERVED_INTERRUPTS: &[usize] = INTERRUPT_TO_PRIORITY;
#[doc(hidden)]
#[link_section = ".trap.rust"]
#[export_name = "_start_trap_rust_hal"]
pub unsafe extern "C" fn start_trap_rust_hal(trap_frame: *mut TrapFrame) {
assert!(
mcause::read().is_exception(),
"Arrived into _start_trap_rust_hal but mcause is not an exception!"
);
extern "C" {
fn ExceptionHandler(tf: *mut TrapFrame);
}
ExceptionHandler(trap_frame);
}
#[doc(hidden)]
#[no_mangle]
pub fn _setup_interrupts() {
extern "C" {
static _vector_table: *const u32;
}
unsafe {
for peripheral_interrupt in 0..255 {
crate::soc::peripherals::Interrupt::try_from(peripheral_interrupt)
.map(|intr| {
#[cfg(multi_core)]
disable(Cpu::AppCpu, intr);
disable(Cpu::ProCpu, intr);
})
.ok();
}
let vec_table = &_vector_table as *const _ as usize;
mtvec::write(vec_table, mtvec::TrapMode::Vectored);
crate::interrupt::init_vectoring();
};
#[cfg(plic)]
unsafe {
core::arch::asm!("csrw mie, {0}", in(reg) u32::MAX);
}
}
pub fn enable_direct(
interrupt: Interrupt,
level: Priority,
cpu_interrupt: CpuInterrupt,
) -> Result<(), Error> {
if RESERVED_INTERRUPTS.contains(&(cpu_interrupt as _)) {
return Err(Error::CpuInterruptReserved);
}
if matches!(level, Priority::None) {
return Err(Error::InvalidInterruptPriority);
}
unsafe {
map(Cpu::current(), interrupt, cpu_interrupt);
set_priority(Cpu::current(), cpu_interrupt, level);
enable_cpu_interrupt(cpu_interrupt);
}
Ok(())
}
pub fn disable(_core: Cpu, interrupt: Interrupt) {
unsafe {
let interrupt_number = interrupt as isize;
let intr_map_base = crate::soc::registers::INTERRUPT_MAP_BASE as *mut u32;
intr_map_base
.offset(interrupt_number)
.write_volatile(DISABLED_CPU_INTERRUPT);
}
}
#[inline]
pub fn status(_core: Cpu) -> InterruptStatus {
#[cfg(large_intr_status)]
unsafe {
InterruptStatus::from(
(*crate::peripherals::INTERRUPT_CORE0::PTR)
.intr_status_reg_0()
.read()
.bits(),
(*crate::peripherals::INTERRUPT_CORE0::PTR)
.intr_status_reg_1()
.read()
.bits(),
(*crate::peripherals::INTERRUPT_CORE0::PTR)
.int_status_reg_2()
.read()
.bits(),
)
}
#[cfg(very_large_intr_status)]
unsafe {
InterruptStatus::from(
(*crate::peripherals::INTERRUPT_CORE0::PTR)
.intr_status_reg_0()
.read()
.bits(),
(*crate::peripherals::INTERRUPT_CORE0::PTR)
.intr_status_reg_1()
.read()
.bits(),
(*crate::peripherals::INTERRUPT_CORE0::PTR)
.intr_status_reg_2()
.read()
.bits(),
(*crate::peripherals::INTERRUPT_CORE0::PTR)
.intr_status_reg_3()
.read()
.bits(),
)
}
#[cfg(not(any(large_intr_status, very_large_intr_status)))]
unsafe {
InterruptStatus::from(
(*crate::peripherals::INTERRUPT_CORE0::PTR)
.intr_status_reg_0()
.read()
.bits(),
(*crate::peripherals::INTERRUPT_CORE0::PTR)
.intr_status_reg_1()
.read()
.bits(),
)
}
}
pub unsafe fn map(_core: Cpu, interrupt: Interrupt, which: CpuInterrupt) {
let interrupt_number = interrupt as isize;
let cpu_interrupt_number = which as isize;
#[cfg(not(multi_core))]
let intr_map_base = crate::soc::registers::INTERRUPT_MAP_BASE as *mut u32;
#[cfg(multi_core)]
let intr_map_base = match _core {
Cpu::ProCpu => crate::soc::registers::INTERRUPT_MAP_BASE as *mut u32,
Cpu::AppCpu => crate::soc::registers::INTERRUPT_MAP_BASE_APP_CPU as *mut u32,
};
intr_map_base
.offset(interrupt_number)
.write_volatile(cpu_interrupt_number as u32 + EXTERNAL_INTERRUPT_OFFSET);
}
#[inline]
unsafe fn assigned_cpu_interrupt(interrupt: Interrupt) -> Option<CpuInterrupt> {
let interrupt_number = interrupt as isize;
let intr_map_base = crate::soc::registers::INTERRUPT_MAP_BASE as *mut u32;
let cpu_intr = intr_map_base.offset(interrupt_number).read_volatile();
if cpu_intr > 0 && cpu_intr != DISABLED_CPU_INTERRUPT {
Some(core::mem::transmute::<u32, CpuInterrupt>(
cpu_intr - EXTERNAL_INTERRUPT_OFFSET,
))
} else {
None
}
}
pub(crate) fn bound_cpu_interrupt_for(_cpu: Cpu, interrupt: Interrupt) -> Option<CpuInterrupt> {
unsafe { assigned_cpu_interrupt(interrupt) }
}
mod vectored {
use procmacros::ram;
use super::*;
#[doc(hidden)]
pub(crate) unsafe fn init_vectoring() {
for (prio, num) in PRIORITY_TO_INTERRUPT.iter().enumerate() {
set_kind(
Cpu::current(),
core::mem::transmute::<u32, CpuInterrupt>(*num as u32),
InterruptKind::Level,
);
set_priority(
Cpu::current(),
core::mem::transmute::<u32, CpuInterrupt>(*num as u32),
core::mem::transmute::<u8, Priority>((prio as u8) + 1),
);
enable_cpu_interrupt(core::mem::transmute::<u32, CpuInterrupt>(*num as u32));
}
}
#[inline]
fn configured_interrupts(
core: Cpu,
status: InterruptStatus,
priority: Priority,
) -> InterruptStatus {
unsafe {
let mut res = InterruptStatus::empty();
for interrupt_nr in status.iterator() {
if let Some(cpu_interrupt) =
assigned_cpu_interrupt(core::mem::transmute::<u16, Interrupt>(
interrupt_nr as u16,
))
{
if priority_by_core(core, cpu_interrupt) == priority {
res.set(interrupt_nr as u16);
}
}
}
res
}
}
pub fn enable(interrupt: Interrupt, level: Priority) -> Result<(), Error> {
if matches!(level, Priority::None) {
return Err(Error::InvalidInterruptPriority);
}
unsafe {
let cpu_interrupt = core::mem::transmute::<u32, CpuInterrupt>(
PRIORITY_TO_INTERRUPT[(level as usize) - 1] as u32,
);
map(Cpu::current(), interrupt, cpu_interrupt);
enable_cpu_interrupt(cpu_interrupt);
}
Ok(())
}
pub unsafe fn bind_interrupt(interrupt: Interrupt, handler: unsafe extern "C" fn()) {
let ptr = &peripherals::__EXTERNAL_INTERRUPTS[interrupt as usize]._handler as *const _
as *mut unsafe extern "C" fn();
ptr.write_volatile(handler);
}
pub fn bound_handler(interrupt: Interrupt) -> Option<unsafe extern "C" fn()> {
unsafe {
let addr = peripherals::__EXTERNAL_INTERRUPTS[interrupt as usize]._handler;
if addr as usize == 0 {
return None;
}
Some(addr)
}
}
#[no_mangle]
#[ram]
unsafe fn handle_interrupts(cpu_intr: CpuInterrupt, context: &mut TrapFrame) {
let core = Cpu::current();
let status = status(core);
clear(core, cpu_intr);
let prio: Priority =
unsafe { core::mem::transmute(INTERRUPT_TO_PRIORITY[cpu_intr as usize - 1] as u8) };
let configured_interrupts = configured_interrupts(core, status, prio);
for interrupt_nr in configured_interrupts.iterator() {
let interrupt: Interrupt = unsafe { core::mem::transmute(interrupt_nr as u16) };
handle_interrupt(interrupt, context);
}
}
#[inline(always)]
unsafe fn handle_interrupt(interrupt: Interrupt, save_frame: &mut TrapFrame) {
extern "C" {
fn EspDefaultHandler(interrupt: Interrupt);
}
let handler = peripherals::__EXTERNAL_INTERRUPTS[interrupt as usize]._handler;
if core::ptr::eq(
handler as *const _,
EspDefaultHandler as *const unsafe extern "C" fn(),
) {
EspDefaultHandler(interrupt);
} else {
let handler: fn(&mut TrapFrame) =
core::mem::transmute::<unsafe extern "C" fn(), fn(&mut TrapFrame)>(handler);
handler(save_frame);
}
}
macro_rules! interrupt_handler {
($num:literal) => {
core::arch::global_asm! {
concat!(
r#"
.section .rwtext, "ax"
.global interrupt"#,$num,r#"
interrupt"#,$num,r#":
mv a1, a0
li a0,"#,$num,r#"
j handle_interrupts
"#
)
}
};
}
interrupt_handler!(1);
interrupt_handler!(2);
interrupt_handler!(3);
interrupt_handler!(4);
interrupt_handler!(5);
interrupt_handler!(6);
interrupt_handler!(7);
interrupt_handler!(8);
interrupt_handler!(9);
interrupt_handler!(10);
interrupt_handler!(11);
interrupt_handler!(12);
interrupt_handler!(13);
interrupt_handler!(14);
interrupt_handler!(15);
#[cfg(plic)]
interrupt_handler!(16);
#[cfg(plic)]
interrupt_handler!(17);
#[cfg(plic)]
interrupt_handler!(18);
#[cfg(plic)]
interrupt_handler!(19);
}
#[cfg(not(plic))]
mod classic {
use super::{CpuInterrupt, InterruptKind, Priority};
use crate::Cpu;
#[cfg_attr(place_switch_tables_in_ram, link_section = ".rwtext")]
pub(super) static DISABLED_CPU_INTERRUPT: u32 = 0;
#[cfg_attr(place_switch_tables_in_ram, link_section = ".rwtext")]
pub(super) static EXTERNAL_INTERRUPT_OFFSET: u32 = 0;
#[cfg_attr(place_switch_tables_in_ram, link_section = ".rwtext")]
pub(super) static PRIORITY_TO_INTERRUPT: &[usize] =
&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15];
#[cfg_attr(place_switch_tables_in_ram, link_section = ".rwtext")]
pub(super) static INTERRUPT_TO_PRIORITY: &[usize] =
&[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15];
pub unsafe fn enable_cpu_interrupt(which: CpuInterrupt) {
let cpu_interrupt_number = which as isize;
let intr = &*crate::peripherals::INTERRUPT_CORE0::PTR;
intr.cpu_int_enable()
.modify(|r, w| w.bits((1 << cpu_interrupt_number) | r.bits()));
}
pub fn set_kind(_core: Cpu, which: CpuInterrupt, kind: InterruptKind) {
unsafe {
let intr = &*crate::peripherals::INTERRUPT_CORE0::PTR;
let cpu_interrupt_number = which as isize;
let interrupt_type = match kind {
InterruptKind::Level => 0,
InterruptKind::Edge => 1,
};
intr.cpu_int_type().modify(|r, w| {
w.bits(
r.bits() & !(1 << cpu_interrupt_number)
| (interrupt_type << cpu_interrupt_number),
)
});
}
}
pub unsafe fn set_priority(_core: Cpu, which: CpuInterrupt, priority: Priority) {
let intr = &*crate::peripherals::INTERRUPT_CORE0::PTR;
intr.cpu_int_pri(which as usize)
.write(|w| w.map().bits(priority as u8));
}
#[inline]
pub fn clear(_core: Cpu, which: CpuInterrupt) {
unsafe {
let cpu_interrupt_number = which as isize;
let intr = &*crate::peripherals::INTERRUPT_CORE0::PTR;
intr.cpu_int_clear()
.write(|w| w.bits(1 << cpu_interrupt_number));
}
}
#[inline]
pub(super) fn priority_by_core(_core: Cpu, cpu_interrupt: CpuInterrupt) -> Priority {
unsafe { priority(cpu_interrupt) }
}
#[inline]
pub(super) unsafe extern "C" fn priority(cpu_interrupt: CpuInterrupt) -> Priority {
let intr = &*crate::peripherals::INTERRUPT_CORE0::PTR;
core::mem::transmute::<u8, Priority>(
intr.cpu_int_pri(cpu_interrupt as usize).read().map().bits(),
)
}
#[no_mangle]
#[link_section = ".trap"]
pub(super) unsafe extern "C" fn _handle_priority() -> u32 {
use super::mcause;
let interrupt_id: usize = mcause::read().code(); let intr = &*crate::peripherals::INTERRUPT_CORE0::PTR;
let interrupt_priority = intr
.cpu_int_pri(0)
.as_ptr()
.add(interrupt_id)
.read_volatile();
let prev_interrupt_priority = intr.cpu_int_thresh().read().bits();
if interrupt_priority < 15 {
intr.cpu_int_thresh()
.write(|w| w.bits(interrupt_priority + 1)); unsafe {
riscv::interrupt::enable();
}
}
prev_interrupt_priority
}
#[no_mangle]
#[link_section = ".trap"]
pub(super) unsafe extern "C" fn _restore_priority(stored_prio: u32) {
riscv::interrupt::disable();
let intr = &*crate::peripherals::INTERRUPT_CORE0::PTR;
intr.cpu_int_thresh().write(|w| w.bits(stored_prio));
}
}
#[cfg(plic)]
mod plic {
use super::{CpuInterrupt, InterruptKind, Priority};
use crate::Cpu;
#[cfg_attr(place_switch_tables_in_ram, link_section = ".rwtext")]
pub(super) static DISABLED_CPU_INTERRUPT: u32 = 31;
#[cfg_attr(place_switch_tables_in_ram, link_section = ".rwtext")]
pub(super) static EXTERNAL_INTERRUPT_OFFSET: u32 = 0;
#[cfg_attr(place_switch_tables_in_ram, link_section = ".rwtext")]
pub(super) static PRIORITY_TO_INTERRUPT: &[usize] =
&[1, 2, 5, 6, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19];
#[cfg_attr(place_switch_tables_in_ram, link_section = ".rwtext")]
pub(super) static INTERRUPT_TO_PRIORITY: &[usize] = &[
1, 2, 0, 0, 3, 4, 0, 0, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
];
pub unsafe fn enable_cpu_interrupt(which: CpuInterrupt) {
unsafe {
let plic = &*crate::peripherals::PLIC_MX::PTR;
plic.mxint_enable().modify(|r, w| {
let old = r.cpu_mxint_enable().bits();
let new = old | 1 << (which as isize);
w.cpu_mxint_enable().bits(new)
});
}
}
pub fn set_kind(_core: Cpu, which: CpuInterrupt, kind: InterruptKind) {
unsafe {
let plic = &*crate::peripherals::PLIC_MX::PTR;
let interrupt_type = match kind {
InterruptKind::Level => 0,
InterruptKind::Edge => 1,
};
plic.mxint_type().modify(|r, w| {
let old = r.cpu_mxint_type().bits();
let new = old & !(1 << (which as isize)) | (interrupt_type << (which as isize));
w.cpu_mxint_type().bits(new)
});
}
}
pub unsafe fn set_priority(_core: Cpu, which: CpuInterrupt, priority: Priority) {
unsafe {
let plic = &*crate::peripherals::PLIC_MX::PTR;
plic.mxint_pri(which as usize)
.modify(|_, w| w.cpu_mxint_pri().bits(priority as u8));
}
}
#[inline]
pub fn clear(_core: Cpu, which: CpuInterrupt) {
unsafe {
let plic = &*crate::peripherals::PLIC_MX::PTR;
plic.mxint_clear().modify(|r, w| {
let old = r.cpu_mxint_clear().bits();
let new = old | (1 << (which as isize));
w.cpu_mxint_clear().bits(new)
});
}
}
#[inline]
pub(super) unsafe extern "C" fn priority_by_core(
_core: Cpu,
cpu_interrupt: CpuInterrupt,
) -> Priority {
unsafe { priority(cpu_interrupt) }
}
#[inline]
pub(super) unsafe extern "C" fn priority(cpu_interrupt: CpuInterrupt) -> Priority {
let plic = &*crate::peripherals::PLIC_MX::PTR;
let prio = plic
.mxint_pri(cpu_interrupt as usize)
.read()
.cpu_mxint_pri()
.bits();
core::mem::transmute::<u8, Priority>(prio)
}
#[no_mangle]
#[link_section = ".trap"]
pub(super) unsafe extern "C" fn _handle_priority() -> u32 {
use super::mcause;
let plic = &*crate::peripherals::PLIC_MX::PTR;
let interrupt_id: usize = mcause::read().code(); let interrupt_priority = plic.mxint_pri(interrupt_id).read().cpu_mxint_pri().bits();
let prev_interrupt_priority = plic.mxint_thresh().read().cpu_mxint_thresh().bits();
if interrupt_priority < 15 {
plic.mxint_thresh()
.write(|w| w.cpu_mxint_thresh().bits(interrupt_priority + 1));
unsafe {
riscv::interrupt::enable();
}
}
prev_interrupt_priority as u32
}
#[no_mangle]
#[link_section = ".trap"]
pub(super) unsafe extern "C" fn _restore_priority(stored_prio: u32) {
riscv::interrupt::disable();
let plic = &*crate::peripherals::PLIC_MX::PTR;
plic.mxint_thresh()
.write(|w| w.cpu_mxint_thresh().bits(stored_prio as u8));
}
}