#![allow(non_upper_case_globals)]
use core::arch::asm;
use aarch64_define::{
ESR_ELx_CM, ESR_ELx_EC_DABT_CUR, ESR_ELx_EC_IABT_CUR, ESR_ELx_EC_IABT_LOW, ESR_ELx_FSC_PERM,
ESR_ELx_FSC_TYPE, ESR_ELx_WNR, KERNEL_DS, PSR_MODE_EL0t, PSR_MODE_MASK, Ptregs, esr_elx_ec,
};
use crate::{
bsp::__irqchip_irq_handler,
irq::irqflags::{local_daif_mask, local_irq_disable, local_irq_restore},
print, println,
processor::{cpu_relax, this_processor_id},
sched::{
preempt::{NMI_OFFSET, preempt_count_add, preempt_count_sub},
task::current,
},
smp::smp_send_stop,
space::{
addr::Uaddr,
mm::{VmFlags, pgtabledef::PAGE_SIZE},
uaccess::USER_SPACE_SIZE,
},
};
#[doc(hidden)]
#[unsafe(no_mangle)]
#[allow(unreachable_pub)]
pub extern "C" fn handle_arch_irq(_regs: &mut Ptregs) {
unsafe {
__irqchip_irq_handler();
}
}
const HANDLER: [&str; 4] = ["Synchronous Abort", "IRQ", "FIQ", "Error"];
const ESR_CLASS_STR: [&str; 0x3f] = [
"Unknown/Uncategorized", "WFI/WFE", "UNRECOGNIZED EC", "CP15 MCR/MRC", "CP15 MCRR/MRRC", "CP14 MCR/MRC", "CP14 LDC/STC", "ASIMD", "CP10 MRC/VMRS", "UNRECOGNIZED EC", "UNRECOGNIZED EC", "UNRECOGNIZED EC", "CP14 MCRR/MRRC", "UNRECOGNIZED EC", "PSTATE.IL", "UNRECOGNIZED EC", "UNRECOGNIZED EC", "SVC (AArch32)", "HVC (AArch32)", "SMC (AArch32)", "UNRECOGNIZED EC", "SVC (AArch64)", "HVC (AArch64)", "SMC (AArch64)", "MSR/MRS (AArch64)", "SVE", "UNRECOGNIZED EC", "UNRECOGNIZED EC", "UNRECOGNIZED EC", "UNRECOGNIZED EC", "UNRECOGNIZED EC", "EL3 IMP DEF", "IABT (lower EL)", "IABT (current EL)", "PC Alignment", "UNRECOGNIZED EC", "DABT (lower EL)", "DABT (current EL)", "SP Alignment", "UNRECOGNIZED EC", "FP (AArch32)", "UNRECOGNIZED EC", "UNRECOGNIZED EC", "UNRECOGNIZED EC", "FP (AArch64)", "UNRECOGNIZED EC", "UNRECOGNIZED EC", "SError", "Breakpoint (lower EL)", "Breakpoint (current EL)", "Software Step (lower EL)", "Software Step (current EL)", "Watchpoint (lower EL)", "Watchpoint (current EL)", "UNRECOGNIZED EC", "UNRECOGNIZED EC", "BKPT (AArch32)", "UNRECOGNIZED EC", "Vector catch (AArch32)", "UNRECOGNIZED EC", "BRK (AArch64)", "UNRECOGNIZED EC", "UNRECOGNIZED EC", ];
const fn esr_get_class_string(esr: u32) -> &'static str {
ESR_CLASS_STR[esr_elx_ec(esr as usize)]
}
#[doc(hidden)]
#[unsafe(no_mangle)]
#[allow(unreachable_pub)]
pub extern "C" fn bad_mode(_regs: &mut Ptregs, reason: usize, esr: u32) {
println!(
"Bad mod in {} handler detected on CPU{}, code {:#x} -- {}",
HANDLER[reason],
this_processor_id(),
esr,
esr_get_class_string(esr)
);
local_daif_mask();
panic!("bad mode");
}
fn do_bad(_: usize, _: u32, _: &mut Ptregs) -> bool {
false
}
fn do_sea(addr: usize, esr: u32, regs: &mut Ptregs) -> bool {
let inf = esr_to_fault_info(esr);
arm64_notify_die(inf.name, regs, addr, esr);
true
}
fn do_alignment_fault(addr: usize, esr: u32, regs: &mut Ptregs) -> bool {
do_bad_area(addr, esr, regs);
true
}
fn do_translation_fault(addr: usize, esr: u32, regs: &mut Ptregs) -> bool {
if addr < USER_SPACE_SIZE {
return do_page_fault(addr, esr, regs);
}
do_bad_area(addr, esr, regs);
true
}
fn do_bad_area(addr: usize, esr: u32, regs: &mut Ptregs) {
if user_mode(regs) {
let inf = esr_to_fault_info(esr);
arm64_force_sig_fault(0, esr as usize, addr as u64, inf.name);
} else {
__do_kernel_fault(addr, esr, regs);
}
}
fn is_el0_instruction_abort(esr: u32) -> bool {
esr_elx_ec(esr as usize) == ESR_ELx_EC_IABT_LOW
}
fn is_el1_permission_fault(_addr: usize, esr: u32, _regs: &mut Ptregs) -> bool {
let ec = esr_elx_ec(esr as usize);
if ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR {
return false;
}
let fsc_type = esr as usize & ESR_ELx_FSC_TYPE;
if fsc_type == ESR_ELx_FSC_PERM {
return true;
}
false
}
fn is_el1_instruction_abort(esr: u32) -> bool {
esr_elx_ec(esr as usize) == ESR_ELx_EC_IABT_CUR
}
fn do_page_fault(addr: usize, esr: u32, regs: &mut Ptregs) -> bool {
let mut vm_flags = VmFlags::VM_READ;
if is_el0_instruction_abort(esr) {
vm_flags = VmFlags::VM_EXEC;
} else if (esr as usize & ESR_ELx_WNR != 0) && (esr as usize & ESR_ELx_CM == 0) {
vm_flags = VmFlags::VM_WRITE;
}
if addr < USER_SPACE_SIZE && is_el1_permission_fault(addr, esr, regs) {
if regs.get_orig_addr_limit() == KERNEL_DS as u64 {
die_kernel_fault("access to user memory with fs=KERNEL_DS", addr, esr, regs);
}
if is_el1_instruction_abort(esr) {
die_kernel_fault("execution of user memory", addr, esr, regs);
}
if !search_exception_tables(regs.get_pc()) {
die_kernel_fault("access to user memory outside uaccess routines", addr, esr, regs);
}
}
let mm = current().mm();
if let Some(m) = mm {
m.handle_mm_fault(Uaddr::from(addr), vm_flags).unwrap();
} else {
__do_kernel_fault(addr, esr, regs);
}
true
}
fn search_exception_tables(_pc: u64) -> bool {
false
}
fn fixup_exception(_regs: &mut Ptregs) -> bool {
false
}
fn die_kernel_fault(msg: &str, addr: usize, _esr: u32, _regs: &mut Ptregs) {
panic!("Unable to handle kernel {} at virtual address {:#016x}", msg, addr);
}
fn __do_kernel_fault(addr: usize, esr: u32, regs: &mut Ptregs) {
if !is_el1_instruction_abort(esr) && fixup_exception(regs) {
return;
}
let msg;
if is_el1_permission_fault(addr, esr, regs) {
if esr as usize & ESR_ELx_WNR != 0 {
msg = "write to read-only memory";
} else {
msg = "read from unreadable memory";
}
} else if addr < PAGE_SIZE {
msg = "NULL pointer dereference";
} else {
msg = "paging request";
}
die_kernel_fault(msg, addr, esr, regs);
}
struct FaultInfo {
f: fn(addr: usize, esr: u32, regs: &mut Ptregs) -> bool,
name: &'static str,
}
const FAULT_INFO: [FaultInfo; 64] = [
FaultInfo { f: do_bad, name: "ttbr address size fault" },
FaultInfo { f: do_bad, name: "level 1 address size fault" },
FaultInfo { f: do_bad, name: "level 2 address size fault" },
FaultInfo { f: do_bad, name: "level 3 address size fault" },
FaultInfo { f: do_translation_fault, name: "level 0 translation fault" },
FaultInfo { f: do_translation_fault, name: "level 1 translation fault" },
FaultInfo { f: do_translation_fault, name: "level 2 translation fault" },
FaultInfo { f: do_translation_fault, name: "level 3 translation fault" },
FaultInfo { f: do_bad, name: "unknown 8" },
FaultInfo { f: do_page_fault, name: "level 1 access flag fault" },
FaultInfo { f: do_page_fault, name: "level 2 access flag fault" },
FaultInfo { f: do_page_fault, name: "level 3 access flag fault" },
FaultInfo { f: do_bad, name: "unknown 12" },
FaultInfo { f: do_page_fault, name: "level 1 permission fault" },
FaultInfo { f: do_page_fault, name: "level 2 permission fault" },
FaultInfo { f: do_page_fault, name: "level 3 permission fault" },
FaultInfo { f: do_sea, name: "synchronous external abort" },
FaultInfo { f: do_bad, name: "unknown 17" },
FaultInfo { f: do_bad, name: "unknown 18" },
FaultInfo { f: do_bad, name: "unknown 19" },
FaultInfo { f: do_sea, name: "synchronous external abort" },
FaultInfo { f: do_sea, name: "level 0 (translation table walk)" },
FaultInfo { f: do_sea, name: "level 1 (translation table walk)" },
FaultInfo { f: do_sea, name: "level 2 (translation table walk)" },
FaultInfo { f: do_sea, name: "level 3 (translation table walk)" },
FaultInfo { f: do_bad, name: "unknown 25" },
FaultInfo { f: do_bad, name: "unknown 26" },
FaultInfo { f: do_bad, name: "unknown 27" },
FaultInfo { f: do_sea, name: "level 0 synchronous parity error (translation table walk)" },
FaultInfo { f: do_sea, name: "level 1 synchronous parity error (translation table walk)" },
FaultInfo { f: do_sea, name: "level 2 synchronous parity error (translation table walk)" },
FaultInfo { f: do_sea, name: "level 3 synchronous parity error (translation table walk)" },
FaultInfo { f: do_bad, name: "unknown 32" },
FaultInfo { f: do_alignment_fault, name: "alignment fault" },
FaultInfo { f: do_bad, name: "unknown 34" },
FaultInfo { f: do_bad, name: "unknown 35" },
FaultInfo { f: do_bad, name: "unknown 36" },
FaultInfo { f: do_bad, name: "unknown 37" },
FaultInfo { f: do_bad, name: "unknown 38" },
FaultInfo { f: do_bad, name: "unknown 39" },
FaultInfo { f: do_bad, name: "unknown 40" },
FaultInfo { f: do_bad, name: "unknown 41" },
FaultInfo { f: do_bad, name: "unknown 42" },
FaultInfo { f: do_bad, name: "unknown 43" },
FaultInfo { f: do_bad, name: "unknown 44" },
FaultInfo { f: do_bad, name: "unknown 45" },
FaultInfo { f: do_bad, name: "unknown 46" },
FaultInfo { f: do_bad, name: "unknown 47" },
FaultInfo { f: do_bad, name: "TLB conflict abort" },
FaultInfo { f: do_bad, name: "Unsupported atomic hardware update fault" },
FaultInfo { f: do_bad, name: "unknown 50" },
FaultInfo { f: do_bad, name: "unknown 51" },
FaultInfo { f: do_bad, name: "implementation fault (lockdown abort)" },
FaultInfo { f: do_bad, name: "implementation fault (unsupported exclusive)" },
FaultInfo { f: do_bad, name: "unknown 54" },
FaultInfo { f: do_bad, name: "unknown 55" },
FaultInfo { f: do_bad, name: "unknown 56" },
FaultInfo { f: do_bad, name: "unknown 57" },
FaultInfo { f: do_bad, name: "unknown 58" },
FaultInfo { f: do_bad, name: "unknown 59" },
FaultInfo { f: do_bad, name: "unknown 60" },
FaultInfo { f: do_bad, name: "section domain fault" },
FaultInfo { f: do_bad, name: "page domain fault" },
FaultInfo { f: do_bad, name: "unknown 63" },
];
const fn esr_to_fault_info(esr: u32) -> &'static FaultInfo {
&FAULT_INFO[esr as usize & 0x3f]
}
fn user_mode(regs: &mut Ptregs) -> bool {
(regs.get_pstate() as usize & PSR_MODE_MASK) == PSR_MODE_EL0t
}
fn arm64_notify_die(str: &str, regs: &mut Ptregs, addr: usize, err: u32) {
if user_mode(regs) {
arm64_force_sig_fault(0, err as usize, addr as u64, str);
} else {
die(str, regs, err);
}
}
fn die(str: &str, _regs: &mut Ptregs, _err: u32) {
panic!("Fatal exception: {}", str);
}
#[doc(hidden)]
#[unsafe(no_mangle)]
#[allow(unreachable_pub)]
pub extern "C" fn do_mem_abort(addr: usize, esr: u32, regs: &mut Ptregs) {
let inf = esr_to_fault_info(esr);
if (inf.f)(addr, esr, regs) {
return;
}
assert!(user_mode(regs), "Unhandled fault at {addr:#x}");
arm64_notify_die(inf.name, regs, addr, esr);
}
#[doc(hidden)]
#[unsafe(no_mangle)]
#[allow(unreachable_pub)]
pub extern "C" fn do_sp_pc_abort(addr: usize, esr: u32, regs: &mut Ptregs) {
if user_mode(regs) {
local_irq_restore(0);
}
arm64_notify_die("SP/PC alignment exception", regs, addr, esr);
}
#[doc(hidden)]
#[unsafe(no_mangle)]
#[allow(unreachable_pub)]
pub extern "C" fn do_undefinstr(regs: &mut Ptregs) {
arm64_notify_die("do_undefinstr", regs, 0, 0);
}
#[doc(hidden)]
#[unsafe(no_mangle)]
#[allow(unreachable_pub)]
pub extern "C" fn do_debug_exception(addr: usize, esr: u32, regs: &mut Ptregs) {
arm64_notify_die("do_debug_exception", regs, addr, esr);
}
#[doc(hidden)]
#[unsafe(no_mangle)]
#[allow(unreachable_pub)]
pub extern "C" fn preempt_schedule_irq() {
println!("preempt_schedule_irq");
}
#[doc(hidden)]
#[unsafe(no_mangle)]
#[allow(unreachable_pub)]
pub extern "C" fn do_el0_ia_bp_hardening(addr: usize, esr: u32, regs: &mut Ptregs) {
local_irq_restore(0);
do_mem_abort(addr, esr, regs);
}
#[doc(hidden)]
#[unsafe(no_mangle)]
#[allow(unreachable_pub)]
pub extern "C" fn do_fpsimd_acc(_esr: u32, _regs: &mut Ptregs) {
println!("do_fpsimd_acc");
}
#[doc(hidden)]
#[unsafe(no_mangle)]
#[allow(unreachable_pub)]
pub extern "C" fn do_sve_acc(esr: u32, regs: &mut Ptregs) {
arm64_notify_die("do_sve_acc", regs, 0, esr);
}
#[doc(hidden)]
#[unsafe(no_mangle)]
#[allow(unreachable_pub)]
pub extern "C" fn do_fpsimd_exc(esr: u32, regs: &mut Ptregs) {
arm64_notify_die("do_fpsimd_exc", regs, 0, esr);
}
#[doc(hidden)]
#[unsafe(no_mangle)]
#[allow(unreachable_pub)]
pub extern "C" fn do_sysinstr(esr: u32, regs: &mut Ptregs) {
arm64_notify_die("do_sysinstr", regs, 0, esr);
}
#[doc(hidden)]
#[unsafe(no_mangle)]
#[allow(unreachable_pub)]
pub extern "C" fn do_notify_resume(_regs: &mut Ptregs, _thread_flags: u32) {
panic!("do_notify_resume");
}
#[doc(hidden)]
#[unsafe(no_mangle)]
#[allow(unreachable_pub)]
pub extern "C" fn el0_svc_handler(_regs: &mut Ptregs) {
panic!("el0_svc_handler");
}
#[doc(hidden)]
#[unsafe(no_mangle)]
#[allow(unreachable_pub)]
pub extern "C" fn do_serror(_regs: &mut Ptregs, _esr: u32) {
nmi_enter();
nmi_exit();
panic!("do_serror");
}
#[doc(hidden)]
#[unsafe(no_mangle)]
#[allow(unreachable_pub)]
pub extern "C" fn bad_el0_sync(regs: &mut Ptregs, _: i32, esr: u32) {
let pc = regs.get_pc();
arm64_force_sig_fault(0, esr as usize, pc, "Bad EL0 synchronous exception");
}
fn arm64_force_sig_fault(signo: usize, code: usize, addr: u64, str: &str) {
print!("{}: unhandled exception: ", current().name());
if code != 0 {
print!("{}, ESR {:#x}, ", esr_get_class_string(code as u32), code);
}
println!("{} in {:#x}, sig {}", str, addr, signo);
local_irq_disable();
smp_send_stop();
loop {
cpu_relax();
}
}
#[inline(always)]
fn nmi_enter() {
preempt_count_add(NMI_OFFSET as u32);
}
#[inline(always)]
fn nmi_exit() {
preempt_count_sub(NMI_OFFSET as u32);
}
pub(super) fn arch_secondary_info() {
let mpidr: usize;
unsafe {
asm!(
"mrs {0}, mpidr_el1",
out(reg) mpidr
);
}
let midr_el1: usize;
unsafe {
asm!(
"mrs {0}, midr_el1",
out(reg) midr_el1
);
}
println!(
"CPU{}: Booted secondary processor {:#010x} [{:#08x}]",
this_processor_id(),
mpidr & 0xff00ffffff,
midr_el1
);
}