ax_cpu/arch/aarch64/
trap.rs

1use aarch64_cpu::registers::*;
2use tock_registers::interfaces::Readable;
3
4use super::context::TrapFrame;
5use crate::{TrapOrigin, trap::PageFaultFlags};
6
7/// Untrusted register image produced and consumed by trap assembly.
8#[repr(transparent)]
9struct RawTrapFrame(TrapFrame);
10
11const _: () = {
12    assert!(core::mem::size_of::<RawTrapFrame>() == core::mem::size_of::<TrapFrame>());
13    assert!(core::mem::align_of::<RawTrapFrame>() == core::mem::align_of::<TrapFrame>());
14};
15
16/// Lifetime-bound view of a kernel-origin AArch64 trap frame.
17pub struct KernelTrapFrame<'a> {
18    raw: &'a mut RawTrapFrame,
19    _not_send: core::marker::PhantomData<*mut ()>,
20}
21
22impl<'a> KernelTrapFrame<'a> {
23    /// Returns the privilege domain represented by this view.
24    pub const fn origin(&self) -> TrapOrigin {
25        TrapOrigin::Kernel
26    }
27
28    /// Copies the saved register image for inspection or probe emulation.
29    pub const fn snapshot(&self) -> TrapFrame {
30        self.raw.0
31    }
32
33    /// Applies task-register changes while preserving origin and saved SP.
34    pub fn apply_registers(&mut self, updated: &TrapFrame) {
35        const MODE_MASK: u64 = 0b1_1111;
36        let saved_mode = self.raw.0.spsr & MODE_MASK;
37        let sp = self.raw.0.sp;
38        self.raw.0 = *updated;
39        self.raw.0.spsr = (self.raw.0.spsr & !MODE_MASK) | saved_mode;
40        self.raw.0.sp = sp;
41    }
42
43    /// Returns the saved instruction pointer.
44    pub const fn ip(&self) -> usize {
45        self.raw.0.ip()
46    }
47
48    /// Sets the saved instruction pointer.
49    pub const fn set_ip(&mut self, ip: usize) {
50        self.raw.0.set_ip(ip);
51    }
52
53    /// Creates the typed view at the assembly boundary.
54    ///
55    /// # Safety
56    ///
57    /// `raw` must be the uniquely borrowed, live kernel-origin frame built by
58    /// the AArch64 vector entry and must remain valid for `'a`.
59    unsafe fn from_raw(raw: &'a mut RawTrapFrame) -> Self {
60        debug_assert_eq!(raw.0.origin(), TrapOrigin::Kernel);
61        Self {
62            raw,
63            _not_send: core::marker::PhantomData,
64        }
65    }
66}
67
68impl core::fmt::Debug for KernelTrapFrame<'_> {
69    fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
70        self.snapshot().fmt(formatter)
71    }
72}
73
74#[repr(u8)]
75#[derive(Clone, Copy, Debug, Eq, PartialEq)]
76/// AArch64 vector entry kind.
77pub enum TrapKind {
78    /// A synchronous exception.
79    Synchronous = 0,
80    /// A physical or virtual IRQ.
81    Irq         = 1,
82    /// A fast interrupt.
83    Fiq         = 2,
84    /// An asynchronous system error.
85    SError      = 3,
86}
87
88impl TrapKind {
89    const fn from_raw(value: u8) -> Option<Self> {
90        match value {
91            0 => Some(Self::Synchronous),
92            1 => Some(Self::Irq),
93            2 => Some(Self::Fiq),
94            3 => Some(Self::SError),
95            _ => None,
96        }
97    }
98}
99
100#[repr(u8)]
101#[derive(Clone, Copy, Debug, Eq, PartialEq)]
102/// Execution domain selecting an AArch64 vector slot.
103pub enum TrapSource {
104    /// Current exception level using SP_EL0.
105    CurrentSpEl0 = 0,
106    /// Current exception level using its own stack.
107    CurrentSpElx = 1,
108    /// A lower level executing AArch64.
109    LowerAArch64 = 2,
110    /// A lower level executing AArch32.
111    LowerAArch32 = 3,
112}
113
114impl TrapSource {
115    const fn from_raw(value: u8) -> Option<Self> {
116        match value {
117            0 => Some(Self::CurrentSpEl0),
118            1 => Some(Self::CurrentSpElx),
119            2 => Some(Self::LowerAArch64),
120            3 => Some(Self::LowerAArch32),
121            _ => None,
122        }
123    }
124}
125
126core::arch::global_asm!(
127    include_str!("entry/gpr.S"),
128    include_str!("entry/trap.S"),
129    trapframe_size = const core::mem::size_of::<RawTrapFrame>(),
130    elr_offset = const core::mem::offset_of!(TrapFrame, elr),
131    sp_offset = const core::mem::offset_of!(TrapFrame, sp),
132    TRAP_KIND_SYNC = const TrapKind::Synchronous as u8,
133    TRAP_KIND_IRQ = const TrapKind::Irq as u8,
134    TRAP_KIND_FIQ = const TrapKind::Fiq as u8,
135    TRAP_KIND_SERROR = const TrapKind::SError as u8,
136    TRAP_SRC_CURR_EL0 = const TrapSource::CurrentSpEl0 as u8,
137    TRAP_SRC_CURR_ELX = const TrapSource::CurrentSpElx as u8,
138    TRAP_SRC_LOWER_AARCH64 = const TrapSource::LowerAArch64 as u8,
139    TRAP_SRC_LOWER_AARCH32 = const TrapSource::LowerAArch32 as u8,
140);
141
142#[inline(always)]
143pub(super) fn is_valid_page_fault(iss: u64) -> bool {
144    // Only handle Translation fault and Permission fault
145    matches!(iss & 0b111100, 0b0100 | 0b1100) // IFSC or DFSC bits
146}
147
148fn handle_breakpoint(tf: &mut KernelTrapFrame<'_>) {
149    if crate::trap::breakpoint_handler(tf) {
150        return;
151    }
152    tf.set_ip(tf.ip() + 4);
153}
154
155fn handle_page_fault(
156    tf: &mut KernelTrapFrame<'_>,
157    access_flags: PageFaultFlags,
158    esr: u64,
159    far: usize,
160) {
161    let vaddr = va!(far);
162    #[cfg(feature = "exception-table")]
163    if tf.raw.0.fixup_nofault_exception() {
164        return;
165    }
166    if crate::trap::call_page_fault_handler_with_parent_irqs(
167        vaddr,
168        access_flags,
169        tf.raw.0.spsr & (1 << 7) == 0,
170    ) {
171        return;
172    }
173    #[cfg(feature = "exception-table")]
174    if tf.raw.0.fixup_exception() {
175        return;
176    }
177    let snapshot = tf.snapshot();
178    let bt = crate::trap::diagnostics::BacktraceDisplay(snapshot.backtrace_registers());
179    panic!(
180        "Unhandled Page Fault @ {:#x}, fault_vaddr={:#x}, ESR={:#x} ({:?}):\n{:#x?}\n{}",
181        tf.raw.0.elr, vaddr, esr, access_flags, snapshot, bt
182    );
183}
184
185#[unsafe(no_mangle)]
186unsafe extern "C" fn aarch64_trap_handler(
187    raw: *mut RawTrapFrame,
188    raw_kind: u8,
189    raw_source: u8,
190    level: u8,
191) {
192    let kind = TrapKind::from_raw(raw_kind)
193        .unwrap_or_else(|| panic!("invalid AArch64 trap kind {raw_kind:#x}"));
194    let source = TrapSource::from_raw(raw_source)
195        .unwrap_or_else(|| panic!("invalid AArch64 trap source {raw_source:#x}"));
196    // SAFETY: the vector assembly passes its aligned, live stack frame and
197    // retains exclusive ownership until this handler returns.
198    let raw = unsafe { &mut *raw };
199    if matches!(
200        source,
201        TrapSource::CurrentSpEl0 | TrapSource::LowerAArch64 | TrapSource::LowerAArch32
202    ) {
203        let bt = crate::trap::diagnostics::BacktraceDisplay(raw.0.backtrace_registers());
204        panic!(
205            "Invalid exception {:?} from {:?}:\n{:#x?}\n{}",
206            kind, source, raw.0, bt
207        );
208    }
209    let mut tf = unsafe { KernelTrapFrame::from_raw(raw) };
210    match kind {
211        TrapKind::Fiq | TrapKind::SError => {
212            let snapshot = tf.snapshot();
213            let bt = crate::trap::diagnostics::BacktraceDisplay(snapshot.backtrace_registers());
214            panic!("Unhandled exception {:?}:\n{:#x?}\n{}", kind, snapshot, bt);
215        }
216        TrapKind::Irq => {
217            crate::trap::dispatch_irq(
218                0,
219                crate::trap::TrapOrigin::Kernel,
220                Some(raw.0.interrupted_context()),
221            );
222        }
223        TrapKind::Synchronous => {
224            // Capture the selected exception bank before invoking any host hook.
225            let (esr, far) = match level {
226                1 => (ESR_EL1.get(), FAR_EL1.get() as usize),
227                2 => (ESR_EL2.get(), FAR_EL2.get() as usize),
228                _ => panic!("invalid exception level {level}"),
229            };
230            let iss = esr & 0x01ff_ffff;
231            let ec = (esr >> 26) & 0x3f;
232            match ec {
233                0x21 if is_valid_page_fault(iss) => {
234                    handle_page_fault(&mut tf, PageFaultFlags::EXECUTE, esr, far);
235                }
236                0x25 if is_valid_page_fault(iss) => {
237                    let write = iss & (1 << 6) != 0;
238                    let cache_maintenance = iss & (1 << 8) != 0;
239                    let access = if write && !cache_maintenance {
240                        PageFaultFlags::WRITE
241                    } else {
242                        PageFaultFlags::READ
243                    };
244                    handle_page_fault(&mut tf, access, esr, far);
245                }
246                0x3c => handle_breakpoint(&mut tf),
247                _ => {
248                    let snapshot = tf.snapshot();
249                    let bt =
250                        crate::trap::diagnostics::BacktraceDisplay(snapshot.backtrace_registers());
251                    panic!(
252                        "Unhandled EL{level} synchronous exception @ {:#x}: ESR={esr:#x}, \
253                         FAR={far:#x}\n{bt}",
254                        tf.ip()
255                    );
256                }
257            }
258        }
259    }
260}
261
262#[unsafe(no_mangle)]
263unsafe extern "C" fn __ax_cpu_boot_trap(raw: *const RawTrapFrame, kind: u8, source: u8, level: u8) {
264    // SAFETY: the shared vector save sequence owns this aligned initialized
265    // frame until the synchronous boot callback returns to its restore sequence.
266    let frame = unsafe { &(*raw).0 };
267    let (syndrome, fault_address) = match level {
268        1 => (ESR_EL1.get(), FAR_EL1.get()),
269        2 => (ESR_EL2.get(), FAR_EL2.get()),
270        _ => panic!("invalid boot exception level"),
271    };
272    let exception = crate::trap::boot::BootException {
273        registers: frame.x,
274        pc: frame.elr as usize,
275        sp: frame.sp as usize,
276        status: frame.spsr,
277        syndrome,
278        fault_address: crate::VirtAddr::from_usize(fault_address as usize),
279        level,
280        kind: TrapKind::from_raw(kind).expect("CPU vector kind"),
281        source: TrapSource::from_raw(source).expect("CPU vector source"),
282    };
283    crate::trap::boot::boot_trap_handler::handle(&exception);
284}
ax_cpu/arch/aarch64/trap.rs

ax_cpu/arch/aarch64/
trap.rs
