ax_cpu/x86_64/
trap.rs

1use x86::{controlregs::cr2, irq::*};
2use x86_64::{registers::rflags::RFlags, structures::idt::PageFaultErrorCode};
3
4use super::{TrapFrame, gdt};
5use crate::{TrapOrigin, trap::PageFaultFlags};
6
7/// Untrusted register image produced and consumed by trap assembly.
8///
9/// Kernel-origin traps do not push `rsp` or `ss`; the raw type therefore ends
10/// at `rflags`. The public user register image includes those two fields, but
11/// constructing a reference to that larger type here would read beyond the
12/// initialized hardware frame.
13#[repr(C)]
14struct RawTrapFrame {
15    rax: u64,
16    rcx: u64,
17    rdx: u64,
18    rbx: u64,
19    rbp: u64,
20    rsi: u64,
21    rdi: u64,
22    r8: u64,
23    r9: u64,
24    r10: u64,
25    r11: u64,
26    r12: u64,
27    r13: u64,
28    r14: u64,
29    r15: u64,
30    vector: u64,
31    error_code: u64,
32    rip: u64,
33    cs: u64,
34    rflags: u64,
35}
36
37const _: () = {
38    assert!(core::mem::size_of::<RawTrapFrame>() == core::mem::offset_of!(TrapFrame, rsp));
39    assert!(
40        core::mem::offset_of!(RawTrapFrame, vector) == core::mem::offset_of!(TrapFrame, vector)
41    );
42    assert!(core::mem::offset_of!(RawTrapFrame, rip) == core::mem::offset_of!(TrapFrame, rip));
43    assert!(
44        core::mem::offset_of!(RawTrapFrame, rflags) == core::mem::offset_of!(TrapFrame, rflags)
45    );
46};
47
48/// Lifetime-bound view of a kernel-origin x86 trap frame.
49///
50/// The view deliberately provides no mutable dereference to the assembly
51/// image. Probe/debug integrations can edit a copy and apply it through
52/// [`Self::apply_registers`], which preserves the trap origin and vector
53/// metadata.
54pub struct KernelTrapFrame<'a> {
55    raw: &'a mut RawTrapFrame,
56    _not_send: core::marker::PhantomData<*mut ()>,
57}
58
59impl<'a> KernelTrapFrame<'a> {
60    /// Returns the privilege domain represented by this view.
61    pub const fn origin(&self) -> TrapOrigin {
62        TrapOrigin::Kernel
63    }
64
65    /// Copies the saved register image for inspection or probe emulation.
66    pub fn snapshot(&self) -> TrapFrame {
67        TrapFrame {
68            rax: self.raw.rax,
69            rcx: self.raw.rcx,
70            rdx: self.raw.rdx,
71            rbx: self.raw.rbx,
72            rbp: self.raw.rbp,
73            rsi: self.raw.rsi,
74            rdi: self.raw.rdi,
75            r8: self.raw.r8,
76            r9: self.raw.r9,
77            r10: self.raw.r10,
78            r11: self.raw.r11,
79            r12: self.raw.r12,
80            r13: self.raw.r13,
81            r14: self.raw.r14,
82            r15: self.raw.r15,
83            vector: self.raw.vector,
84            error_code: self.raw.error_code,
85            rip: self.raw.rip,
86            cs: self.raw.cs,
87            rflags: self.raw.rflags,
88            rsp: self.raw as *const RawTrapFrame as u64
89                + core::mem::size_of::<RawTrapFrame>() as u64,
90            ss: gdt::KDATA.0 as u64,
91        }
92    }
93
94    /// Applies task-register changes while preserving trap-origin metadata.
95    pub fn apply_registers(&mut self, updated: &TrapFrame) {
96        self.raw.rax = updated.rax;
97        self.raw.rcx = updated.rcx;
98        self.raw.rdx = updated.rdx;
99        self.raw.rbx = updated.rbx;
100        self.raw.rbp = updated.rbp;
101        self.raw.rsi = updated.rsi;
102        self.raw.rdi = updated.rdi;
103        self.raw.r8 = updated.r8;
104        self.raw.r9 = updated.r9;
105        self.raw.r10 = updated.r10;
106        self.raw.r11 = updated.r11;
107        self.raw.r12 = updated.r12;
108        self.raw.r13 = updated.r13;
109        self.raw.r14 = updated.r14;
110        self.raw.r15 = updated.r15;
111        self.raw.rip = updated.rip;
112        self.raw.rflags = updated.rflags;
113    }
114
115    /// Returns the saved instruction pointer.
116    pub const fn ip(&self) -> usize {
117        self.raw.rip as usize
118    }
119
120    /// Sets the saved instruction pointer.
121    pub const fn set_ip(&mut self, ip: usize) {
122        self.raw.rip = ip as u64;
123    }
124
125    /// Creates the typed view at the assembly boundary.
126    ///
127    /// # Safety
128    ///
129    /// `raw` must be the uniquely borrowed, live kernel-origin frame built by
130    /// the x86 trap entry and must remain valid for `'a`.
131    unsafe fn from_raw(raw: &'a mut RawTrapFrame) -> Self {
132        debug_assert_eq!(raw.cs & 0b11, 0);
133        Self {
134            raw,
135            _not_send: core::marker::PhantomData,
136        }
137    }
138}
139
140impl core::fmt::Debug for KernelTrapFrame<'_> {
141    fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
142        self.snapshot().fmt(formatter)
143    }
144}
145
146core::arch::global_asm!(
147    include_str!("trap.S"),
148    trapframe_size = const core::mem::size_of::<TrapFrame>(),
149    kernel_stack_pointer_offset = const core::mem::size_of::<TrapFrame>()
150        + 2 * core::mem::size_of::<u64>(),
151    UDATA = const gdt::UDATA.0,
152    UCODE64 = const gdt::UCODE64.0,
153    SYSCALL_VECTOR = const LEGACY_SYSCALL_VECTOR,
154);
155
156pub(super) const LEGACY_SYSCALL_VECTOR: u8 = 0x80;
157pub(super) const IRQ_VECTOR_START: u8 = 0x20;
158pub(super) const IRQ_VECTOR_END: u8 = 0xff;
159
160fn handle_page_fault(tf: &mut KernelTrapFrame<'_>) {
161    let access_flags = err_code_to_flags(tf.raw.error_code)
162        .unwrap_or_else(|e| panic!("Invalid #PF error code: {:#x}", e));
163    let vaddr = va!(unsafe { cr2() });
164    #[cfg(feature = "exception-table")]
165    {
166        let mut updated = tf.snapshot();
167        if updated.fixup_nofault_exception() {
168            tf.apply_registers(&updated);
169            return;
170        }
171    }
172    if crate::trap::call_page_fault_handler_with_parent_irqs(
173        vaddr,
174        access_flags,
175        RFlags::from_bits_truncate(tf.raw.rflags).contains(RFlags::INTERRUPT_FLAG),
176    ) {
177        return;
178    }
179    #[cfg(feature = "exception-table")]
180    {
181        let mut updated = tf.snapshot();
182        if updated.fixup_exception() {
183            tf.apply_registers(&updated);
184            return;
185        }
186    }
187    let snapshot = tf.snapshot();
188    let bt = snapshot.backtrace();
189    panic!(
190        "Unhandled #PF @ {:#x}, fault_vaddr={:#x}, error_code={:#x} ({:?}):\n{:#x?}\n{}",
191        tf.raw.rip,
192        vaddr,
193        tf.raw.error_code,
194        access_flags,
195        snapshot,
196        bt.kind("trap")
197    );
198}
199
200fn handle_breakpoint(tf: &mut KernelTrapFrame<'_>) {
201    debug!("#BP @ {:#x} ", tf.raw.rip);
202    let _ = crate::trap::breakpoint_handler(tf);
203}
204
205fn handle_debug(tf: &mut KernelTrapFrame<'_>) {
206    debug!("#DB @ {:#x} ", tf.raw.rip);
207    if crate::trap::debug_handler(tf) {
208        return;
209    }
210    // Kernel-mode #DB was not claimed by any handler.
211    // Unclaimed user-mode #DB is routed through the user-space exception loop
212    // (.Ltrap_user → .Lexit_user in trap.S), so `x86_trap_handler` is only
213    // reached for kernel-mode traps. An unhandled kernel #DB is a fatal
214    // condition: if resumed the CPU re-executes the faulting instruction,
215    // likely looping into a triple fault.
216    warn!("Unhandled kernel #DB @ {:#x}", tf.raw.rip);
217    let snapshot = tf.snapshot();
218    let bt = snapshot.backtrace();
219    panic!(
220        "Unhandled #DB @ {:#x}, error_code={:#x}:\n{:#x?}\n{}",
221        tf.raw.rip,
222        tf.raw.error_code,
223        snapshot,
224        bt.kind("trap")
225    );
226}
227
228#[unsafe(no_mangle)]
229unsafe extern "C" fn x86_trap_handler(raw: *mut RawTrapFrame) {
230    // SAFETY: every x86 trap vector allocates one complete, exclusively owned
231    // frame and passes its aligned stack address in the C argument register.
232    let raw = unsafe { &mut *raw };
233    let mut tf = unsafe { KernelTrapFrame::from_raw(raw) };
234    match tf.raw.vector as u8 {
235        PAGE_FAULT_VECTOR => handle_page_fault(&mut tf),
236        BREAKPOINT_VECTOR => handle_breakpoint(&mut tf),
237        DEBUG_VECTOR => handle_debug(&mut tf),
238        GENERAL_PROTECTION_FAULT_VECTOR => {
239            let snapshot = tf.snapshot();
240            let bt = snapshot.backtrace();
241            panic!(
242                "#GP @ {:#x}, error_code={:#x}:\n{:#x?}\n{}",
243                tf.raw.rip,
244                tf.raw.error_code,
245                snapshot,
246                bt.kind("trap")
247            );
248        }
249        IRQ_VECTOR_START..=IRQ_VECTOR_END => {
250            crate::trap::dispatch_irq(tf.raw.vector as _, crate::trap::TrapOrigin::Kernel);
251        }
252        _ => {
253            let snapshot = tf.snapshot();
254            let bt = snapshot.backtrace();
255            panic!(
256                "Unhandled exception {} ({}, error_code={:#x}) @ {:#x}:\n{:#x?}\n{}",
257                tf.raw.vector,
258                vec_to_str(tf.raw.vector),
259                tf.raw.error_code,
260                tf.raw.rip,
261                snapshot,
262                bt.kind("trap")
263            );
264        }
265    }
266}
267
268#[unsafe(no_mangle)]
269unsafe extern "C" fn x86_double_fault_handler(raw: *mut RawTrapFrame) -> ! {
270    let fault_vaddr = unsafe { cr2() };
271    // SAFETY: vector 8 enters through its dedicated IST stack, builds the same
272    // register prefix as every other x86 trap, and never returns or aliases it.
273    let raw = unsafe { &*raw };
274    panic!(
275        "Fatal #DF @ {:#x}, fault_vaddr={:#x}, error_code={:#x}, cs={:#x}, rflags={:#x}",
276        raw.rip, fault_vaddr, raw.error_code, raw.cs, raw.rflags,
277    );
278}
279
280fn vec_to_str(vec: u64) -> &'static str {
281    if vec < 32 {
282        EXCEPTIONS[vec as usize].mnemonic
283    } else {
284        "Unknown"
285    }
286}
287
288pub(super) fn err_code_to_flags(err_code: u64) -> Result<PageFaultFlags, u64> {
289    let code = PageFaultErrorCode::from_bits_truncate(err_code);
290    let reserved_bits = (PageFaultErrorCode::CAUSED_BY_WRITE
291        | PageFaultErrorCode::USER_MODE
292        | PageFaultErrorCode::INSTRUCTION_FETCH
293        | PageFaultErrorCode::PROTECTION_VIOLATION)
294        .complement();
295    if code.intersects(reserved_bits) {
296        Err(err_code)
297    } else {
298        let mut flags = PageFaultFlags::empty();
299        if code.contains(PageFaultErrorCode::CAUSED_BY_WRITE) {
300            flags |= PageFaultFlags::WRITE;
301        } else {
302            flags |= PageFaultFlags::READ;
303        }
304        if code.contains(PageFaultErrorCode::USER_MODE) {
305            flags |= PageFaultFlags::USER;
306        }
307        if code.contains(PageFaultErrorCode::INSTRUCTION_FETCH) {
308            flags |= PageFaultFlags::EXECUTE;
309        }
310        Ok(flags)
311    }
312}
ax_cpu/x86_64/trap.rs

ax_cpu/x86_64/
trap.rs
