ax_cpu/arch/x86_64/
trap.rs

1use x86::{controlregs::cr2, irq::*};
2use x86_64::{registers::rflags::RFlags, structures::idt::PageFaultErrorCode};
3
4use super::context::TrapFrame;
5use crate::{TrapOrigin, trap::PageFaultFlags};
6
7// In 64-bit mode the CPU saves SS:RSP even for same-privilege traps. Use
8// the complete hardware return frame, including when IST changes the stack.
9// Linux arch/x86/entry/entry_64.S, restore_regs_and_return_to_kernel, relies
10// on the same SS:RSP contract. A frame-address calculation loses alignment
11// and IST information and is not the interrupted stack pointer.
12type RawTrapFrame = TrapFrame;
13
14/// Lifetime-bound view of a kernel-origin x86 trap frame.
15///
16/// The view deliberately provides no mutable dereference to the assembly
17/// image. Probe/debug integrations can edit a copy and apply it through
18/// [`Self::apply_registers`], which preserves the trap origin and vector
19/// metadata.
20pub struct KernelTrapFrame<'a> {
21    raw: &'a mut RawTrapFrame,
22    _not_send: core::marker::PhantomData<*mut ()>,
23}
24
25impl<'a> KernelTrapFrame<'a> {
26    /// Returns the privilege domain represented by this view.
27    pub const fn origin(&self) -> TrapOrigin {
28        TrapOrigin::Kernel
29    }
30
31    /// Copies the saved register image for inspection or probe emulation.
32    pub fn snapshot(&self) -> TrapFrame {
33        *self.raw
34    }
35
36    /// Applies task-register changes while preserving trap-origin metadata.
37    pub fn apply_registers(&mut self, updated: &TrapFrame) {
38        self.raw.regs = updated.regs;
39        self.raw.rip = updated.rip;
40        self.raw.rflags = updated.rflags;
41    }
42
43    /// Returns the saved instruction pointer.
44    pub const fn ip(&self) -> usize {
45        self.raw.rip as usize
46    }
47
48    /// Sets the saved instruction pointer.
49    pub const fn set_ip(&mut self, ip: usize) {
50        self.raw.rip = ip as u64;
51    }
52
53    /// Creates the typed view at the assembly boundary.
54    ///
55    /// # Safety
56    ///
57    /// `raw` must be the uniquely borrowed, live kernel-origin frame built by
58    /// the x86 trap entry and must remain valid for `'a`.
59    unsafe fn from_raw(raw: &'a mut RawTrapFrame) -> Self {
60        debug_assert_eq!(raw.cs & 0b11, 0);
61        Self {
62            raw,
63            _not_send: core::marker::PhantomData,
64        }
65    }
66}
67
68impl core::fmt::Debug for KernelTrapFrame<'_> {
69    fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
70        self.snapshot().fmt(formatter)
71    }
72}
73
74pub(super) const LEGACY_SYSCALL_VECTOR: u8 = 0x80;
75pub(super) const IRQ_VECTOR_START: u8 = 0x20;
76pub(super) const IRQ_VECTOR_END: u8 = 0xff;
77
78fn handle_page_fault(tf: &mut KernelTrapFrame<'_>) {
79    let access_flags = err_code_to_flags(tf.raw.error_code)
80        .unwrap_or_else(|e| panic!("Invalid #PF error code: {:#x}", e));
81    let vaddr = va!(unsafe { cr2() });
82    #[cfg(feature = "exception-table")]
83    {
84        let mut updated = tf.snapshot();
85        if updated.fixup_nofault_exception() {
86            tf.apply_registers(&updated);
87            return;
88        }
89    }
90    if crate::trap::call_page_fault_handler_with_parent_irqs(
91        vaddr,
92        access_flags,
93        RFlags::from_bits_truncate(tf.raw.rflags).contains(RFlags::INTERRUPT_FLAG),
94    ) {
95        return;
96    }
97    #[cfg(feature = "exception-table")]
98    {
99        let mut updated = tf.snapshot();
100        if updated.fixup_exception() {
101            tf.apply_registers(&updated);
102            return;
103        }
104    }
105    let snapshot = tf.snapshot();
106    let bt = crate::trap::diagnostics::BacktraceDisplay(snapshot.backtrace_registers());
107    panic!(
108        "Unhandled #PF @ {:#x}, fault_vaddr={:#x}, error_code={:#x} ({:?}):\n{:#x?}\n{}",
109        tf.raw.rip, vaddr, tf.raw.error_code, access_flags, snapshot, bt
110    );
111}
112
113fn handle_breakpoint(tf: &mut KernelTrapFrame<'_>) {
114    debug!("#BP @ {:#x} ", tf.raw.rip);
115    let _ = crate::trap::breakpoint_handler(tf);
116}
117
118fn handle_debug(tf: &mut KernelTrapFrame<'_>) {
119    debug!("#DB @ {:#x} ", tf.raw.rip);
120    if crate::trap::debug_handler(tf) {
121        return;
122    }
123    // Kernel-mode #DB was not claimed by any handler.
124    // Unclaimed user-mode #DB is routed through the user-space exception loop
125    // (.Ltrap_user → .Lexit_user in trap.S), so `x86_trap_handler` is only
126    // reached for kernel-mode traps. An unhandled kernel #DB is a fatal
127    // condition: if resumed the CPU re-executes the faulting instruction,
128    // likely looping into a triple fault.
129    warn!("Unhandled kernel #DB @ {:#x}", tf.raw.rip);
130    let snapshot = tf.snapshot();
131    let bt = crate::trap::diagnostics::BacktraceDisplay(snapshot.backtrace_registers());
132    panic!(
133        "Unhandled #DB @ {:#x}, error_code={:#x}:\n{:#x?}\n{}",
134        tf.raw.rip, tf.raw.error_code, snapshot, bt
135    );
136}
137
138#[unsafe(no_mangle)]
139unsafe extern "C" fn x86_trap_handler(raw: *mut RawTrapFrame) {
140    // SAFETY: every x86 trap vector allocates one complete, exclusively owned
141    // frame and passes its aligned stack address in the C argument register.
142    let raw = unsafe { &mut *raw };
143    let mut tf = unsafe { KernelTrapFrame::from_raw(raw) };
144    match tf.raw.vector as u8 {
145        PAGE_FAULT_VECTOR => handle_page_fault(&mut tf),
146        BREAKPOINT_VECTOR => handle_breakpoint(&mut tf),
147        DEBUG_VECTOR => handle_debug(&mut tf),
148        GENERAL_PROTECTION_FAULT_VECTOR => {
149            let snapshot = tf.snapshot();
150            let bt = crate::trap::diagnostics::BacktraceDisplay(snapshot.backtrace_registers());
151            panic!(
152                "#GP @ {:#x}, error_code={:#x}:\n{:#x?}\n{}",
153                tf.raw.rip, tf.raw.error_code, snapshot, bt
154            );
155        }
156        IRQ_VECTOR_START..=IRQ_VECTOR_END => {
157            crate::trap::dispatch_irq(
158                tf.raw.vector as _,
159                crate::trap::TrapOrigin::Kernel,
160                Some(tf.snapshot().interrupted_context()),
161            );
162        }
163        _ => {
164            let snapshot = tf.snapshot();
165            let bt = crate::trap::diagnostics::BacktraceDisplay(snapshot.backtrace_registers());
166            panic!(
167                "Unhandled exception {} ({}, error_code={:#x}) @ {:#x}:\n{:#x?}\n{}",
168                tf.raw.vector,
169                vec_to_str(tf.raw.vector),
170                tf.raw.error_code,
171                tf.raw.rip,
172                snapshot,
173                bt
174            );
175        }
176    }
177}
178
179#[unsafe(no_mangle)]
180unsafe extern "C" fn x86_double_fault_handler(raw: *mut RawTrapFrame) -> ! {
181    let fault_vaddr = unsafe { cr2() };
182    // SAFETY: vector 8 enters through its dedicated IST stack, builds the same
183    // register prefix as every other x86 trap, and never returns or aliases it.
184    let raw = unsafe { &*raw };
185    panic!(
186        "Fatal #DF @ {:#x}, fault_vaddr={:#x}, error_code={:#x}, cs={:#x}, rflags={:#x}",
187        raw.rip, fault_vaddr, raw.error_code, raw.cs, raw.rflags,
188    );
189}
190
191fn vec_to_str(vec: u64) -> &'static str {
192    if vec < 32 {
193        EXCEPTIONS[vec as usize].mnemonic
194    } else {
195        "Unknown"
196    }
197}
198
199pub(super) fn err_code_to_flags(err_code: u64) -> Result<PageFaultFlags, u64> {
200    let code = PageFaultErrorCode::from_bits_truncate(err_code);
201    let reserved_bits = (PageFaultErrorCode::CAUSED_BY_WRITE
202        | PageFaultErrorCode::USER_MODE
203        | PageFaultErrorCode::INSTRUCTION_FETCH
204        | PageFaultErrorCode::PROTECTION_VIOLATION)
205        .complement();
206    if code.intersects(reserved_bits) {
207        Err(err_code)
208    } else {
209        let mut flags = PageFaultFlags::empty();
210        if code.contains(PageFaultErrorCode::CAUSED_BY_WRITE) {
211            flags |= PageFaultFlags::WRITE;
212        } else {
213            flags |= PageFaultFlags::READ;
214        }
215        if code.contains(PageFaultErrorCode::USER_MODE) {
216            flags |= PageFaultFlags::USER;
217        }
218        if code.contains(PageFaultErrorCode::INSTRUCTION_FETCH) {
219            flags |= PageFaultFlags::EXECUTE;
220        }
221        Ok(flags)
222    }
223}
ax_cpu/arch/x86_64/trap.rs

ax_cpu/arch/x86_64/
trap.rs
