ax_cpu/arch/aarch64/boot.rs
1// SPDX-License-Identifier: Apache-2.0 AND MPL-2.0
2// Exception-level handoff migrated from someboot; original author:
3// 周睿 <zrufo747@outlook.com>. Native trap initialization retains its Apache origin.
4//! Helper functions to initialize the CPU states on systems bootstrapping.
5
6use aarch64_cpu::{asm::barrier, registers::*};
7
8/// Transfers execution to a mapped entry on a new stack without returning.
9///
10/// # Safety
11/// The target must be valid executable code accepting a fresh boot handoff.
12/// The stack must be aligned to 16 bytes, writable and exclusively owned.
13/// IRQ state, translation, CPU anchor and TLS must satisfy the target contract;
14/// no live stack-owned resource may require destruction after this transfer.
15#[unsafe(naked)]
16pub unsafe extern "C" fn jump_to(_entry: crate::VirtAddr, _stack: crate::VirtAddr) -> ! {
17 core::arch::naked_asm!("mov sp, x1", "br x0");
18}
19
20impl super::mmu::El1 {
21 /// Installs native EL1 vectors and removes the boot lower-address mapping.
22 ///
23 /// # Safety
24 /// Execute at EL1 with IRQs masked, a valid kernel stack and kernel CPU
25 /// anchor installed. No live mapping may depend on the boot TTBR0_EL1 root.
26 pub unsafe fn init_trap() {
27 #[cfg(feature = "uspace")]
28 CNTKCTL_EL1.modify(CNTKCTL_EL1::EL0VCTEN::TrappedNone + CNTKCTL_EL1::EL0PCTEN::TrappedNone);
29 unsafe extern "C" {
30 fn __ax_cpu_vector_el1();
31 }
32 VBAR_EL1.set(__ax_cpu_vector_el1 as *const () as u64);
33 TTBR0_EL1.set(0);
34 barrier::isb(barrier::SY);
35 }
36}
37
38impl super::mmu::El2 {
39 /// Installs native non-VHE EL2 vectors without modifying a guest EL1 bank.
40 ///
41 /// # Safety
42 /// Execute at non-VHE EL2 with IRQs masked and a valid kernel stack and CPU
43 /// anchor installed. The mapped CPU vector text must remain executable.
44 pub unsafe fn init_trap() {
45 unsafe extern "C" {
46 fn __ax_cpu_vector_el2();
47 }
48 VBAR_EL2.set(__ax_cpu_vector_el2 as *const () as u64);
49 barrier::isb(barrier::SY);
50 }
51}
52
53impl super::mmu::El1 {
54 /// Installs the early vector using the boot owner's exception policy.
55 ///
56 /// # Safety
57 /// Execute at the corresponding EL with a valid stack and IRQs masked.
58 /// The vector and boot handler must remain mapped and callable; runtime
59 /// TLS and CPU-local services need not be initialized.
60 pub unsafe fn init_boot_trap() {
61 unsafe extern "C" {
62 fn __ax_cpu_boot_vector_el1();
63 }
64 VBAR_EL1.set(__ax_cpu_boot_vector_el1 as *const () as u64);
65 barrier::isb(barrier::SY);
66 }
67 /// Reads the currently installed vector base for this exception level.
68 pub fn vector_base() -> crate::VirtAddr {
69 crate::VirtAddr::from_usize(VBAR_EL1.get() as usize)
70 }
71}
72
73impl super::mmu::El2 {
74 /// Installs the early vector using the boot owner's exception policy.
75 ///
76 /// # Safety
77 /// Execute at the corresponding EL with a valid stack and IRQs masked.
78 /// The vector and boot handler must remain mapped and callable; runtime
79 /// TLS and CPU-local services need not be initialized.
80 pub unsafe fn init_boot_trap() {
81 unsafe extern "C" {
82 fn __ax_cpu_boot_vector_el2();
83 }
84 VBAR_EL2.set(__ax_cpu_boot_vector_el2 as *const () as u64);
85 barrier::isb(barrier::SY);
86 }
87 /// Reads the currently installed vector base for this exception level.
88 pub fn vector_base() -> crate::VirtAddr {
89 crate::VirtAddr::from_usize(VBAR_EL2.get() as usize)
90 }
91}
92
93/// Selects the current exception level's dedicated stack and clears SP_EL0.
94///
95/// # Safety
96/// The current EL stack must already be valid; no live owner may rely on SP_EL0.
97pub unsafe fn select_privileged_stack() {
98 SPSel.write(SPSel::SP::ELx);
99 SP_EL0.set(0);
100}
101
102// The argument is assigned to x0 in the final machine window. In particular,
103// no compiler-generated call may clobber a secondary CPU's handoff pointer.
104unsafe fn exception_return(argument: usize) -> ! {
105 // SAFETY: the mode-specific caller installed the return PC, stack and PSTATE.
106 unsafe {
107 core::arch::asm!("isb", "eret", in("x0") argument, options(noreturn, nostack));
108 }
109}
110
111impl super::mmu::El1 {
112 /// Transfers from EL2 or EL3 to a non-secure AArch64 EL1 boot entry.
113 /// Physical timer access is enabled and the virtual counter offset is zero.
114 ///
115 /// # Safety
116 /// Execute at EL2/EL3 before admitting other owners of the lower register
117 /// banks. Entry and its 16-byte-aligned exclusive stack must be accessible
118 /// at EL1 with the currently installed translation state. Entry accepts its
119 /// sole argument in x0. No live resource may need destruction on this stack.
120 pub unsafe fn enter(entry: crate::VirtAddr, stack: crate::VirtAddr, argument: usize) -> ! {
121 CNTHCTL_EL2.modify(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);
122 CNTVOFF_EL2.set(0);
123 HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64);
124 SP_EL1.set(stack.as_usize() as u64);
125 if CurrentEL.read(CurrentEL::EL) == 3 {
126 SCR_EL3.write(
127 SCR_EL3::NS::NonSecure + SCR_EL3::HCE::HvcEnabled + SCR_EL3::RW::NextELIsAarch64,
128 );
129 SPSR_EL3.write(
130 SPSR_EL3::M::EL1h
131 + SPSR_EL3::D::Masked
132 + SPSR_EL3::A::Masked
133 + SPSR_EL3::I::Masked
134 + SPSR_EL3::F::Masked,
135 );
136 ELR_EL3.set(entry.as_usize() as u64);
137 } else {
138 SPSR_EL2.write(
139 SPSR_EL2::M::EL1h
140 + SPSR_EL2::D::Masked
141 + SPSR_EL2::A::Masked
142 + SPSR_EL2::I::Masked
143 + SPSR_EL2::F::Masked,
144 );
145 ELR_EL2.set(entry.as_usize() as u64);
146 }
147 // SAFETY: the selected return bank retains the target, stack and masked PSTATE.
148 unsafe { exception_return(argument) }
149 }
150}
151
152impl super::mmu::El2 {
153 /// Transfers from EL3 to a non-secure AArch64 EL2 boot entry with IRQs masked.
154 ///
155 /// # Safety
156 /// Execute at EL3 with exclusive ownership of the lower exception registers.
157 /// Entry and its 16-byte-aligned exclusive stack must be accessible at EL2;
158 /// entry accepts its argument in x0. No live stack resource needs destruction.
159 pub unsafe fn enter(entry: crate::VirtAddr, stack: crate::VirtAddr, argument: usize) -> ! {
160 SCR_EL3.write(
161 SCR_EL3::NS::NonSecure + SCR_EL3::HCE::HvcEnabled + SCR_EL3::RW::NextELIsAarch64,
162 );
163 SPSR_EL3.write(
164 SPSR_EL3::M::EL2h
165 + SPSR_EL3::D::Masked
166 + SPSR_EL3::A::Masked
167 + SPSR_EL3::I::Masked
168 + SPSR_EL3::F::Masked,
169 );
170 ELR_EL3.set(entry.as_usize() as u64);
171 SP_EL2.set(stack.as_usize() as u64);
172 // SAFETY: all exception-return state is installed for the owned EL2 entry.
173 unsafe { exception_return(argument) }
174 }
175}