aarch64_rt/
lib.rs

1// Copyright 2025 The aarch64-rt Authors.
2// This project is dual-licensed under Apache 2.0 and MIT terms.
3// See LICENSE-APACHE and LICENSE-MIT for details.
4
5//! Startup code for aarch64 Cortex-A processors.
6
7#![no_std]
8#![deny(clippy::undocumented_unsafe_blocks)]
9#![deny(unsafe_op_in_unsafe_fn)]
10
11#[cfg(any(
12    all(feature = "el1", feature = "el2"),
13    all(feature = "el1", feature = "el3"),
14    all(feature = "el2", feature = "el3"),
15))]
16compile_error!("Only one `el` feature may be enabled at once.");
17
18#[cfg(feature = "exceptions")]
19use core::arch::asm;
20use core::arch::global_asm;
21
22global_asm!(include_str!("entry.S"));
23
24#[cfg(not(feature = "initial-pagetable"))]
25global_asm!(include_str!("dummy_enable_mmu.S"),);
26#[cfg(all(feature = "el1", feature = "initial-pagetable"))]
27global_asm!(include_str!("el1_enable_mmu.S"),);
28#[cfg(all(feature = "el2", feature = "initial-pagetable"))]
29global_asm!(include_str!("el2_enable_mmu.S"));
30#[cfg(all(feature = "el3", feature = "initial-pagetable"))]
31global_asm!(include_str!("el3_enable_mmu.S"));
32
33#[cfg(feature = "exceptions")]
34global_asm!(include_str!("exceptions.S"));
35
36unsafe extern "C" {
37    /// An assembly entry point for secondary cores.
38    ///
39    /// It will enable the MMU, disable trapping of floating point instructions, initialise the
40    /// stack pointer to `stack_end` and then jump to the function pointer at the bottom of the
41    /// stack with the u64 value second on the stack as a parameter.
42    pub unsafe fn secondary_entry(stack_end: *mut u64) -> !;
43}
44
45/// Sets the appropriate vbar to point to our `vector_table`, if the `exceptions` feature is
46/// enabled.
47#[unsafe(no_mangle)]
48extern "C" fn set_exception_vector() {
49    // SAFETY: We provide a valid vector table.
50    #[cfg(all(feature = "el1", feature = "exceptions"))]
51    unsafe {
52        asm!(
53            "adr x9, vector_table_el1",
54            "msr vbar_el1, x9",
55            options(nomem, nostack),
56            out("x9") _,
57        );
58    }
59    // SAFETY: We provide a valid vector table.
60    #[cfg(all(feature = "el2", feature = "exceptions"))]
61    unsafe {
62        asm!(
63            "adr x9, vector_table_el2",
64            "msr vbar_el2, x9",
65            options(nomem, nostack),
66            out("x9") _,
67        );
68    }
69    // SAFETY: We provide a valid vector table.
70    #[cfg(all(feature = "el3", feature = "exceptions"))]
71    unsafe {
72        asm!(
73            "adr x9, vector_table_el3",
74            "msr vbar_el3, x9",
75            options(nomem, nostack),
76            out("x9") _,
77        );
78    }
79}
80
81#[unsafe(no_mangle)]
82extern "C" fn rust_entry(arg0: u64, arg1: u64, arg2: u64, arg3: u64) -> ! {
83    set_exception_vector();
84    __main(arg0, arg1, arg2, arg3)
85}
86
87unsafe extern "Rust" {
88    /// Main function provided by the application using the `main!` macro.
89    safe fn __main(arg0: u64, arg1: u64, arg2: u64, arg3: u64) -> !;
90}
91
92/// Marks the main function of the binary and reserves space for the boot stack.
93///
94/// Example:
95///
96/// ```rust
97/// use aarch64_rt::entry;
98///
99/// entry!(main);
100/// fn main() -> ! {
101///     info!("Hello world");
102/// }
103/// ```
104///
105/// 40 pages (160 KiB) is reserved for the boot stack by default; a different size may be configured
106/// by passing the number of pages as a second argument to the macro, e.g. `entry!(main, 10);` to
107/// reserve only 10 pages.
108#[macro_export]
109macro_rules! entry {
110    ($name:path) => {
111        entry!($name, 40);
112    };
113    ($name:path, $boot_stack_pages:expr) => {
114        #[unsafe(export_name = "boot_stack")]
115        #[unsafe(link_section = ".stack.boot_stack")]
116        static mut __BOOT_STACK: $crate::Stack<$boot_stack_pages> = $crate::Stack::new();
117
118        // Export a symbol with a name matching the extern declaration above.
119        #[unsafe(export_name = "__main")]
120        fn __main(arg0: u64, arg1: u64, arg2: u64, arg3: u64) -> ! {
121            // Ensure that the main function provided by the application has the correct type.
122            $name(arg0, arg1, arg2, arg3)
123        }
124    };
125}
126
127/// Provides an initial pagetable which can be used before any Rust code is run.
128///
129/// The `initial-pagetable` feature must be enabled for this to be used.
130#[cfg(feature = "initial-pagetable")]
131#[macro_export]
132macro_rules! initial_pagetable {
133    ($value:expr) => {
134        #[unsafe(export_name = "initial_pagetable")]
135        #[unsafe(link_section = ".rodata.initial_pagetable")]
136        static INITIAL_PAGETABLE: $crate::InitialPagetable = $value;
137    };
138}
139
140/// A hardcoded pagetable.
141#[repr(C, align(4096))]
142pub struct InitialPagetable(pub [usize; 512]);
143
144/// A stack for some CPU core.
145///
146/// This is used by the [`entry!`] macro to reserve space for the boot stack.
147#[repr(C, align(4096))]
148pub struct Stack<const NUM_PAGES: usize>([StackPage; NUM_PAGES]);
149
150impl<const NUM_PAGES: usize> Stack<NUM_PAGES> {
151    /// Creates a new zero-initialised stack.
152    pub const fn new() -> Self {
153        Self([const { StackPage::new() }; NUM_PAGES])
154    }
155}
156
157impl<const NUM_PAGES: usize> Default for Stack<NUM_PAGES> {
158    fn default() -> Self {
159        Self::new()
160    }
161}
162
163#[repr(C, align(4096))]
164struct StackPage([u8; 4096]);
165
166impl StackPage {
167    const fn new() -> Self {
168        Self([0; 4096])
169    }
170}
171
172#[cfg(feature = "psci")]
173/// Issues a PSCI CPU_ON call to start the CPU core with the given MPIDR.
174///
175/// This starts the core with an assembly entry point which will enable the MMU, disable trapping of
176/// floating point instructions, initialise the stack pointer to the given value, and then jump to
177/// the given Rust entry point function, passing it the given argument value.
178///
179/// # Safety
180///
181/// `stack` must point to a region of memory which is reserved for this core's stack. It must remain
182/// valid as long as the core is running, and there must not be any other access to it during that
183/// time. It must be mapped both for the current core to write to it (to pass initial parameters)
184/// and in the initial page table which the core being started will used, with the same memory
185/// attributes for both.
186pub unsafe fn start_core<C: smccc::Call, const N: usize>(
187    mpidr: u64,
188    stack: *mut Stack<N>,
189    rust_entry: extern "C" fn(arg: u64) -> !,
190    arg: u64,
191) -> Result<(), smccc::psci::Error> {
192    assert!(stack.is_aligned());
193    // The stack grows downwards on aarch64, so get a pointer to the end of the stack.
194    let stack_end = stack.wrapping_add(1);
195
196    // Write Rust entry point to the stack, so the assembly entry point can jump to it.
197    let params = stack_end as *mut u64;
198    // SAFETY: Our caller promised that the stack is valid and nothing else will access it.
199    unsafe {
200        *params.wrapping_sub(1) = rust_entry as _;
201        *params.wrapping_sub(2) = arg;
202    }
203    // Wait for the stores above to complete before starting the secondary CPU core.
204    dsb_st();
205
206    smccc::psci::cpu_on::<C>(mpidr, secondary_entry as _, stack_end as _)
207}
208
209/// Data synchronisation barrier that waits for stores to complete, for the full system.
210#[cfg(feature = "psci")]
211fn dsb_st() {
212    // SAFETY: A synchronisation barrier is always safe.
213    unsafe {
214        asm!("dsb st", options(nostack));
215    }
216}
aarch64_rt/lib.rs

aarch64_rt/
lib.rs
