ax_cpu/aarch64/
init.rs

1//! Helper functions to initialize the CPU states on systems bootstrapping.
2
3use aarch64_cpu::{asm::barrier, registers::*};
4use ax_memory_addr::PhysAddr;
5
6use super::paging::{ADDRESS_BITS, MAIR_VALUE};
7
8/// Swtich current exception level to EL1.
9///
10/// It usually used in the system booting process, where the startup code is
11/// running in EL2 or EL3. Besides, the stack is not available and the MMU is
12/// not enabled.
13///
14/// # Safety
15///
16/// This function is unsafe as it changes the CPU mode.
17pub unsafe fn switch_to_el1() {
18    SPSel.write(SPSel::SP::ELx);
19    SP_EL0.set(0);
20    let current_el = CurrentEL.read(CurrentEL::EL);
21    if current_el >= 2 {
22        if current_el == 3 {
23            // Set EL2 to 64bit and enable the HVC instruction.
24            SCR_EL3.write(
25                SCR_EL3::NS::NonSecure + SCR_EL3::HCE::HvcEnabled + SCR_EL3::RW::NextELIsAarch64,
26            );
27            // Set the return address and exception level.
28            SPSR_EL3.write(
29                SPSR_EL3::M::EL1h
30                    + SPSR_EL3::D::Masked
31                    + SPSR_EL3::A::Masked
32                    + SPSR_EL3::I::Masked
33                    + SPSR_EL3::F::Masked,
34            );
35            ELR_EL3.set(LR.get());
36        }
37        // Disable EL1 timer traps and the timer offset.
38        CNTHCTL_EL2.modify(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET);
39        CNTVOFF_EL2.set(0);
40        // Set EL1 to 64bit.
41        HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64);
42        // Set the return address and exception level.
43        SPSR_EL2.write(
44            SPSR_EL2::M::EL1h
45                + SPSR_EL2::D::Masked
46                + SPSR_EL2::A::Masked
47                + SPSR_EL2::I::Masked
48                + SPSR_EL2::F::Masked,
49        );
50        SP_EL1.set(SP.get());
51        ELR_EL2.set(LR.get());
52        aarch64_cpu::asm::eret();
53    }
54}
55
56/// Configures and enables the MMU on the current CPU.
57///
58/// It first sets `MAIR_EL1`, `TCR_EL1`, `TTBR0_EL1`, `TTBR1_EL1` registers to
59/// the conventional values, and then enables the MMU and caches by setting
60/// `SCTLR_EL1`.
61///
62/// # Safety
63///
64/// This function is unsafe as it changes the address translation configuration.
65pub unsafe fn init_mmu(root_paddr: PhysAddr) {
66    MAIR_EL1.set(MAIR_VALUE);
67
68    // Enable TTBR0 and TTBR1 walks, page size = 4K, vaddr size = 48 bits, paddr size = 48 bits.
69    let tcr_flags0 = TCR_EL1::EPD0::EnableTTBR0Walks
70        + TCR_EL1::TG0::KiB_4
71        + TCR_EL1::SH0::Inner
72        + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable
73        + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable
74        + TCR_EL1::T0SZ.val((64 - ADDRESS_BITS) as u64);
75    let tcr_flags1 = TCR_EL1::EPD1::EnableTTBR1Walks
76        + TCR_EL1::TG1::KiB_4
77        + TCR_EL1::SH1::Inner
78        + TCR_EL1::ORGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable
79        + TCR_EL1::IRGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable
80        + TCR_EL1::T1SZ.val((64 - ADDRESS_BITS) as u64);
81    let asid_size = if ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::ASIDBits) == 2 {
82        TCR_EL1::AS::ASID16Bits
83    } else {
84        TCR_EL1::AS::ASID8Bits
85    };
86    TCR_EL1.write(TCR_EL1::IPS::Bits_48 + asid_size + tcr_flags0 + tcr_flags1);
87    barrier::isb(barrier::SY);
88
89    // Set both TTBR0 and TTBR1
90    let root_paddr = root_paddr.as_usize() as u64;
91    TTBR0_EL1.set(root_paddr);
92    TTBR1_EL1.set(root_paddr);
93
94    // Flush the entire TLB
95    crate::asm::flush_tlb(None);
96
97    // Enable the MMU, I/D cache, and EL0-accessible cache instructions.
98    // UCT/DZE/UCI let userspace execute `MRS CTR_EL0`, `DC ZVA`, and
99    // `DC CVAU` / `IC IVAU`. musl, glibc, and Mesa emit these
100    // unconditionally during early process startup. Without them, the
101    // first userspace cache-line lookup traps as EC=0x18 and the process
102    // dies with SIGTRAP before reaching `main()`.
103    SCTLR_EL1.modify(
104        SCTLR_EL1::M::Enable
105            + SCTLR_EL1::C::Cacheable
106            + SCTLR_EL1::I::Cacheable
107            + SCTLR_EL1::UCT::DontTrap
108            + SCTLR_EL1::DZE::DontTrap
109            + SCTLR_EL1::UCI::DontTrap,
110    );
111    // Disable SPAN (bit 23; not exposed as a named field by aarch64-cpu).
112    SCTLR_EL1.set(SCTLR_EL1.get() | (1 << 23));
113    barrier::isb(barrier::SY);
114}
115
116/// Initializes trap handling on the current CPU.
117///
118/// In detail, it initializes the exception vector, and sets `TTBR0_EL1` to 0 to
119/// block low address access.
120pub fn init_trap() {
121    #[cfg(feature = "exception-table")]
122    crate::exception_table::init_exception_table();
123    #[cfg(feature = "uspace")]
124    {
125        CNTKCTL_EL1.modify(CNTKCTL_EL1::EL0VCTEN::TrappedNone + CNTKCTL_EL1::EL0PCTEN::TrappedNone);
126        // Start this CPU's free-running PMU cycle counter and let EL0 read it, so
127        // `PMCCNTR_EL0` yields real cycle counts at both EL1 and EL0. This is the
128        // exact-frequency oracle the DVFS calibration reads in-kernel and that
129        // `cpuprobe`'s `mhz_pmc` reads from userspace (both were 0/trapping before,
130        // because these registers were only ever set on the lazy perf_event_open
131        // path). Guarded on PMUv3 being present; a live system-wide `perf stat -e
132        // cycles` may momentarily reset/disable this shared counter, which is fine
133        // for a boot/idle calibration.
134        if crate::pmu::probe().is_some() {
135            crate::pmu::init_cpu();
136            crate::pmu::cycles::configure(false, false);
137            crate::pmu::cycles::enable();
138        }
139        barrier::isb(barrier::SY);
140    }
141    unsafe extern "C" {
142        fn exception_vector_base();
143    }
144    unsafe {
145        crate::asm::write_exception_vector_base(exception_vector_base as *const () as usize);
146        crate::asm::write_user_page_table(0.into());
147    }
148}
ax_cpu/aarch64/init.rs

ax_cpu/aarch64/
init.rs
