cpu_local/
area.rs

1#[cfg(not(all(target_arch = "aarch64", not(feature = "host-test"))))]
2use core::sync::atomic::Ordering;
3use core::{
4    mem::{MaybeUninit, align_of, offset_of, size_of},
5    ptr::NonNull,
6    sync::atomic::AtomicUsize,
7};
8
9use crate::{CpuIndex, CpuLocalError, ExecutionContextHeader, preempt::PreemptionState};
10
11const fn runtime_anchor_reserved_size() -> usize {
12    64 - 6 * size_of::<usize>() - size_of::<PreemptionState>()
13}
14
15/// CPU-local scalar state shared by trap entry and context publication.
16#[repr(C, align(64))]
17pub struct CpuRuntimeAnchor {
18    current_context: AtomicUsize,
19    architecture_state: [AtomicUsize; 5],
20    preemption_state: PreemptionState,
21    reserved: [u8; runtime_anchor_reserved_size()],
22}
23
24impl CpuRuntimeAnchor {
25    const fn for_boot_context(boot_context: usize) -> Self {
26        let current_context = if cfg!(all(target_arch = "x86_64", not(feature = "host-test")))
27            || cfg!(all(
28                kernel_tls,
29                not(all(target_arch = "aarch64", not(feature = "host-test")))
30            )) {
31            boot_context
32        } else {
33            0
34        };
35        Self {
36            current_context: AtomicUsize::new(current_context),
37            architecture_state: [const { AtomicUsize::new(0) }; 5],
38            preemption_state: PreemptionState::bootstrap_disabled(),
39            reserved: [0; runtime_anchor_reserved_size()],
40        }
41    }
42
43    /// Acquires the context pointer in an anchor-backed image mode.
44    #[cfg(not(all(target_arch = "aarch64", not(feature = "host-test"))))]
45    pub(crate) fn current_context_raw(&self) -> usize {
46        self.current_context.load(Ordering::Acquire)
47    }
48
49    #[cfg(not(all(target_arch = "aarch64", not(feature = "host-test"))))]
50    pub(crate) const fn current_context_slot(&self) -> &AtomicUsize {
51        &self.current_context
52    }
53
54    #[cfg(all(target_arch = "x86_64", not(feature = "host-test")))]
55    pub(crate) const fn preemption_state(&self) -> &PreemptionState {
56        &self.preemption_state
57    }
58}
59
60/// Permanent context header used before the runtime publishes a context.
61#[repr(transparent)]
62pub struct BootContextHeader(ExecutionContextHeader);
63
64impl BootContextHeader {
65    const fn for_area(area_base: usize) -> Self {
66        Self(ExecutionContextHeader::boot(area_base))
67    }
68
69    /// Returns the permanent pinned header.
70    pub const fn header(&self) -> &ExecutionContextHeader {
71        &self.0
72    }
73}
74
75const fn area_header_reserved_size() -> usize {
76    64 - size_of::<u32>() * 2 - size_of::<usize>()
77}
78
79/// Immutable identity stored at the beginning of each initialized CPU area.
80#[repr(C, align(64))]
81pub struct CpuAreaHeader {
82    cpu_index: u32,
83    reserved_word: u32,
84    self_base: usize,
85    reserved: [u8; area_header_reserved_size()],
86}
87
88impl CpuAreaHeader {
89    const fn new(cpu_index: CpuIndex, self_base: usize) -> Self {
90        Self {
91            cpu_index: cpu_index.as_u32(),
92            reserved_word: 0,
93            self_base,
94            reserved: [0; area_header_reserved_size()],
95        }
96    }
97
98    /// Returns the logical CPU index assigned to this area.
99    #[inline(always)]
100    pub const fn cpu_index(&self) -> CpuIndex {
101        match CpuIndex::from_u32(self.cpu_index) {
102            Some(index) => index,
103            None => panic!("initialized CPU area contains the reserved CPU index"),
104        }
105    }
106
107    /// Returns the permanent runtime base recorded by this area.
108    pub const fn self_base(&self) -> usize {
109        self.self_base
110    }
111}
112
113/// Fixed three-cache-line prefix of every initialized runtime CPU area.
114#[repr(C, align(64))]
115pub struct CpuAreaPrefix {
116    header: CpuAreaHeader,
117    runtime: CpuRuntimeAnchor,
118    boot_context: BootContextHeader,
119}
120
121impl CpuAreaPrefix {
122    /// Constructs the prefix value for one exclusively owned offline area.
123    ///
124    /// # Errors
125    ///
126    /// Returns an address error when `area_base` is null, misaligned, or its
127    /// fixed boot-context address overflows.
128    pub fn initialize(cpu_index: CpuIndex, area_base: usize) -> Result<Self, CpuLocalError> {
129        validate_area_base(area_base)?;
130        area_base
131            .checked_add(CPU_AREA_BOOT_CONTEXT_OFFSET)
132            .ok_or(CpuLocalError::AddressOverflow)?;
133        Ok(Self {
134            header: CpuAreaHeader::new(cpu_index, area_base),
135            runtime: CpuRuntimeAnchor::for_boot_context(area_base + CPU_AREA_BOOT_CONTEXT_OFFSET),
136            boot_context: BootContextHeader::for_area(area_base),
137        })
138    }
139
140    /// Returns immutable area identity.
141    pub const fn header(&self) -> &CpuAreaHeader {
142        &self.header
143    }
144
145    /// Returns CPU runtime and trap state.
146    pub const fn runtime_anchor(&self) -> &CpuRuntimeAnchor {
147        &self.runtime
148    }
149
150    /// Returns the permanent boot execution-context header.
151    pub const fn boot_context(&self) -> &BootContextHeader {
152        &self.boot_context
153    }
154}
155
156/// Permanent typed reference to one fully initialized runtime CPU area.
157#[derive(Clone, Copy, Debug, Eq, PartialEq)]
158pub struct CpuAreaRef {
159    prefix: NonNull<CpuAreaPrefix>,
160}
161
162// SAFETY: the initialization contract keeps the immutable prefix and runtime
163// anchor mapped until shutdown. Mutable CPU-owned fields provide their own
164// atomic or external synchronization contracts.
165unsafe impl Send for CpuAreaRef {}
166// SAFETY: see the Send implementation; sharing this descriptor does not grant
167// mutable access to non-atomic per-CPU values.
168unsafe impl Sync for CpuAreaRef {}
169
170impl CpuAreaRef {
171    /// Reconstructs a reference from an initialized shutdown-lifetime prefix.
172    ///
173    /// # Safety
174    ///
175    /// `area_base` must point to a fully initialized [`CpuAreaPrefix`] that
176    /// remains mapped until shutdown. No caller may mutate its identity fields.
177    #[doc(hidden)]
178    pub unsafe fn from_initialized_base(area_base: usize) -> Result<Self, CpuLocalError> {
179        #[cfg(feature = "host-test")]
180        crate::register::host_test::record_initialized_area_validation();
181        validate_area_base(area_base)?;
182        let prefix = NonNull::new(area_base as *mut CpuAreaPrefix)
183            .ok_or(CpuLocalError::InvalidAreaBase { base: area_base })?;
184        // SAFETY: forwarded caller contract provides a live initialized prefix.
185        let header = unsafe { prefix.as_ref() }.header();
186        CpuIndex::from_u32(header.cpu_index).ok_or(CpuLocalError::AreaIdentityMismatch)?;
187        if header.self_base != area_base {
188            return Err(CpuLocalError::AreaIdentityMismatch);
189        }
190        let expected_boot = area_base
191            .checked_add(CPU_AREA_BOOT_CONTEXT_OFFSET)
192            .ok_or(CpuLocalError::AddressOverflow)?;
193        if unsafe { prefix.as_ref() }
194            .boot_context()
195            .header()
196            .raw_cpu_binding()
197            .map(|(boot_area, _)| boot_area)
198            != Some(area_base)
199            || expected_boot
200                != core::ptr::addr_of!(unsafe { prefix.as_ref() }.boot_context.0) as usize
201        {
202            return Err(CpuLocalError::AreaIdentityMismatch);
203        }
204        Ok(Self { prefix })
205    }
206
207    /// Reconstructs the area selected by an already validated installed base.
208    ///
209    /// # Safety
210    ///
211    /// `area_base` must be the exact base of a [`CpuAreaRef`] previously
212    /// validated and installed through the architecture register boundary.
213    /// That area must retain its immutable identity and remain mapped until
214    /// shutdown.
215    pub(crate) unsafe fn from_installed_base(area_base: usize) -> Self {
216        debug_assert!(area_base != 0);
217        debug_assert!(area_base.is_multiple_of(align_of::<CpuAreaPrefix>()));
218        // SAFETY: the caller guarantees that the installed base is non-null
219        // and still names the previously validated shutdown-lifetime prefix.
220        let prefix = unsafe { NonNull::new_unchecked(area_base as *mut CpuAreaPrefix) };
221        Self { prefix }
222    }
223
224    /// Returns this area's logical CPU index.
225    #[inline(always)]
226    pub const fn cpu_index(self) -> CpuIndex {
227        self.prefix().header().cpu_index()
228    }
229
230    /// Returns the exact runtime prefix address used as area identity.
231    #[inline(always)]
232    pub fn base(self) -> usize {
233        self.prefix.as_ptr() as usize
234    }
235
236    /// Returns the initialized fixed prefix.
237    #[inline(always)]
238    pub const fn prefix(self) -> &'static CpuAreaPrefix {
239        // SAFETY: construction requires a shutdown-lifetime mapping.
240        unsafe { self.prefix.as_ref() }
241    }
242
243    /// Returns this area's runtime/trap anchor.
244    #[inline(always)]
245    pub fn runtime_anchor(self) -> &'static CpuRuntimeAnchor {
246        self.prefix().runtime_anchor()
247    }
248}
249
250fn validate_area_base(area_base: usize) -> Result<(), CpuLocalError> {
251    if area_base == 0 || !area_base.is_multiple_of(align_of::<CpuAreaPrefix>()) {
252        Err(CpuLocalError::InvalidAreaBase { base: area_base })
253    } else {
254        Ok(())
255    }
256}
257
258/// Size in bytes of the immutable area header.
259pub const CPU_AREA_HEADER_SIZE: usize = size_of::<CpuAreaHeader>();
260/// Byte offset of CPU runtime/trap state.
261pub const CPU_AREA_RUNTIME_ANCHOR_OFFSET: usize = offset_of!(CpuAreaPrefix, runtime);
262/// Byte offset of the permanent boot execution-context header.
263pub const CPU_AREA_BOOT_CONTEXT_OFFSET: usize = offset_of!(CpuAreaPrefix, boot_context);
264/// Byte offset of the runtime self pointer.
265pub const CPU_AREA_SELF_BASE_OFFSET: usize = offset_of!(CpuAreaHeader, self_base);
266/// Byte offset of the logical CPU index.
267pub const CPU_AREA_CPU_INDEX_OFFSET: usize = offset_of!(CpuAreaHeader, cpu_index);
268/// Byte offset of the current-context slot used by anchor-backed image modes.
269pub const CPU_AREA_CURRENT_CONTEXT_OFFSET: usize =
270    CPU_AREA_RUNTIME_ANCHOR_OFFSET + offset_of!(CpuRuntimeAnchor, current_context);
271/// Byte offset of architecture-owned CPU trap state.
272pub const CPU_AREA_ARCH_STATE_OFFSET: usize =
273    CPU_AREA_RUNTIME_ANCHOR_OFFSET + offset_of!(CpuRuntimeAnchor, architecture_state);
274/// Reserved bytes available to the architecture-owned CPU trap state.
275pub const CPU_AREA_ARCH_STATE_SIZE: usize = 5 * size_of::<usize>();
276/// Byte offset of the x86_64 CPU-owned preemption word.
277pub const CPU_AREA_PREEMPTION_STATE_OFFSET: usize =
278    CPU_AREA_RUNTIME_ANCHOR_OFFSET + offset_of!(CpuRuntimeAnchor, preemption_state);
279
280const _: () = {
281    assert!(size_of::<CpuAreaHeader>() == 64);
282    assert!(align_of::<CpuAreaHeader>() == 64);
283    assert!(size_of::<CpuRuntimeAnchor>() == 64);
284    assert!(align_of::<CpuRuntimeAnchor>() == 64);
285    assert!(size_of::<BootContextHeader>() == 64);
286    assert!(align_of::<BootContextHeader>() == 64);
287    assert!(size_of::<CpuAreaPrefix>() == 192);
288    assert!(align_of::<CpuAreaPrefix>() == 64);
289    assert!(CPU_AREA_RUNTIME_ANCHOR_OFFSET == 64);
290    assert!(CPU_AREA_BOOT_CONTEXT_OFFSET == 128);
291};
292
293#[doc(hidden)]
294#[used]
295#[unsafe(no_mangle)]
296#[unsafe(link_section = ".percpu.template.header")]
297pub static mut __CPU_LOCAL_AREA_PREFIX: MaybeUninit<CpuAreaPrefix> = MaybeUninit::uninit();
298
299#[doc(hidden)]
300#[used]
301#[unsafe(no_mangle)]
302#[unsafe(link_section = ".percpu.template.end")]
303pub static __CPU_LOCAL_TEMPLATE_END: u8 = 0;
304
305#[cfg(test)]
306mod tests {
307    use super::*;
308
309    #[test]
310    fn cpu_area_ref_keeps_one_area_identity() {
311        assert_eq!(size_of::<CpuAreaRef>(), size_of::<usize>());
312        assert_eq!(size_of::<Option<CpuAreaRef>>(), size_of::<usize>());
313    }
314}
cpu_local/area.rs

cpu_local/
area.rs
