cpu_local/
area.rs

1#[cfg(not(all(target_arch = "aarch64", not(feature = "host-test"))))]
2use core::sync::atomic::Ordering;
3use core::{
4    mem::{MaybeUninit, align_of, offset_of, size_of},
5    ptr::NonNull,
6    sync::atomic::AtomicUsize,
7};
8
9use crate::{CpuIndex, CpuLocalError, ExecutionContextHeader, preempt::PreemptionState};
10
11const fn runtime_anchor_reserved_size() -> usize {
12    64 - 5 * size_of::<usize>() - size_of::<PreemptionState>()
13}
14
15/// CPU-local scalar state shared by trap entry and context publication.
16#[repr(C, align(64))]
17pub struct CpuRuntimeAnchor {
18    current_context: AtomicUsize,
19    architecture_state: [AtomicUsize; 4],
20    preemption_state: PreemptionState,
21    reserved: [u8; runtime_anchor_reserved_size()],
22}
23
24impl CpuRuntimeAnchor {
25    const fn for_boot_context(boot_context: usize) -> Self {
26        let current_context = if cfg!(all(target_arch = "x86_64", not(feature = "host-test")))
27            || cfg!(all(
28                feature = "tls",
29                not(all(target_arch = "aarch64", not(feature = "host-test")))
30            )) {
31            boot_context
32        } else {
33            0
34        };
35        Self {
36            current_context: AtomicUsize::new(current_context),
37            architecture_state: [const { AtomicUsize::new(0) }; 4],
38            preemption_state: PreemptionState::bootstrap_disabled(),
39            reserved: [0; runtime_anchor_reserved_size()],
40        }
41    }
42
43    /// Acquires the context pointer in an anchor-backed image mode.
44    #[cfg(not(all(target_arch = "aarch64", not(feature = "host-test"))))]
45    pub(crate) fn current_context_raw(&self) -> usize {
46        self.current_context.load(Ordering::Acquire)
47    }
48
49    #[cfg(not(all(target_arch = "aarch64", not(feature = "host-test"))))]
50    pub(crate) const fn current_context_slot(&self) -> &AtomicUsize {
51        &self.current_context
52    }
53
54    #[cfg(all(target_arch = "x86_64", not(feature = "host-test")))]
55    pub(crate) const fn preemption_state(&self) -> &PreemptionState {
56        &self.preemption_state
57    }
58}
59
60/// Permanent context header used before the runtime publishes a context.
61#[repr(transparent)]
62pub struct BootContextHeader(ExecutionContextHeader);
63
64impl BootContextHeader {
65    const fn for_area(area_base: usize) -> Self {
66        Self(ExecutionContextHeader::boot(area_base))
67    }
68
69    /// Returns the permanent pinned header.
70    pub const fn header(&self) -> &ExecutionContextHeader {
71        &self.0
72    }
73}
74
75const fn area_header_reserved_size() -> usize {
76    64 - size_of::<u32>() * 2 - size_of::<usize>()
77}
78
79/// Immutable identity stored at the beginning of each initialized CPU area.
80#[repr(C, align(64))]
81pub struct CpuAreaHeader {
82    cpu_index: u32,
83    reserved_word: u32,
84    self_base: usize,
85    reserved: [u8; area_header_reserved_size()],
86}
87
88impl CpuAreaHeader {
89    const fn new(cpu_index: CpuIndex, self_base: usize) -> Self {
90        Self {
91            cpu_index: cpu_index.as_u32(),
92            reserved_word: 0,
93            self_base,
94            reserved: [0; area_header_reserved_size()],
95        }
96    }
97
98    /// Returns the logical CPU index assigned to this area.
99    pub const fn cpu_index(&self) -> CpuIndex {
100        match CpuIndex::from_u32(self.cpu_index) {
101            Some(index) => index,
102            None => panic!("initialized CPU area contains the reserved CPU index"),
103        }
104    }
105
106    /// Returns the permanent runtime base recorded by this area.
107    pub const fn self_base(&self) -> usize {
108        self.self_base
109    }
110}
111
112/// Fixed three-cache-line prefix of every initialized runtime CPU area.
113#[repr(C, align(64))]
114pub struct CpuAreaPrefix {
115    header: CpuAreaHeader,
116    runtime: CpuRuntimeAnchor,
117    boot_context: BootContextHeader,
118}
119
120impl CpuAreaPrefix {
121    /// Constructs the prefix value for one exclusively owned offline area.
122    ///
123    /// # Errors
124    ///
125    /// Returns an address error when `area_base` is null, misaligned, or its
126    /// fixed boot-context address overflows.
127    pub fn initialize(cpu_index: CpuIndex, area_base: usize) -> Result<Self, CpuLocalError> {
128        validate_area_base(area_base)?;
129        area_base
130            .checked_add(CPU_AREA_BOOT_CONTEXT_OFFSET)
131            .ok_or(CpuLocalError::AddressOverflow)?;
132        Ok(Self {
133            header: CpuAreaHeader::new(cpu_index, area_base),
134            runtime: CpuRuntimeAnchor::for_boot_context(area_base + CPU_AREA_BOOT_CONTEXT_OFFSET),
135            boot_context: BootContextHeader::for_area(area_base),
136        })
137    }
138
139    /// Returns immutable area identity.
140    pub const fn header(&self) -> &CpuAreaHeader {
141        &self.header
142    }
143
144    /// Returns CPU runtime and trap state.
145    pub const fn runtime_anchor(&self) -> &CpuRuntimeAnchor {
146        &self.runtime
147    }
148
149    /// Returns the permanent boot execution-context header.
150    pub const fn boot_context(&self) -> &BootContextHeader {
151        &self.boot_context
152    }
153}
154
155/// Permanent typed reference to one fully initialized runtime CPU area.
156#[derive(Clone, Copy, Debug, Eq, PartialEq)]
157pub struct CpuAreaRef {
158    prefix: NonNull<CpuAreaPrefix>,
159    cpu_index: CpuIndex,
160}
161
162// SAFETY: the initialization contract keeps the immutable prefix and runtime
163// anchor mapped until shutdown. Mutable CPU-owned fields provide their own
164// atomic or external synchronization contracts.
165unsafe impl Send for CpuAreaRef {}
166// SAFETY: see the Send implementation; sharing this descriptor does not grant
167// mutable access to non-atomic per-CPU values.
168unsafe impl Sync for CpuAreaRef {}
169
170impl CpuAreaRef {
171    /// Reconstructs a reference from an initialized shutdown-lifetime prefix.
172    ///
173    /// # Safety
174    ///
175    /// `area_base` must point to a fully initialized [`CpuAreaPrefix`] that
176    /// remains mapped until shutdown. No caller may mutate its identity fields.
177    #[doc(hidden)]
178    pub unsafe fn from_initialized_base(area_base: usize) -> Result<Self, CpuLocalError> {
179        #[cfg(feature = "host-test")]
180        crate::register::host_test::record_initialized_area_validation();
181        validate_area_base(area_base)?;
182        let prefix = NonNull::new(area_base as *mut CpuAreaPrefix)
183            .ok_or(CpuLocalError::InvalidAreaBase { base: area_base })?;
184        // SAFETY: forwarded caller contract provides a live initialized prefix.
185        let header = unsafe { prefix.as_ref() }.header();
186        let cpu_index =
187            CpuIndex::from_u32(header.cpu_index).ok_or(CpuLocalError::AreaIdentityMismatch)?;
188        if header.self_base != area_base {
189            return Err(CpuLocalError::AreaIdentityMismatch);
190        }
191        let expected_boot = area_base
192            .checked_add(CPU_AREA_BOOT_CONTEXT_OFFSET)
193            .ok_or(CpuLocalError::AddressOverflow)?;
194        if unsafe { prefix.as_ref() }
195            .boot_context()
196            .header()
197            .raw_cpu_binding()
198            .map(|(boot_area, _)| boot_area)
199            != Some(area_base)
200            || expected_boot
201                != core::ptr::addr_of!(unsafe { prefix.as_ref() }.boot_context.0) as usize
202        {
203            return Err(CpuLocalError::AreaIdentityMismatch);
204        }
205        Ok(Self { prefix, cpu_index })
206    }
207
208    /// Returns this area's logical CPU index.
209    pub const fn cpu_index(self) -> CpuIndex {
210        self.cpu_index
211    }
212
213    /// Returns the exact runtime prefix address used as area identity.
214    pub fn base(self) -> usize {
215        self.prefix.as_ptr() as usize
216    }
217
218    /// Returns the initialized fixed prefix.
219    pub fn prefix(self) -> &'static CpuAreaPrefix {
220        // SAFETY: construction requires a shutdown-lifetime mapping.
221        unsafe { self.prefix.as_ref() }
222    }
223
224    /// Returns this area's runtime/trap anchor.
225    pub fn runtime_anchor(self) -> &'static CpuRuntimeAnchor {
226        self.prefix().runtime_anchor()
227    }
228}
229
230fn validate_area_base(area_base: usize) -> Result<(), CpuLocalError> {
231    if area_base == 0 || !area_base.is_multiple_of(align_of::<CpuAreaPrefix>()) {
232        Err(CpuLocalError::InvalidAreaBase { base: area_base })
233    } else {
234        Ok(())
235    }
236}
237
238/// Size in bytes of the immutable area header.
239pub const CPU_AREA_HEADER_SIZE: usize = size_of::<CpuAreaHeader>();
240/// Byte offset of CPU runtime/trap state.
241pub const CPU_AREA_RUNTIME_ANCHOR_OFFSET: usize = offset_of!(CpuAreaPrefix, runtime);
242/// Byte offset of the permanent boot execution-context header.
243pub const CPU_AREA_BOOT_CONTEXT_OFFSET: usize = offset_of!(CpuAreaPrefix, boot_context);
244/// Byte offset of the runtime self pointer.
245pub const CPU_AREA_SELF_BASE_OFFSET: usize = offset_of!(CpuAreaHeader, self_base);
246/// Byte offset of the logical CPU index.
247pub const CPU_AREA_CPU_INDEX_OFFSET: usize = offset_of!(CpuAreaHeader, cpu_index);
248/// Byte offset of the current-context slot used by anchor-backed image modes.
249pub const CPU_AREA_CURRENT_CONTEXT_OFFSET: usize =
250    CPU_AREA_RUNTIME_ANCHOR_OFFSET + offset_of!(CpuRuntimeAnchor, current_context);
251/// Byte offset of architecture-owned CPU trap state.
252pub const CPU_AREA_ARCH_STATE_OFFSET: usize =
253    CPU_AREA_RUNTIME_ANCHOR_OFFSET + offset_of!(CpuRuntimeAnchor, architecture_state);
254/// Reserved bytes available to the architecture-owned CPU trap state.
255pub const CPU_AREA_ARCH_STATE_SIZE: usize = 4 * size_of::<usize>();
256/// Byte offset of the x86_64 CPU-owned preemption word.
257pub const CPU_AREA_PREEMPTION_STATE_OFFSET: usize =
258    CPU_AREA_RUNTIME_ANCHOR_OFFSET + offset_of!(CpuRuntimeAnchor, preemption_state);
259
260const _: () = {
261    assert!(size_of::<CpuAreaHeader>() == 64);
262    assert!(align_of::<CpuAreaHeader>() == 64);
263    assert!(size_of::<CpuRuntimeAnchor>() == 64);
264    assert!(align_of::<CpuRuntimeAnchor>() == 64);
265    assert!(size_of::<BootContextHeader>() == 64);
266    assert!(align_of::<BootContextHeader>() == 64);
267    assert!(size_of::<CpuAreaPrefix>() == 192);
268    assert!(align_of::<CpuAreaPrefix>() == 64);
269    assert!(CPU_AREA_RUNTIME_ANCHOR_OFFSET == 64);
270    assert!(CPU_AREA_BOOT_CONTEXT_OFFSET == 128);
271};
272
273#[doc(hidden)]
274#[used]
275#[unsafe(no_mangle)]
276#[unsafe(link_section = ".percpu.template.header")]
277pub static mut __CPU_LOCAL_AREA_PREFIX: MaybeUninit<CpuAreaPrefix> = MaybeUninit::uninit();
278
279#[doc(hidden)]
280#[used]
281#[unsafe(no_mangle)]
282#[unsafe(link_section = ".percpu.template.end")]
283pub static __CPU_LOCAL_TEMPLATE_END: u8 = 0;
cpu_local/area.rs

cpu_local/
area.rs
