cpu_local/
area.rs

1use core::{
2    mem::{MaybeUninit, align_of, offset_of, size_of},
3    ptr::NonNull,
4    sync::atomic::{AtomicUsize, Ordering},
5};
6
7use crate::{CpuIndex, CpuLocalError, CurrentThreadHeader};
8
9/// CPU-local scalar state shared by trap entry and scheduler publication.
10#[repr(C, align(64))]
11pub struct CpuRuntimeAnchor {
12    current_thread: AtomicUsize,
13    architecture_state: [AtomicUsize; 4],
14    reserved: [u8; 64 - 5 * size_of::<usize>()],
15}
16
17impl CpuRuntimeAnchor {
18    const fn for_boot_thread(boot_thread: usize) -> Self {
19        Self {
20            current_thread: AtomicUsize::new(boot_thread),
21            architecture_state: [const { AtomicUsize::new(0) }; 4],
22            reserved: [0; 64 - 5 * size_of::<usize>()],
23        }
24    }
25
26    /// Acquires the current-thread pointer published by the scheduler.
27    pub fn current_thread_raw(&self) -> usize {
28        self.current_thread.load(Ordering::Acquire)
29    }
30
31    pub(crate) const fn current_thread_slot(&self) -> &AtomicUsize {
32        &self.current_thread
33    }
34}
35
36/// Permanent current header used before the scheduler publishes a task.
37#[repr(transparent)]
38pub struct BootThreadHeader(CurrentThreadHeader);
39
40impl BootThreadHeader {
41    const fn for_area(area_base: usize) -> Self {
42        Self(CurrentThreadHeader::boot(area_base))
43    }
44
45    /// Returns the permanent pinned header.
46    pub const fn header(&self) -> &CurrentThreadHeader {
47        &self.0
48    }
49}
50
51const fn area_header_reserved_size() -> usize {
52    64 - size_of::<u32>() * 2 - size_of::<usize>()
53}
54
55/// Immutable identity stored at the beginning of each initialized CPU area.
56#[repr(C, align(64))]
57pub struct CpuAreaHeader {
58    cpu_index: u32,
59    reserved_word: u32,
60    self_base: usize,
61    reserved: [u8; area_header_reserved_size()],
62}
63
64impl CpuAreaHeader {
65    const fn new(cpu_index: CpuIndex, self_base: usize) -> Self {
66        Self {
67            cpu_index: cpu_index.as_u32(),
68            reserved_word: 0,
69            self_base,
70            reserved: [0; area_header_reserved_size()],
71        }
72    }
73
74    /// Returns the logical CPU index assigned to this area.
75    pub const fn cpu_index(&self) -> CpuIndex {
76        match CpuIndex::from_u32(self.cpu_index) {
77            Some(index) => index,
78            None => panic!("initialized CPU area contains the reserved CPU index"),
79        }
80    }
81
82    /// Returns the permanent runtime base recorded by this area.
83    pub const fn self_base(&self) -> usize {
84        self.self_base
85    }
86}
87
88/// Fixed three-cache-line prefix of every initialized runtime CPU area.
89#[repr(C, align(64))]
90pub struct CpuAreaPrefix {
91    header: CpuAreaHeader,
92    runtime: CpuRuntimeAnchor,
93    boot_thread: BootThreadHeader,
94}
95
96impl CpuAreaPrefix {
97    /// Constructs the prefix value for one exclusively owned offline area.
98    ///
99    /// # Errors
100    ///
101    /// Returns an address error when `area_base` is null, misaligned, or its
102    /// fixed boot-thread address overflows.
103    pub fn initialize(cpu_index: CpuIndex, area_base: usize) -> Result<Self, CpuLocalError> {
104        validate_area_base(area_base)?;
105        area_base
106            .checked_add(CPU_AREA_BOOT_THREAD_OFFSET)
107            .ok_or(CpuLocalError::AddressOverflow)?;
108        Ok(Self {
109            header: CpuAreaHeader::new(cpu_index, area_base),
110            runtime: CpuRuntimeAnchor::for_boot_thread(area_base + CPU_AREA_BOOT_THREAD_OFFSET),
111            boot_thread: BootThreadHeader::for_area(area_base),
112        })
113    }
114
115    /// Returns immutable area identity.
116    pub const fn header(&self) -> &CpuAreaHeader {
117        &self.header
118    }
119
120    /// Returns CPU runtime and trap state.
121    pub const fn runtime_anchor(&self) -> &CpuRuntimeAnchor {
122        &self.runtime
123    }
124
125    /// Returns the permanent boot current-thread header.
126    pub const fn boot_thread(&self) -> &BootThreadHeader {
127        &self.boot_thread
128    }
129}
130
131/// Permanent typed reference to one fully initialized runtime CPU area.
132#[derive(Clone, Copy, Debug, Eq, PartialEq)]
133pub struct CpuAreaRef {
134    prefix: NonNull<CpuAreaPrefix>,
135    cpu_index: CpuIndex,
136}
137
138// SAFETY: the initialization contract keeps the immutable prefix and runtime
139// anchor mapped until shutdown. Mutable CPU-owned fields provide their own
140// atomic or external synchronization contracts.
141unsafe impl Send for CpuAreaRef {}
142// SAFETY: see the Send implementation; sharing this descriptor does not grant
143// mutable access to non-atomic per-CPU values.
144unsafe impl Sync for CpuAreaRef {}
145
146impl CpuAreaRef {
147    /// Reconstructs a reference from an initialized shutdown-lifetime prefix.
148    ///
149    /// # Safety
150    ///
151    /// `area_base` must point to a fully initialized [`CpuAreaPrefix`] that
152    /// remains mapped until shutdown. No caller may mutate its identity fields.
153    #[doc(hidden)]
154    pub unsafe fn from_initialized_base(area_base: usize) -> Result<Self, CpuLocalError> {
155        validate_area_base(area_base)?;
156        let prefix = NonNull::new(area_base as *mut CpuAreaPrefix)
157            .ok_or(CpuLocalError::InvalidAreaBase { base: area_base })?;
158        // SAFETY: forwarded caller contract provides a live initialized prefix.
159        let header = unsafe { prefix.as_ref() }.header();
160        let cpu_index =
161            CpuIndex::from_u32(header.cpu_index).ok_or(CpuLocalError::AreaIdentityMismatch)?;
162        if header.self_base != area_base {
163            return Err(CpuLocalError::AreaIdentityMismatch);
164        }
165        let expected_boot = area_base
166            .checked_add(CPU_AREA_BOOT_THREAD_OFFSET)
167            .ok_or(CpuLocalError::AddressOverflow)?;
168        if unsafe { prefix.as_ref() }
169            .runtime_anchor()
170            .current_thread_raw()
171            == 0
172            || unsafe { prefix.as_ref() }
173                .boot_thread()
174                .header()
175                .raw_cpu_binding()
176                .map(|(boot_area, _)| boot_area)
177                != Some(area_base)
178            || expected_boot
179                != core::ptr::addr_of!(unsafe { prefix.as_ref() }.boot_thread.0) as usize
180        {
181            return Err(CpuLocalError::AreaIdentityMismatch);
182        }
183        Ok(Self { prefix, cpu_index })
184    }
185
186    /// Returns this area's logical CPU index.
187    pub const fn cpu_index(self) -> CpuIndex {
188        self.cpu_index
189    }
190
191    /// Returns the exact runtime prefix address used as area identity.
192    pub fn base(self) -> usize {
193        self.prefix.as_ptr() as usize
194    }
195
196    /// Returns the initialized fixed prefix.
197    pub fn prefix(self) -> &'static CpuAreaPrefix {
198        // SAFETY: construction requires a shutdown-lifetime mapping.
199        unsafe { self.prefix.as_ref() }
200    }
201
202    /// Returns this area's runtime/trap anchor.
203    pub fn runtime_anchor(self) -> &'static CpuRuntimeAnchor {
204        self.prefix().runtime_anchor()
205    }
206}
207
208fn validate_area_base(area_base: usize) -> Result<(), CpuLocalError> {
209    if area_base == 0 || !area_base.is_multiple_of(align_of::<CpuAreaPrefix>()) {
210        Err(CpuLocalError::InvalidAreaBase { base: area_base })
211    } else {
212        Ok(())
213    }
214}
215
216/// Size in bytes of the immutable area header.
217pub const CPU_AREA_HEADER_SIZE: usize = size_of::<CpuAreaHeader>();
218/// Byte offset of CPU runtime/trap state.
219pub const CPU_AREA_RUNTIME_ANCHOR_OFFSET: usize = offset_of!(CpuAreaPrefix, runtime);
220/// Byte offset of the permanent boot current-thread header.
221pub const CPU_AREA_BOOT_THREAD_OFFSET: usize = offset_of!(CpuAreaPrefix, boot_thread);
222/// Byte offset of the runtime self pointer.
223pub const CPU_AREA_SELF_BASE_OFFSET: usize = offset_of!(CpuAreaHeader, self_base);
224/// Byte offset of the logical CPU index.
225pub const CPU_AREA_CPU_INDEX_OFFSET: usize = offset_of!(CpuAreaHeader, cpu_index);
226/// Byte offset of the current-thread slot.
227pub const CPU_AREA_CURRENT_THREAD_OFFSET: usize =
228    CPU_AREA_RUNTIME_ANCHOR_OFFSET + offset_of!(CpuRuntimeAnchor, current_thread);
229/// Byte offset of architecture-owned CPU trap state.
230pub const CPU_AREA_ARCH_STATE_OFFSET: usize =
231    CPU_AREA_RUNTIME_ANCHOR_OFFSET + offset_of!(CpuRuntimeAnchor, architecture_state);
232/// Reserved bytes available to the architecture-owned CPU trap state.
233pub const CPU_AREA_ARCH_STATE_SIZE: usize = 4 * size_of::<usize>();
234
235const _: () = {
236    assert!(size_of::<CpuAreaHeader>() == 64);
237    assert!(align_of::<CpuAreaHeader>() == 64);
238    assert!(size_of::<CpuRuntimeAnchor>() == 64);
239    assert!(align_of::<CpuRuntimeAnchor>() == 64);
240    assert!(size_of::<BootThreadHeader>() == 64);
241    assert!(align_of::<BootThreadHeader>() == 64);
242    assert!(size_of::<CpuAreaPrefix>() == 192);
243    assert!(align_of::<CpuAreaPrefix>() == 64);
244    assert!(CPU_AREA_RUNTIME_ANCHOR_OFFSET == 64);
245    assert!(CPU_AREA_BOOT_THREAD_OFFSET == 128);
246};
247
248#[doc(hidden)]
249#[used]
250#[unsafe(no_mangle)]
251#[unsafe(link_section = ".percpu.template.header")]
252pub static mut __CPU_LOCAL_AREA_PREFIX: MaybeUninit<CpuAreaPrefix> = MaybeUninit::uninit();
253
254#[doc(hidden)]
255#[used]
256#[unsafe(no_mangle)]
257#[unsafe(link_section = ".percpu.template.end")]
258pub static __CPU_LOCAL_TEMPLATE_END: u8 = 0;
cpu_local/area.rs

cpu_local/
area.rs
