link_section/
sections.rs

1#[cfg(not(target_family = "wasm"))]
2use core::sync::atomic::{AtomicU8, Ordering};
3use core::{cell::UnsafeCell, ptr};
4
5use crate::__support::{Bounds, MovableBounds, SyncUnsafeCell};
6
7/// An untyped link section that can be used to store any type. The underlying
8/// data is not enumerable.
9#[repr(C)]
10pub struct Section {
11    name: &'static str,
12    bounds: Bounds,
13}
14
15impl Section {
16    /// # Safety
17    ///
18    /// For macro-generated use only. `bounds` must match the linker-resolved
19    /// (or WASM-initialized) section range for `name`.
20    #[doc(hidden)]
21    pub const unsafe fn new(name: &'static str, bounds: Bounds) -> Self {
22        Self { name, bounds }
23    }
24
25    /// The byte length of the section.
26    #[inline]
27    pub fn byte_len(&self) -> usize {
28        self.bounds.byte_len()
29    }
30
31    /// The start address of the section.
32    #[inline]
33    pub fn start_ptr(&self) -> *const () {
34        self.bounds.start_ptr()
35    }
36    /// The end address of the section.
37    #[inline]
38    pub fn end_ptr(&self) -> *const () {
39        self.bounds.end_ptr()
40    }
41
42    /// Ensures that a section exists at the given path.
43    #[doc(hidden)]
44    pub const fn __validate<T: IsUntypedSection>(_section: &T) {}
45}
46
47impl ::core::fmt::Debug for Section {
48    fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
49        f.debug_struct("Section")
50            .field("name", &self.name)
51            .field("start", &self.start_ptr())
52            .field("end", &self.end_ptr())
53            .field("byte_len", &self.byte_len())
54            .finish()
55    }
56}
57
58unsafe impl Sync for Section {}
59unsafe impl Send for Section {}
60
61// Waiting on Rust 1.78
62// #[diagnostic::on_unimplemented(message = "This is not an untyped section")]
63/// Marker: untyped [`Section`] handle.
64pub trait IsUntypedSection {}
65
66macro_rules! impl_section_new {
67    ($generic:ident) => {
68        /// # Safety
69        ///
70        /// For macro-generated use only. `bounds` must match the linker-resolved
71        /// (or WASM-initialized) section range for `name`.
72        #[doc(hidden)]
73        pub const unsafe fn new(name: &'static str, bounds: Bounds) -> Self {
74            assert!(
75                ::core::mem::size_of::<$generic>() > 0,
76                "Zero-sized types are not supported"
77            );
78            Self {
79                name,
80                bounds,
81                _phantom: ::core::marker::PhantomData,
82            }
83        }
84    };
85}
86
87macro_rules! impl_bounds_fns {
88    ($generic:ident) => {
89        /// The start address of the section.
90        #[inline(always)]
91        pub fn start_ptr(&self) -> *const T {
92            self.bounds.start_ptr() as *const T
93        }
94
95        /// The end address of the section.
96        #[inline(always)]
97        pub fn end_ptr(&self) -> *const T {
98            self.bounds.end_ptr() as *const T
99        }
100
101        /// The stride of the typed section.
102        #[inline(always)]
103        pub const fn stride(&self) -> usize {
104            assert!(
105                ::core::mem::size_of::<T>() > 0
106                    && ::core::mem::size_of::<T>() * 2 == ::core::mem::size_of::<[T; 2]>()
107            );
108            ::core::mem::size_of::<T>()
109        }
110
111        /// The byte length of the section.
112        #[inline]
113        pub fn byte_len(&self) -> usize {
114            self.bounds.byte_len()
115        }
116
117        /// The number of elements in the section.
118        #[inline]
119        pub fn len(&self) -> usize {
120            self.byte_len() / self.stride()
121        }
122
123        /// True if the section is empty.
124        #[inline]
125        pub fn is_empty(&self) -> bool {
126            self.len() == 0
127        }
128
129        /// The section as a slice.
130        #[inline]
131        pub fn as_slice(&self) -> &[T] {
132            if self.is_empty() {
133                &[]
134            } else {
135                unsafe { ::core::slice::from_raw_parts(self.start_ptr(), self.len()) }
136            }
137        }
138
139        /// The offset of the item in the section, if it is in the section.
140        ///
141        /// This is O(1), as it performs direct pointer arithmetic.
142        #[inline]
143        pub fn offset_of(&self, item: impl $crate::SectionItemLocation<T>) -> Option<usize> {
144            let ptr = item.item_ptr();
145            if ptr < self.start_ptr() || ptr >= self.end_ptr() {
146                None
147            } else {
148                Some(unsafe { ptr.offset_from(self.start_ptr()) as usize })
149            }
150        }
151    };
152}
153
154macro_rules! impl_bounds_traits {
155    ($name:ident < $generic:ident >) => {
156        impl<'a, $generic> ::core::iter::IntoIterator for &'a $name<$generic> {
157            type Item = &'a $generic;
158            type IntoIter = ::core::slice::Iter<'a, $generic>;
159            fn into_iter(self) -> Self::IntoIter {
160                self.as_slice().iter()
161            }
162        }
163
164        impl<T> ::core::ops::Deref for $name<$generic> {
165            type Target = [$generic];
166            fn deref(&self) -> &Self::Target {
167                self.as_slice()
168            }
169        }
170
171        impl<T> ::core::fmt::Debug for $name<$generic> {
172            fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
173                f.debug_struct(stringify!($name))
174                    .field("name", &self.name)
175                    .field("start", &self.start_ptr())
176                    .field("end", &self.end_ptr())
177                    .field("len", &self.len())
178                    .field("stride", &self.stride())
179                    .finish()
180            }
181        }
182
183        impl<T> $crate::__support::SectionItemType for $name<$generic> {
184            type Item = $generic;
185        }
186
187        impl<T> $crate::__support::SectionItemTyped<$generic> for $name<$generic> {
188            type Item = $generic;
189        }
190
191        unsafe impl<$generic> Sync for $name<$generic> where $generic: Sync {}
192        unsafe impl<$generic> Send for $name<$generic> where $generic: Send {}
193    };
194}
195
196/// A typed link section that can be used to store any sized type. The
197/// underlying data is immutable and enumerable. `static` and `const` items are
198/// stored directly in the section.
199///
200/// `static` items are guaranteed to have a valid return from
201/// [`TypedSection::offset_of`] if they are in the section.
202///
203/// Platform note: WASM platforms require `const` items. Use
204/// [`TypedReferenceSection`] for cross-platform support for `static` items.
205#[repr(C)]
206pub struct TypedSection<T: 'static> {
207    name: &'static str,
208    bounds: Bounds,
209    _phantom: ::core::marker::PhantomData<T>,
210}
211
212impl<T: 'static> TypedSection<T> {
213    impl_section_new!(T);
214    impl_bounds_fns!(T);
215}
216
217impl_bounds_traits!(TypedSection<T>);
218
219/// A mutable typed link section that can be used to store any sized type. The
220/// underlying data is (unsafely) mutable and enumerable.
221///
222/// Only `const` items may be submitted to a [`TypedMutableSection`].
223///
224/// Mutating the section (for example via [`TypedMutableSection::as_mut_slice`])
225/// requires exclusive access. See [Exclusive access](crate#exclusive-access) for
226/// more information.
227#[repr(C)]
228pub struct TypedMutableSection<T: 'static> {
229    name: &'static str,
230    bounds: Bounds,
231    _phantom: ::core::marker::PhantomData<T>,
232}
233
234impl<T: 'static> TypedMutableSection<T> {
235    impl_section_new!(T);
236    impl_bounds_fns!(T);
237
238    /// The start address of the section.
239    #[inline]
240    pub fn start_ptr_mut(&self) -> *mut T {
241        self.bounds.start_ptr() as *mut T
242    }
243
244    /// The start address of the section.
245    #[inline]
246    pub fn end_ptr_mut(&self) -> *mut T {
247        self.bounds.end_ptr() as *mut T
248    }
249
250    /// The section as a mutable slice.
251    ///
252    /// # Safety
253    ///
254    /// Mutating the section requires exclusive access. See
255    /// [Exclusive access](crate#exclusive-access) for more information.
256    #[allow(clippy::mut_from_ref)]
257    #[inline]
258    pub unsafe fn as_mut_slice(&self) -> &mut [T] {
259        if self.is_empty() {
260            &mut []
261        } else {
262            unsafe { ::core::slice::from_raw_parts_mut(self.start_ptr() as *mut T, self.len()) }
263        }
264    }
265}
266
267impl_bounds_traits!(TypedMutableSection<T>);
268
269/// A movable typed link section that can be used to store any sized type. The
270/// underlying data is (unsafely) mutable, enumerable, and expected to be
271/// reordered during startup initialization. Each item is paired with a
272/// [`MovableBackref`] that updates a stable [`MovableRef`] slot when the
273/// section is sorted.
274///
275/// Only `static` items may be submitted to a [`TypedMovableSection`].
276///
277/// Mutating or reordering the section requires exclusive access. See
278/// [Exclusive access](crate#exclusive-access) for more information.
279/// [`TypedMovableSection::sort_unstable`] also updates every [`MovableRef`]; any
280/// `&T` obtained before sorting may be stale afterward.
281#[repr(C)]
282pub struct TypedMovableSection<T: 'static> {
283    name: &'static str,
284    bounds: MovableBounds,
285    #[cfg(not(target_family = "wasm"))]
286    backref_state: AtomicU8,
287    _phantom: ::core::marker::PhantomData<T>,
288}
289
290impl<T: 'static> TypedMovableSection<T> {
291    /// # Safety
292    ///
293    /// For macro-generated use only. `bounds` must describe the final layout of
294    /// the linker (or WASM runtime) section after all items are registered.
295    #[doc(hidden)]
296    pub const unsafe fn new(name: &'static str, bounds: MovableBounds) -> Self {
297        assert!(
298            ::core::mem::size_of::<T>() > 0,
299            "Zero-sized types are not supported"
300        );
301        Self {
302            name,
303            bounds,
304            #[cfg(not(target_family = "wasm"))]
305            backref_state: AtomicU8::new(0),
306            _phantom: ::core::marker::PhantomData,
307        }
308    }
309
310    impl_bounds_fns!(T);
311
312    /// The section as a mutable slice.
313    ///
314    /// # Safety
315    ///
316    /// Mutating the section requires exclusive access. See
317    /// [Exclusive access](crate#exclusive-access) for more information.
318    #[allow(clippy::mut_from_ref)]
319    #[inline]
320    pub unsafe fn as_mut_slice(&self) -> &mut [T] {
321        if self.is_empty() {
322            &mut []
323        } else {
324            unsafe { ::core::slice::from_raw_parts_mut(self.start_ptr() as *mut T, self.len()) }
325        }
326    }
327
328    /// The backrefs as a mutable slice, ordered to match the current value
329    /// section addresses.
330    ///
331    /// # Safety
332    ///
333    /// Mutating the backref section requires exclusive access. See
334    /// [Exclusive access](crate#exclusive-access) for more information. Do not
335    /// call while any other slice into either the value or backref linker
336    /// sections is live.
337    #[allow(clippy::mut_from_ref)]
338    #[inline]
339    pub unsafe fn as_mut_backrefs(&self) -> &mut [MovableBackref<T>] {
340        let backrefs_len =
341            self.bounds.backrefs_byte_len() / ::core::mem::size_of::<MovableBackref<T>>();
342        let backrefs = if backrefs_len == 0 {
343            &mut []
344        } else {
345            unsafe {
346                ::core::slice::from_raw_parts_mut(
347                    self.bounds.backrefs_start_ptr() as *mut MovableBackref<T>,
348                    backrefs_len,
349                )
350            }
351        };
352        #[cfg(not(target_family = "wasm"))]
353        unsafe {
354            self.fixup_backrefs(backrefs)
355        };
356        backrefs
357    }
358
359    /// As we cannot guarantee that the linker placed the items and backrefs in
360    /// the same order, we need to sort the backrefs to match the items.
361    ///
362    /// The exception, of course, is WASM where we placed both of them
363    /// ourselves.
364    #[cfg(not(target_family = "wasm"))]
365    unsafe fn fixup_backrefs(&self, backrefs: &mut [MovableBackref<T>]) {
366        match self
367            .backref_state
368            .compare_exchange(0, 1, Ordering::Acquire, Ordering::Acquire)
369        {
370            Ok(_) => {}
371            Err(2) => return,
372            Err(_) => panic!("movable section backrefs already being initialized"),
373        }
374
375        if backrefs.len() != self.len() {
376            panic!("movable section backref count does not match item count");
377        }
378
379        backrefs.sort_unstable_by_key(|backref| backref.current_ptr());
380        self.backref_state.store(2, Ordering::Release);
381    }
382
383    /// Sort the section and backrefs in place.
384    ///
385    /// This algorithm is currently implemented as a quicksort.
386    ///
387    /// # Safety
388    ///
389    /// Reordering the section requires exclusive access. See
390    /// [Exclusive access](crate#exclusive-access) for more information. After
391    /// this returns, every [`MovableRef`] slot points at the new location of its
392    /// item; any `&T` obtained through [`MovableRef`] before the sort must not
393    /// be used.
394    #[allow(unsafe_code)]
395    pub unsafe fn sort_unstable(&self)
396    where
397        T: Ord,
398    {
399        // Trivial case.
400        let main = unsafe { self.as_mut_slice() };
401        if main.len() <= 1 {
402            return;
403        }
404
405        let refs = unsafe { self.as_mut_backrefs() };
406        debug_assert_eq!(main.len(), refs.len());
407
408        fn partition<T: Ord, R>(main: &mut [T], refs: &mut [R]) -> usize {
409            let n = main.len();
410            if n == 0 {
411                return 0;
412            }
413            let pivot = n - 1;
414            let mut i = 0;
415            for j in 0..pivot {
416                if main[j] <= main[pivot] {
417                    main.swap(i, j);
418                    refs.swap(i, j);
419                    i += 1;
420                }
421            }
422            main.swap(i, pivot);
423            refs.swap(i, pivot);
424            i
425        }
426
427        fn recurse<T: Ord, R>(main: &mut [T], refs: &mut [R]) {
428            let n = main.len();
429            if n <= 1 {
430                return;
431            }
432            let p = partition(main, refs);
433            let (ml, mr) = main.split_at_mut(p);
434            let (rl, rr) = refs.split_at_mut(p);
435            recurse(ml, rl);
436            if mr.len() > 1 {
437                recurse(&mut mr[1..], &mut rr[1..]);
438            }
439        }
440
441        recurse(main, refs);
442
443        // TODO: could we avoid the fixup if no changes are made?
444        for (item, backref) in main.iter().zip(refs.iter()) {
445            unsafe {
446                backref.set_current_ptr(item as *const T);
447            }
448        }
449    }
450}
451
452impl_bounds_traits!(TypedMovableSection<T>);
453
454/// A reference to a movable item through a stable pointer slot.
455///
456/// The slot is updated when a [`TypedMovableSection`] is reordered (for example
457/// by [`TypedMovableSection::sort_unstable`]). Do not keep an `&T` from
458/// dereferencing this handle across such an update.
459#[repr(transparent)]
460pub struct MovableRef<T: 'static> {
461    slot: SyncUnsafeCell<*const T>,
462}
463
464impl<T> MovableRef<T> {
465    #[doc(hidden)]
466    pub const fn new(ptr: *const T) -> Self {
467        Self {
468            slot: SyncUnsafeCell::new(ptr),
469        }
470    }
471
472    /// Get a raw pointer to the stable pointer slot inside this handle. Note
473    /// that both this and the SyncUnsafeCell are transparent.
474    #[doc(hidden)]
475    pub const fn slot_ptr(this: *const Self) -> *const UnsafeCell<*const T> {
476        this.cast::<UnsafeCell<*const T>>()
477    }
478
479    /// Raw pointer to the value currently referenced by this slot.
480    pub const fn as_ptr(&self) -> *const T {
481        unsafe { *self.slot.get() }
482    }
483}
484
485impl<T> ::core::ops::Deref for MovableRef<T> {
486    type Target = T;
487    fn deref(&self) -> &Self::Target {
488        unsafe { self.as_ptr().as_ref().expect("MovableRef not initialized") }
489    }
490}
491
492impl<T> ::core::fmt::Debug for MovableRef<T>
493where
494    T: ::core::fmt::Debug,
495{
496    fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
497        (**self).fmt(f)
498    }
499}
500
501impl<T> ::core::fmt::Display for MovableRef<T>
502where
503    T: ::core::fmt::Display,
504{
505    fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
506        (**self).fmt(f)
507    }
508}
509
510unsafe impl<T> Send for MovableRef<T> where T: Send {}
511unsafe impl<T> Sync for MovableRef<T> where T: Sync {}
512
513/// A backref record submitted alongside each item in a [`TypedMovableSection`].
514/// This points to a [`MovableRef`] that lives outside of the section.
515#[repr(C)]
516pub struct MovableBackref<T: 'static> {
517    slot: *const UnsafeCell<*const T>,
518}
519
520impl<T> MovableBackref<T> {
521    #[doc(hidden)]
522    pub const fn new(slot: *const UnsafeCell<*const T>) -> Self {
523        Self { slot }
524    }
525
526    /// Current value of the stable pointer slot as a pointer.
527    pub fn current_ptr(&self) -> *const T {
528        unsafe { ptr::read(UnsafeCell::raw_get(self.slot)) }
529    }
530
531    /// Update the stable pointer slot.
532    ///
533    /// # Safety
534    ///
535    /// Updating the slot requires exclusive access. See
536    /// [Exclusive access](crate#exclusive-access) for more information. Any live
537    /// `&T` or [`MovableRef`] dereference may alias the old or new target.
538    pub unsafe fn set_current_ptr(&self, ptr: *const T) {
539        unsafe {
540            ptr::write(UnsafeCell::raw_get(self.slot), ptr);
541        }
542    }
543}
544
545unsafe impl<T> Send for MovableBackref<T> where T: Send {}
546unsafe impl<T> Sync for MovableBackref<T> where T: Sync {}
547
548/// A typed link section that can be used to store any sized type. The
549/// underlying data is enumerable.
550#[repr(C)]
551pub struct TypedReferenceSection<T: 'static> {
552    name: &'static str,
553    bounds: Bounds,
554    _phantom: ::core::marker::PhantomData<T>,
555}
556
557impl<T: 'static> TypedReferenceSection<T> {
558    impl_section_new!(T);
559    impl_bounds_fns!(T);
560}
561
562impl_bounds_traits!(TypedReferenceSection<T>);
563
564/// A reference to a value in a link section. This allows platforms like WASM
565/// to reference the value, even though the final location is not known until
566/// after initialization.
567#[repr(transparent)]
568pub struct Ref<T: 'static> {
569    #[cfg(target_family = "wasm")]
570    ptr: ::core::cell::UnsafeCell<*const T>,
571    #[cfg(not(target_family = "wasm"))]
572    t: T,
573}
574
575impl<T> Ref<T> {
576    #[cfg(not(target_family = "wasm"))]
577    #[doc(hidden)]
578    pub const fn new(t: T) -> Self {
579        Self { t }
580    }
581
582    #[cfg(target_family = "wasm")]
583    #[doc(hidden)]
584    pub const fn new() -> Self {
585        Self {
586            ptr: ::core::cell::UnsafeCell::new(::core::ptr::null()),
587        }
588    }
589
590    /// # Safety
591    ///
592    /// For macro/runtime registration only. `ptr` must refer to the item's final
593    /// location in the WASM link section. Requires exclusive access. See
594    /// [Exclusive access](crate#exclusive-access) for more information.
595    #[cfg(target_family = "wasm")]
596    #[doc(hidden)]
597    pub unsafe fn set(&self, ptr: *const T) {
598        *self.ptr.get() = ptr;
599    }
600
601    /// Raw pointer to the value (WASM: cell; otherwise `&T` as `*const T`).
602    pub fn as_ptr(&self) -> *const T {
603        #[cfg(target_family = "wasm")]
604        {
605            unsafe { *self.ptr.get() }
606        }
607        #[cfg(not(target_family = "wasm"))]
608        {
609            &self.t as *const T
610        }
611    }
612}
613
614impl<T> ::core::ops::Deref for Ref<T> {
615    type Target = T;
616    fn deref(&self) -> &Self::Target {
617        #[cfg(target_family = "wasm")]
618        unsafe {
619            ::core::ptr::read(self.ptr.get())
620                .as_ref()
621                .expect("Ref not initialized")
622        }
623        #[cfg(not(target_family = "wasm"))]
624        &self.t
625    }
626}
627
628impl<T> ::core::fmt::Debug for Ref<T>
629where
630    T: ::core::fmt::Debug,
631{
632    fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
633        (**self).fmt(f)
634    }
635}
636
637impl<T> ::core::fmt::Display for Ref<T>
638where
639    T: ::core::fmt::Display,
640{
641    fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
642        (**self).fmt(f)
643    }
644}
645
646unsafe impl<T> Send for Ref<T> where T: Send {}
647unsafe impl<T> Sync for Ref<T> where T: Sync {}
link_section/sections.rs

link_section/
sections.rs
