link_section/
sections.rs

1#[cfg(not(target_family = "wasm"))]
2use core::sync::atomic::{AtomicU8, Ordering};
3use core::{cell::UnsafeCell, ptr};
4
5use crate::__support::{Bounds, MovableBounds, SyncUnsafeCell};
6
7/// An untyped link section that can be used to store any type. The underlying
8/// data is not enumerable.
9#[repr(C)]
10pub struct Section {
11    name: &'static str,
12    bounds: Bounds,
13}
14
15impl Section {
16    /// # Safety
17    ///
18    /// For macro-generated use only. `bounds` must match the linker-resolved
19    /// (or WASM-initialized) section range for `name`.
20    #[doc(hidden)]
21    pub const unsafe fn new(name: &'static str, bounds: Bounds) -> Self {
22        Self { name, bounds }
23    }
24
25    /// The byte length of the section.
26    #[inline]
27    pub fn byte_len(&self) -> usize {
28        self.bounds.byte_len()
29    }
30
31    /// The start address of the section.
32    #[inline]
33    pub fn start_ptr(&self) -> *const () {
34        self.bounds.start_ptr()
35    }
36    /// The end address of the section.
37    #[inline]
38    pub fn end_ptr(&self) -> *const () {
39        self.bounds.end_ptr()
40    }
41
42    /// Ensures that a section exists at the given path.
43    #[doc(hidden)]
44    pub const fn __validate<T: IsUntypedSection>(_section: &T) {}
45}
46
47impl ::core::fmt::Debug for Section {
48    fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
49        f.debug_struct("Section")
50            .field("name", &self.name)
51            .field("start", &self.start_ptr())
52            .field("end", &self.end_ptr())
53            .field("byte_len", &self.byte_len())
54            .finish()
55    }
56}
57
58unsafe impl Sync for Section {}
59unsafe impl Send for Section {}
60
61// Waiting on Rust 1.78
62// #[diagnostic::on_unimplemented(message = "This is not an untyped section")]
63/// Marker: untyped [`Section`] handle.
64pub trait IsUntypedSection {}
65
66macro_rules! impl_section_new {
67    ($generic:ident) => {
68        /// # Safety
69        ///
70        /// For macro-generated use only. `bounds` must match the linker-resolved
71        /// (or WASM-initialized) section range for `name`.
72        #[doc(hidden)]
73        pub const unsafe fn new(name: &'static str, bounds: Bounds) -> Self {
74            assert!(
75                ::core::mem::size_of::<$generic>() > 0,
76                "Zero-sized types are not supported"
77            );
78            Self {
79                name,
80                bounds,
81                _phantom: ::core::marker::PhantomData,
82            }
83        }
84    };
85}
86
87macro_rules! impl_bounds_fns {
88    ($generic:ident) => {
89        /// The start address of the section.
90        #[inline(always)]
91        pub fn start_ptr(&self) -> *const T {
92            self.bounds.start_ptr() as *const T
93        }
94
95        /// The end address of the section.
96        #[inline(always)]
97        pub fn end_ptr(&self) -> *const T {
98            self.bounds.end_ptr() as *const T
99        }
100
101        /// The stride of the typed section.
102        #[inline(always)]
103        pub const fn stride(&self) -> usize {
104            assert!(
105                ::core::mem::size_of::<T>() > 0
106                    && ::core::mem::size_of::<T>() * 2 == ::core::mem::size_of::<[T; 2]>()
107            );
108            ::core::mem::size_of::<T>()
109        }
110
111        /// The byte length of the section.
112        #[inline]
113        pub fn byte_len(&self) -> usize {
114            self.bounds.byte_len()
115        }
116
117        /// The number of elements in the section.
118        #[inline]
119        pub fn len(&self) -> usize {
120            self.byte_len() / self.stride()
121        }
122
123        /// True if the section is empty.
124        #[inline]
125        pub fn is_empty(&self) -> bool {
126            self.len() == 0
127        }
128
129        /// The section as a slice.
130        #[inline]
131        pub fn as_slice(&self) -> &[T] {
132            if self.is_empty() {
133                &[]
134            } else {
135                unsafe { ::core::slice::from_raw_parts(self.start_ptr(), self.len()) }
136            }
137        }
138
139        /// The offset of the item in the section, if it is in the section.
140        ///
141        /// This is O(1), as it performs direct pointer arithmetic.
142        #[inline]
143        pub fn offset_of(&self, item: impl $crate::SectionItemLocation<T>) -> Option<usize> {
144            let ptr = item.item_ptr();
145            if ptr < self.start_ptr() || ptr >= self.end_ptr() {
146                None
147            } else {
148                Some(unsafe { ptr.offset_from(self.start_ptr()) as usize })
149            }
150        }
151    };
152}
153
154macro_rules! impl_bounds_traits {
155    ($name:ident < $generic:ident >) => {
156        impl<'a, $generic> ::core::iter::IntoIterator for &'a $name<$generic> {
157            type Item = &'a $generic;
158            type IntoIter = ::core::slice::Iter<'a, $generic>;
159            fn into_iter(self) -> Self::IntoIter {
160                self.as_slice().iter()
161            }
162        }
163
164        impl<T> ::core::ops::Deref for $name<$generic> {
165            type Target = [$generic];
166            fn deref(&self) -> &Self::Target {
167                self.as_slice()
168            }
169        }
170
171        impl<T> ::core::fmt::Debug for $name<$generic> {
172            fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
173                f.debug_struct(stringify!($name))
174                    .field("name", &self.name)
175                    .field("start", &self.start_ptr())
176                    .field("end", &self.end_ptr())
177                    .field("len", &self.len())
178                    .field("stride", &self.stride())
179                    .finish()
180            }
181        }
182
183        impl<T> $crate::__support::SectionItemType for $name<$generic> {
184            type Item = $generic;
185        }
186
187        impl<T> $crate::__support::SectionItemTyped<$generic> for $name<$generic> {
188            type Item = $generic;
189        }
190
191        unsafe impl<$generic> Sync for $name<$generic> where $generic: Sync {}
192        unsafe impl<$generic> Send for $name<$generic> where $generic: Send {}
193    };
194}
195
196/// A typed link section that can be used to store any sized type. The
197/// underlying data is immutable and enumerable. `static` and `const` items are
198/// stored directly in the section.
199///
200/// `static` items are guaranteed to have a valid return from
201/// [`TypedSection::offset_of`] if they are in the section.
202///
203/// Platform note: WASM platforms require `const` items. Use
204/// [`TypedReferenceSection`] for cross-platform support for `static` items.
205#[repr(C)]
206pub struct TypedSection<T: 'static> {
207    name: &'static str,
208    bounds: Bounds,
209    _phantom: ::core::marker::PhantomData<T>,
210}
211
212impl<T: 'static> TypedSection<T> {
213    impl_section_new!(T);
214    impl_bounds_fns!(T);
215}
216
217impl_bounds_traits!(TypedSection<T>);
218
219/// A mutable typed link section that can be used to store any sized type. The
220/// underlying data is (unsafely) mutable and enumerable.
221///
222/// Only `const` items may be submitted to a [`TypedMutableSection`].
223///
224/// Mutating the section (for example via [`TypedMutableSection::as_mut_slice`])
225/// requires exclusive access. See [Exclusive access](crate#exclusive-access) for
226/// more information.
227#[repr(C)]
228pub struct TypedMutableSection<T: 'static> {
229    name: &'static str,
230    bounds: Bounds,
231    _phantom: ::core::marker::PhantomData<T>,
232}
233
234impl<T: 'static> TypedMutableSection<T> {
235    impl_section_new!(T);
236    impl_bounds_fns!(T);
237
238    /// The start address of the section.
239    #[inline]
240    pub fn start_ptr_mut(&self) -> *mut T {
241        self.bounds.start_ptr() as *mut T
242    }
243
244    /// The start address of the section.
245    #[inline]
246    pub fn end_ptr_mut(&self) -> *mut T {
247        self.bounds.end_ptr() as *mut T
248    }
249
250    /// The section as a mutable slice.
251    ///
252    /// # Safety
253    ///
254    /// Mutating the section requires exclusive access. See
255    /// [Exclusive access](crate#exclusive-access) for more information.
256    #[allow(clippy::mut_from_ref)]
257    #[inline]
258    pub unsafe fn as_mut_slice(&self) -> &mut [T] {
259        if self.is_empty() {
260            &mut []
261        } else {
262            unsafe { ::core::slice::from_raw_parts_mut(self.start_ptr() as *mut T, self.len()) }
263        }
264    }
265}
266
267impl_bounds_traits!(TypedMutableSection<T>);
268
269/// A movable typed link section that can be used to store any sized type. The
270/// underlying data is (unsafely) mutable, enumerable, and expected to be
271/// reordered during startup initialization. Each item is paired with a
272/// [`MovableBackref`] that updates a stable [`MovableRef`] slot when the
273/// section is sorted.
274///
275/// Only `static` items may be submitted to a [`TypedMovableSection`].
276///
277/// Mutating or reordering the section requires exclusive access. See
278/// [Exclusive access](crate#exclusive-access) for more information.
279/// [`TypedMovableSection::sort_unstable`] also updates every [`MovableRef`]; any
280/// `&T` obtained before sorting may be stale afterward.
281#[repr(C)]
282pub struct TypedMovableSection<T: 'static> {
283    name: &'static str,
284    bounds: MovableBounds,
285    #[cfg(not(target_family = "wasm"))]
286    backref_state: AtomicU8,
287    _phantom: ::core::marker::PhantomData<T>,
288}
289
290impl<T: 'static> TypedMovableSection<T> {
291    /// # Safety
292    ///
293    /// For macro-generated use only. `bounds` must describe the final layout of
294    /// the linker (or WASM runtime) section after all items are registered.
295    #[doc(hidden)]
296    pub const unsafe fn new(name: &'static str, bounds: MovableBounds) -> Self {
297        assert!(
298            ::core::mem::size_of::<T>() > 0,
299            "Zero-sized types are not supported"
300        );
301        Self {
302            name,
303            bounds,
304            #[cfg(not(target_family = "wasm"))]
305            backref_state: AtomicU8::new(0),
306            _phantom: ::core::marker::PhantomData,
307        }
308    }
309
310    impl_bounds_fns!(T);
311
312    /// The section as a mutable slice.
313    ///
314    /// # Safety
315    ///
316    /// Mutating the section requires exclusive access. See
317    /// [Exclusive access](crate#exclusive-access) for more information.
318    #[allow(clippy::mut_from_ref)]
319    #[inline]
320    pub unsafe fn as_mut_slice(&self) -> &mut [T] {
321        if self.is_empty() {
322            &mut []
323        } else {
324            unsafe { ::core::slice::from_raw_parts_mut(self.start_ptr() as *mut T, self.len()) }
325        }
326    }
327
328    /// The backrefs as a mutable slice, ordered to match the current value
329    /// section addresses.
330    ///
331    /// # Safety
332    ///
333    /// Mutating the backref section requires exclusive access. See
334    /// [Exclusive access](crate#exclusive-access) for more information. Do not
335    /// call while any other slice into either the value or backref linker
336    /// sections is live.
337    #[allow(clippy::mut_from_ref)]
338    #[inline]
339    pub unsafe fn as_mut_backrefs(&self) -> &mut [MovableBackref<T>] {
340        let backrefs_len =
341            self.bounds.backrefs_byte_len() / ::core::mem::size_of::<MovableBackref<T>>();
342        let backrefs = if backrefs_len == 0 {
343            &mut []
344        } else {
345            unsafe {
346                ::core::slice::from_raw_parts_mut(
347                    self.bounds.backrefs_start_ptr() as *mut MovableBackref<T>,
348                    backrefs_len,
349                )
350            }
351        };
352        #[cfg(not(target_family = "wasm"))]
353        unsafe {
354            self.fixup_backrefs(backrefs)
355        };
356        backrefs
357    }
358
359    /// As we cannot guarantee that the linker placed the items and backrefs in
360    /// the same order, we need to sort the backrefs to match the items.
361    ///
362    /// The exception, of course, is WASM where we placed both of them
363    /// ourselves.
364    #[cfg(not(target_family = "wasm"))]
365    unsafe fn fixup_backrefs(&self, backrefs: &mut [MovableBackref<T>]) {
366        match self
367            .backref_state
368            .compare_exchange(0, 1, Ordering::Acquire, Ordering::Acquire)
369        {
370            Ok(_) => {}
371            Err(2) => return,
372            Err(_) => panic!("movable section backrefs already being initialized"),
373        }
374
375        if backrefs.len() != self.len() {
376            panic!(
377                "movable section backref count ({}) does not match item count ({})",
378                backrefs.len(),
379                self.len()
380            );
381        }
382
383        backrefs.sort_unstable_by_key(|backref| backref.current_ptr());
384        self.backref_state.store(2, Ordering::Release);
385    }
386
387    /// Sort the section and backrefs in place.
388    ///
389    /// This algorithm is currently implemented as a quicksort.
390    ///
391    /// # Safety
392    ///
393    /// Reordering the section requires exclusive access. See
394    /// [Exclusive access](crate#exclusive-access) for more information. After
395    /// this returns, every [`MovableRef`] slot points at the new location of its
396    /// item; any `&T` obtained through [`MovableRef`] before the sort must not
397    /// be used.
398    #[allow(unsafe_code)]
399    pub unsafe fn sort_unstable(&self)
400    where
401        T: Ord,
402    {
403        // Trivial case.
404        let main = unsafe { self.as_mut_slice() };
405        if main.len() <= 1 {
406            return;
407        }
408
409        let refs = unsafe { self.as_mut_backrefs() };
410        debug_assert_eq!(main.len(), refs.len());
411
412        fn partition<T: Ord, R>(main: &mut [T], refs: &mut [R]) -> usize {
413            let n = main.len();
414            if n == 0 {
415                return 0;
416            }
417            let pivot = n - 1;
418            let mut i = 0;
419            for j in 0..pivot {
420                if main[j] <= main[pivot] {
421                    main.swap(i, j);
422                    refs.swap(i, j);
423                    i += 1;
424                }
425            }
426            main.swap(i, pivot);
427            refs.swap(i, pivot);
428            i
429        }
430
431        fn recurse<T: Ord, R>(main: &mut [T], refs: &mut [R]) {
432            let n = main.len();
433            if n <= 1 {
434                return;
435            }
436            let p = partition(main, refs);
437            let (ml, mr) = main.split_at_mut(p);
438            let (rl, rr) = refs.split_at_mut(p);
439            recurse(ml, rl);
440            if mr.len() > 1 {
441                recurse(&mut mr[1..], &mut rr[1..]);
442            }
443        }
444
445        recurse(main, refs);
446
447        // TODO: could we avoid the fixup if no changes are made?
448        for (item, backref) in main.iter().zip(refs.iter()) {
449            unsafe {
450                backref.set_current_ptr(item as *const T);
451            }
452        }
453    }
454}
455
456impl_bounds_traits!(TypedMovableSection<T>);
457
458/// A reference to a movable item through a stable pointer slot.
459///
460/// The slot is updated when a [`TypedMovableSection`] is reordered (for example
461/// by [`TypedMovableSection::sort_unstable`]). Do not keep an `&T` from
462/// dereferencing this handle across such an update.
463#[repr(transparent)]
464pub struct MovableRef<T: 'static> {
465    slot: SyncUnsafeCell<*const T>,
466}
467
468impl<T> MovableRef<T> {
469    #[doc(hidden)]
470    pub const fn new(ptr: *const T) -> Self {
471        Self {
472            slot: SyncUnsafeCell::new(ptr),
473        }
474    }
475
476    /// Get a raw pointer to the stable pointer slot inside this handle. Note
477    /// that both this and the SyncUnsafeCell are transparent.
478    #[doc(hidden)]
479    pub const fn slot_ptr(this: *const Self) -> *const UnsafeCell<*const T> {
480        this.cast::<UnsafeCell<*const T>>()
481    }
482
483    /// Raw pointer to the value currently referenced by this slot.
484    pub const fn as_ptr(&self) -> *const T {
485        unsafe { *self.slot.get() }
486    }
487}
488
489impl<T> ::core::ops::Deref for MovableRef<T> {
490    type Target = T;
491    fn deref(&self) -> &Self::Target {
492        unsafe { self.as_ptr().as_ref().expect("MovableRef not initialized") }
493    }
494}
495
496impl<T> ::core::fmt::Debug for MovableRef<T>
497where
498    T: ::core::fmt::Debug,
499{
500    fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
501        (**self).fmt(f)
502    }
503}
504
505impl<T> ::core::fmt::Display for MovableRef<T>
506where
507    T: ::core::fmt::Display,
508{
509    fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
510        (**self).fmt(f)
511    }
512}
513
514unsafe impl<T> Send for MovableRef<T> where T: Send {}
515unsafe impl<T> Sync for MovableRef<T> where T: Sync {}
516
517/// A backref record submitted alongside each item in a [`TypedMovableSection`].
518/// This points to a [`MovableRef`] that lives outside of the section.
519#[repr(C)]
520pub struct MovableBackref<T: 'static> {
521    slot: *const UnsafeCell<*const T>,
522}
523
524impl<T> MovableBackref<T> {
525    #[doc(hidden)]
526    pub const fn new(slot: *const UnsafeCell<*const T>) -> Self {
527        Self { slot }
528    }
529
530    /// Current value of the stable pointer slot as a pointer.
531    pub fn current_ptr(&self) -> *const T {
532        unsafe { ptr::read(UnsafeCell::raw_get(self.slot)) }
533    }
534
535    /// Update the stable pointer slot.
536    ///
537    /// # Safety
538    ///
539    /// Updating the slot requires exclusive access. See
540    /// [Exclusive access](crate#exclusive-access) for more information. Any live
541    /// `&T` or [`MovableRef`] dereference may alias the old or new target.
542    pub unsafe fn set_current_ptr(&self, ptr: *const T) {
543        unsafe {
544            ptr::write(UnsafeCell::raw_get(self.slot), ptr);
545        }
546    }
547}
548
549unsafe impl<T> Send for MovableBackref<T> where T: Send {}
550unsafe impl<T> Sync for MovableBackref<T> where T: Sync {}
551
552/// A typed link section that can be used to store any sized type. The
553/// underlying data is enumerable.
554#[repr(C)]
555pub struct TypedReferenceSection<T: 'static> {
556    name: &'static str,
557    bounds: Bounds,
558    _phantom: ::core::marker::PhantomData<T>,
559}
560
561impl<T: 'static> TypedReferenceSection<T> {
562    impl_section_new!(T);
563    impl_bounds_fns!(T);
564}
565
566impl_bounds_traits!(TypedReferenceSection<T>);
567
568/// A reference to a value in a link section. This allows platforms like WASM
569/// to reference the value, even though the final location is not known until
570/// after initialization.
571#[repr(transparent)]
572pub struct Ref<T: 'static> {
573    #[cfg(target_family = "wasm")]
574    ptr: ::core::cell::UnsafeCell<*const T>,
575    #[cfg(not(target_family = "wasm"))]
576    t: T,
577}
578
579impl<T> Ref<T> {
580    #[cfg(not(target_family = "wasm"))]
581    #[doc(hidden)]
582    pub const fn new(t: T) -> Self {
583        Self { t }
584    }
585
586    #[cfg(target_family = "wasm")]
587    #[doc(hidden)]
588    pub const fn new() -> Self {
589        Self {
590            ptr: ::core::cell::UnsafeCell::new(::core::ptr::null()),
591        }
592    }
593
594    /// # Safety
595    ///
596    /// For macro/runtime registration only. `ptr` must refer to the item's final
597    /// location in the WASM link section. Requires exclusive access. See
598    /// [Exclusive access](crate#exclusive-access) for more information.
599    #[cfg(target_family = "wasm")]
600    #[doc(hidden)]
601    pub unsafe fn set(&self, ptr: *const T) {
602        *self.ptr.get() = ptr;
603    }
604
605    /// Raw pointer to the value (WASM: cell; otherwise `&T` as `*const T`).
606    pub fn as_ptr(&self) -> *const T {
607        #[cfg(target_family = "wasm")]
608        {
609            unsafe { *self.ptr.get() }
610        }
611        #[cfg(not(target_family = "wasm"))]
612        {
613            &self.t as *const T
614        }
615    }
616}
617
618impl<T> ::core::ops::Deref for Ref<T> {
619    type Target = T;
620    fn deref(&self) -> &Self::Target {
621        #[cfg(target_family = "wasm")]
622        unsafe {
623            ::core::ptr::read(self.ptr.get())
624                .as_ref()
625                .expect("Ref not initialized")
626        }
627        #[cfg(not(target_family = "wasm"))]
628        &self.t
629    }
630}
631
632impl<T> ::core::fmt::Debug for Ref<T>
633where
634    T: ::core::fmt::Debug,
635{
636    fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
637        (**self).fmt(f)
638    }
639}
640
641impl<T> ::core::fmt::Display for Ref<T>
642where
643    T: ::core::fmt::Display,
644{
645    fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
646        (**self).fmt(f)
647    }
648}
649
650unsafe impl<T> Send for Ref<T> where T: Send {}
651unsafe impl<T> Sync for Ref<T> where T: Sync {}
link_section/sections.rs

link_section/
sections.rs
