link_section/
sections.rs

1#[cfg(not(target_family = "wasm"))]
2use core::sync::atomic::{AtomicU8, Ordering};
3use core::{cell::UnsafeCell, ptr};
4
5use crate::__support::{Bounds, MovableBounds};
6
7/// An untyped link section that can be used to store any type. The underlying
8/// data is not enumerable.
9#[repr(C)]
10pub struct Section {
11    name: &'static str,
12    bounds: Bounds,
13}
14
15impl Section {
16    /// # Safety
17    ///
18    /// For macro-generated use only. `bounds` must match the linker-resolved
19    /// (or WASM-initialized) section range for `name`.
20    #[doc(hidden)]
21    pub const unsafe fn new(name: &'static str, bounds: Bounds) -> Self {
22        Self { name, bounds }
23    }
24
25    /// The byte length of the section.
26    #[inline]
27    pub fn byte_len(&self) -> usize {
28        self.bounds.range().byte_len()
29    }
30
31    /// The start address of the section.
32    #[inline]
33    pub fn start_ptr(&self) -> *const () {
34        self.bounds.range().start_ptr()
35    }
36    /// The end address of the section.
37    #[inline]
38    pub fn end_ptr(&self) -> *const () {
39        self.bounds.range().end_ptr()
40    }
41
42    /// Ensures that a section exists at the given path.
43    #[doc(hidden)]
44    pub const fn __validate<T: IsUntypedSection>(_section: &T) {}
45}
46
47impl ::core::fmt::Debug for Section {
48    fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
49        f.debug_struct("Section")
50            .field("name", &self.name)
51            .field("start", &self.start_ptr())
52            .field("end", &self.end_ptr())
53            .field("byte_len", &self.byte_len())
54            .finish()
55    }
56}
57
58unsafe impl Sync for Section {}
59unsafe impl Send for Section {}
60
61// Waiting on Rust 1.78
62// #[diagnostic::on_unimplemented(message = "This is not an untyped section")]
63/// Marker: untyped [`Section`] handle.
64pub trait IsUntypedSection {}
65
66macro_rules! impl_section_new {
67    ($generic:ident) => {
68        /// # Safety
69        ///
70        /// For macro-generated use only. `bounds` must match the linker-resolved
71        /// (or WASM-initialized) section range for `name`.
72        #[doc(hidden)]
73        pub const unsafe fn new(name: &'static str, bounds: Bounds) -> Self {
74            assert!(
75                ::core::mem::size_of::<$generic>() > 0,
76                "Zero-sized types are not supported"
77            );
78            Self {
79                name,
80                bounds,
81                _phantom: ::core::marker::PhantomData,
82            }
83        }
84    };
85}
86
87macro_rules! impl_bounds_fns {
88    ($generic:ident) => {
89        /// The start address of the section.
90        #[inline(always)]
91        pub fn start_ptr(&self) -> *const T {
92            self.bounds.range().start_ptr() as *const T
93        }
94
95        /// The end address of the section.
96        #[inline(always)]
97        pub fn end_ptr(&self) -> *const T {
98            self.bounds.range().end_ptr() as *const T
99        }
100
101        /// The stride of the typed section.
102        #[inline(always)]
103        pub const fn stride(&self) -> usize {
104            assert!(
105                ::core::mem::size_of::<T>() > 0
106                    && ::core::mem::size_of::<T>() * 2 == ::core::mem::size_of::<[T; 2]>()
107            );
108            ::core::mem::size_of::<T>()
109        }
110
111        /// The byte length of the section.
112        #[inline]
113        pub fn byte_len(&self) -> usize {
114            self.bounds.range().byte_len()
115        }
116
117        /// The number of elements in the section.
118        #[inline]
119        pub fn len(&self) -> usize {
120            self.byte_len() / self.stride()
121        }
122
123        /// True if the section is empty.
124        #[inline]
125        pub fn is_empty(&self) -> bool {
126            self.len() == 0
127        }
128
129        /// The section as a slice.
130        #[inline]
131        pub fn as_slice(&self) -> &[T] {
132            // SAFETY: `bounds.range()` is the section's valid, aligned,
133            // readable range of `T`s (linker-resolved or WASM-materialised),
134            // and the returned slice is borrowed from `&self`, so it cannot
135            // outlive the section.
136            unsafe { self.bounds.range().slice_of::<T>(self.stride()) }
137        }
138
139        /// The offset of the item in the section, if it is in the section.
140        ///
141        /// This is O(1), as it performs direct pointer arithmetic.
142        #[inline]
143        pub fn offset_of(&self, item: impl $crate::SectionItemLocation<T>) -> Option<usize> {
144            // Resolve the bounds up-front (complex operation on some platforms)
145            let range = self.bounds.range();
146            let start = range.start_ptr() as *const T;
147            let end = range.end_ptr() as *const T;
148            let ptr = item.item_ptr();
149            if ptr < start || ptr >= end {
150                None
151            } else {
152                Some(unsafe { ptr.offset_from(start) as usize })
153            }
154        }
155    };
156}
157
158macro_rules! impl_bounds_traits {
159    ($name:ident < $generic:ident >) => {
160        impl<'a, $generic> ::core::iter::IntoIterator for &'a $name<$generic> {
161            type Item = &'a $generic;
162            type IntoIter = ::core::slice::Iter<'a, $generic>;
163            fn into_iter(self) -> Self::IntoIter {
164                self.as_slice().iter()
165            }
166        }
167
168        impl<T> ::core::ops::Deref for $name<$generic> {
169            type Target = [$generic];
170            fn deref(&self) -> &Self::Target {
171                self.as_slice()
172            }
173        }
174
175        impl<T> ::core::fmt::Debug for $name<$generic> {
176            fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
177                f.debug_struct(stringify!($name))
178                    .field("name", &self.name)
179                    .field("start", &self.start_ptr())
180                    .field("end", &self.end_ptr())
181                    .field("len", &self.len())
182                    .field("stride", &self.stride())
183                    .finish()
184            }
185        }
186
187        impl<T> $crate::__support::SectionItemType for $name<$generic> {
188            type Item = $generic;
189        }
190
191        impl<T> $crate::__support::SectionItemTyped<$generic> for $name<$generic> {
192            type Item = $generic;
193        }
194
195        unsafe impl<$generic> Sync for $name<$generic> where $generic: Sync {}
196        unsafe impl<$generic> Send for $name<$generic> where $generic: Send {}
197    };
198}
199
200/// A typed link section that can be used to store any sized type. The
201/// underlying data is immutable and enumerable. `static` and `const` items are
202/// stored directly in the section.
203///
204/// `static` items are guaranteed to have a valid return from
205/// [`TypedSection::offset_of`] if they are in the section.
206///
207/// Platform note: WASM platforms require `const` items. Use
208/// [`TypedReferenceSection`] for cross-platform support for `static` items.
209#[repr(C)]
210pub struct TypedSection<T: 'static> {
211    name: &'static str,
212    bounds: Bounds,
213    _phantom: ::core::marker::PhantomData<T>,
214}
215
216impl<T: 'static> TypedSection<T> {
217    impl_section_new!(T);
218    impl_bounds_fns!(T);
219}
220
221impl_bounds_traits!(TypedSection<T>);
222
223/// A mutable typed link section that can be used to store any sized type. The
224/// underlying data is (unsafely) mutable and enumerable.
225///
226/// Only `const` items may be submitted to a [`TypedMutableSection`].
227///
228/// Mutating the section (for example via [`TypedMutableSection::as_mut_slice`])
229/// requires exclusive access. See [Exclusive access](crate#exclusive-access) for
230/// more information.
231#[repr(C)]
232pub struct TypedMutableSection<T: 'static> {
233    name: &'static str,
234    bounds: Bounds,
235    _phantom: ::core::marker::PhantomData<T>,
236}
237
238impl<T: 'static> TypedMutableSection<T> {
239    impl_section_new!(T);
240    impl_bounds_fns!(T);
241
242    /// The start address of the section.
243    #[inline]
244    pub fn start_ptr_mut(&self) -> *mut T {
245        self.bounds.range().start_ptr() as *mut T
246    }
247
248    /// The start address of the section.
249    #[inline]
250    pub fn end_ptr_mut(&self) -> *mut T {
251        self.bounds.range().end_ptr() as *mut T
252    }
253
254    /// The section as a mutable slice.
255    ///
256    /// # Safety
257    ///
258    /// Mutating the section requires exclusive access. See
259    /// [Exclusive access](crate#exclusive-access) for more information.
260    #[allow(clippy::mut_from_ref)]
261    #[inline]
262    pub unsafe fn as_mut_slice(&self) -> &mut [T] {
263        // SAFETY: caller upholds exclusivity (see `# Safety` above); the range
264        // is the section's valid, aligned `T` range.
265        unsafe { self.bounds.range().slice_of_mut::<T>(self.stride()) }
266    }
267}
268
269impl_bounds_traits!(TypedMutableSection<T>);
270
271/// A movable typed link section that can be used to store any sized type. The
272/// underlying data is (unsafely) mutable, enumerable, and expected to be
273/// reordered during startup initialization. Each item is paired with a
274/// [`MovableBackref`] that updates a stable [`MovableRef`] slot when the
275/// section is sorted.
276///
277/// Only `static` items may be submitted to a [`TypedMovableSection`].
278///
279/// Mutating or reordering the section requires exclusive access. See
280/// [Exclusive access](crate#exclusive-access) for more information.
281/// [`TypedMovableSection::sort_unstable`] also updates every [`MovableRef`]; any
282/// `&T` obtained before sorting may be stale afterward.
283#[repr(C)]
284pub struct TypedMovableSection<T: 'static> {
285    name: &'static str,
286    bounds: MovableBounds,
287    #[cfg(not(target_family = "wasm"))]
288    backref_state: AtomicU8,
289    _phantom: ::core::marker::PhantomData<T>,
290}
291
292impl<T: 'static> TypedMovableSection<T> {
293    /// # Safety
294    ///
295    /// For macro-generated use only. `bounds` must describe the final layout of
296    /// the linker (or WASM runtime) section after all items are registered.
297    #[doc(hidden)]
298    pub const unsafe fn new(name: &'static str, bounds: MovableBounds) -> Self {
299        assert!(
300            ::core::mem::size_of::<T>() > 0,
301            "Zero-sized types are not supported"
302        );
303        Self {
304            name,
305            bounds,
306            #[cfg(not(target_family = "wasm"))]
307            backref_state: AtomicU8::new(0),
308            _phantom: ::core::marker::PhantomData,
309        }
310    }
311
312    impl_bounds_fns!(T);
313
314    /// The section as a mutable slice.
315    ///
316    /// # Safety
317    ///
318    /// Mutating the section requires exclusive access. See
319    /// [Exclusive access](crate#exclusive-access) for more information.
320    #[allow(clippy::mut_from_ref)]
321    #[inline]
322    pub unsafe fn as_mut_slice(&self) -> &mut [T] {
323        // SAFETY: caller upholds exclusivity (see `# Safety` above); the range
324        // is the section's valid, aligned `T` range.
325        unsafe { self.bounds.range().slice_of_mut::<T>(self.stride()) }
326    }
327
328    /// The backrefs as a mutable slice, ordered to match the current value
329    /// section addresses.
330    ///
331    /// # Safety
332    ///
333    /// Mutating the backref section requires exclusive access. See
334    /// [Exclusive access](crate#exclusive-access) for more information. Do not
335    /// call while any other slice into either the value or backref linker
336    /// sections is live.
337    #[allow(clippy::mut_from_ref)]
338    #[inline]
339    pub unsafe fn as_mut_backrefs(&self) -> &mut [MovableBackref<T>] {
340        let range = self.bounds.backrefs_range();
341        // SAFETY: caller upholds exclusivity (see `# Safety` above); the range
342        // is the backref section's valid, aligned `MovableBackref<T>` range.
343        let backrefs = unsafe {
344            range.slice_of_mut::<MovableBackref<T>>(::core::mem::size_of::<MovableBackref<T>>())
345        };
346        #[cfg(not(target_family = "wasm"))]
347        unsafe {
348            self.fixup_backrefs(backrefs)
349        };
350        backrefs
351    }
352
353    /// As we cannot guarantee that the linker placed the items and backrefs in
354    /// the same order, we need to sort the backrefs to match the items.
355    ///
356    /// The exception, of course, is WASM where we placed both of them
357    /// ourselves.
358    #[cfg(not(target_family = "wasm"))]
359    unsafe fn fixup_backrefs(&self, backrefs: &mut [MovableBackref<T>]) {
360        match self
361            .backref_state
362            .compare_exchange(0, 1, Ordering::Acquire, Ordering::Acquire)
363        {
364            Ok(_) => {}
365            Err(2) => return,
366            Err(_) => panic!("movable section backrefs already being initialized"),
367        }
368
369        if backrefs.len() != self.len() {
370            panic!(
371                "movable section backref count ({}) does not match item count ({})",
372                backrefs.len(),
373                self.len()
374            );
375        }
376
377        backrefs.sort_unstable_by_key(|backref| backref.current_ptr());
378        self.backref_state.store(2, Ordering::Release);
379    }
380
381    /// Sort the section and backrefs in place.
382    ///
383    /// This algorithm is currently implemented as a quicksort.
384    ///
385    /// # Safety
386    ///
387    /// Reordering the section requires exclusive access. See
388    /// [Exclusive access](crate#exclusive-access) for more information. After
389    /// this returns, every [`MovableRef`] slot points at the new location of its
390    /// item; any `&T` obtained through [`MovableRef`] before the sort must not
391    /// be used.
392    #[allow(unsafe_code)]
393    pub unsafe fn sort_unstable(&self)
394    where
395        T: Ord,
396    {
397        // Trivial case.
398        let main = unsafe { self.as_mut_slice() };
399        if main.len() <= 1 {
400            return;
401        }
402
403        let refs = unsafe { self.as_mut_backrefs() };
404        debug_assert_eq!(main.len(), refs.len());
405
406        fn partition<T: Ord, R>(main: &mut [T], refs: &mut [R]) -> usize {
407            let n = main.len();
408            if n == 0 {
409                return 0;
410            }
411            let pivot = n - 1;
412            let mut i = 0;
413            for j in 0..pivot {
414                if main[j] <= main[pivot] {
415                    main.swap(i, j);
416                    refs.swap(i, j);
417                    i += 1;
418                }
419            }
420            main.swap(i, pivot);
421            refs.swap(i, pivot);
422            i
423        }
424
425        fn recurse<T: Ord, R>(main: &mut [T], refs: &mut [R]) {
426            let n = main.len();
427            if n <= 1 {
428                return;
429            }
430            let p = partition(main, refs);
431            let (ml, mr) = main.split_at_mut(p);
432            let (rl, rr) = refs.split_at_mut(p);
433            recurse(ml, rl);
434            if mr.len() > 1 {
435                recurse(&mut mr[1..], &mut rr[1..]);
436            }
437        }
438
439        recurse(main, refs);
440
441        // TODO: could we avoid the fixup if no changes are made?
442        for (item, backref) in main.iter().zip(refs.iter()) {
443            unsafe {
444                backref.set_current_ptr(item as *const T);
445            }
446        }
447    }
448}
449
450impl_bounds_traits!(TypedMovableSection<T>);
451
452/// A reference to a movable item through a stable pointer slot.
453///
454/// The slot is updated when a [`TypedMovableSection`] is reordered (for example
455/// by [`TypedMovableSection::sort_unstable`]). Do not keep an `&T` from
456/// dereferencing this handle across such an update.
457///
458/// The platform-specific representation lives in [`MovableRefStorage`], whose
459/// `slot` is its first field so [`MovableRef::slot_ptr`] is an offset-0 cast.
460///
461/// [`MovableRefStorage`]: crate::__support::MovableRefStorage
462#[repr(C)]
463pub struct MovableRef<T: 'static> {
464    storage: crate::__support::MovableRefStorage<T>,
465}
466
467impl<T> MovableRef<T> {
468    #[doc(hidden)]
469    pub const fn new(storage: crate::__support::MovableRefStorage<T>) -> Self {
470        Self { storage }
471    }
472
473    /// Get a raw pointer to the stable pointer slot inside this handle. `slot`
474    /// is the first field of the storage, so this is an offset-0 cast.
475    #[doc(hidden)]
476    pub const fn slot_ptr(this: *const Self) -> *const UnsafeCell<*const T> {
477        this.cast::<UnsafeCell<*const T>>()
478    }
479
480    /// Raw pointer to the value currently referenced by this slot, read without
481    /// flattening.
482    pub(crate) const fn as_ptr(&self) -> *const T {
483        self.storage.as_ptr()
484    }
485}
486
487impl<T> ::core::ops::Deref for MovableRef<T> {
488    type Target = T;
489    fn deref(&self) -> &Self::Target {
490        self.storage.get()
491    }
492}
493
494impl<T> ::core::fmt::Debug for MovableRef<T>
495where
496    T: ::core::fmt::Debug,
497{
498    fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
499        (**self).fmt(f)
500    }
501}
502
503impl<T> ::core::fmt::Display for MovableRef<T>
504where
505    T: ::core::fmt::Display,
506{
507    fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
508        (**self).fmt(f)
509    }
510}
511
512unsafe impl<T> Send for MovableRef<T> where T: Send {}
513unsafe impl<T> Sync for MovableRef<T> where T: Sync {}
514
515/// A backref record submitted alongside each item in a [`TypedMovableSection`].
516/// This points to a [`MovableRef`] that lives outside of the section.
517#[repr(C)]
518pub struct MovableBackref<T: 'static> {
519    slot: *const UnsafeCell<*const T>,
520}
521
522impl<T> MovableBackref<T> {
523    #[doc(hidden)]
524    pub const fn new(slot: *const UnsafeCell<*const T>) -> Self {
525        Self { slot }
526    }
527
528    /// Current value of the stable pointer slot as a pointer.
529    pub fn current_ptr(&self) -> *const T {
530        unsafe { ptr::read(UnsafeCell::raw_get(self.slot)) }
531    }
532
533    /// Update the stable pointer slot.
534    ///
535    /// # Safety
536    ///
537    /// Updating the slot requires exclusive access. See
538    /// [Exclusive access](crate#exclusive-access) for more information. Any live
539    /// `&T` or [`MovableRef`] dereference may alias the old or new target.
540    pub unsafe fn set_current_ptr(&self, ptr: *const T) {
541        unsafe {
542            ptr::write(UnsafeCell::raw_get(self.slot), ptr);
543        }
544    }
545}
546
547unsafe impl<T> Send for MovableBackref<T> where T: Send {}
548unsafe impl<T> Sync for MovableBackref<T> where T: Sync {}
549
550/// A typed link section that can be used to store any sized type. The
551/// underlying data is enumerable.
552#[repr(C)]
553pub struct TypedReferenceSection<T: 'static> {
554    name: &'static str,
555    bounds: Bounds,
556    _phantom: ::core::marker::PhantomData<T>,
557}
558
559impl<T: 'static> TypedReferenceSection<T> {
560    impl_section_new!(T);
561    impl_bounds_fns!(T);
562}
563
564impl_bounds_traits!(TypedReferenceSection<T>);
565
566/// A reference to a value in a link section. This allows platforms like WASM
567/// to reference the value, even though the final location is not known until
568/// after initialization.
569#[repr(C)]
570pub struct Ref<T: 'static> {
571    storage: crate::__support::RefStorage<T>,
572}
573
574impl<T> Ref<T> {
575    #[doc(hidden)]
576    pub const fn new(storage: crate::__support::RefStorage<T>) -> Self {
577        Self { storage }
578    }
579
580    /// Raw pointer to the value, read without flattening on WASM. Internal
581    /// plumbing for [`SectionItemLocation`](crate::SectionItemLocation):
582    /// `offset_of` flattens via the section bounds first. Use `Deref` otherwise.
583    pub(crate) fn as_ptr(&self) -> *const T {
584        self.storage.as_ptr()
585    }
586}
587
588impl<T> ::core::ops::Deref for Ref<T> {
589    type Target = T;
590    fn deref(&self) -> &Self::Target {
591        self.storage.get()
592    }
593}
594
595impl<T> ::core::fmt::Debug for Ref<T>
596where
597    T: ::core::fmt::Debug,
598{
599    fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
600        (**self).fmt(f)
601    }
602}
603
604impl<T> ::core::fmt::Display for Ref<T>
605where
606    T: ::core::fmt::Display,
607{
608    fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
609        (**self).fmt(f)
610    }
611}
612
613unsafe impl<T> Send for Ref<T> where T: Send {}
614unsafe impl<T> Sync for Ref<T> where T: Sync {}
link_section/sections.rs

link_section/
sections.rs
