link_section/
sections.rs

1#[cfg(not(target_family = "wasm"))]
2use core::sync::atomic::{AtomicU8, Ordering};
3use core::{cell::UnsafeCell, ptr};
4
5use crate::__support::{Bounds, MovableBounds, SyncUnsafeCell};
6
7/// An untyped link section that can be used to store any type. The underlying
8/// data is not enumerable.
9#[repr(C)]
10pub struct Section {
11    name: &'static str,
12    bounds: Bounds,
13}
14
15impl Section {
16    /// # Safety
17    ///
18    /// For macro-generated use only. `bounds` must match the linker-resolved
19    /// (or WASM-initialized) section range for `name`.
20    #[doc(hidden)]
21    pub const unsafe fn new(name: &'static str, bounds: Bounds) -> Self {
22        Self { name, bounds }
23    }
24
25    /// The byte length of the section.
26    #[inline]
27    pub fn byte_len(&self) -> usize {
28        self.bounds.byte_len()
29    }
30
31    /// The start address of the section.
32    #[inline]
33    pub fn start_ptr(&self) -> *const () {
34        self.bounds.start_ptr()
35    }
36    /// The end address of the section.
37    #[inline]
38    pub fn end_ptr(&self) -> *const () {
39        self.bounds.end_ptr()
40    }
41
42    /// Ensures that a section exists at the given path.
43    #[doc(hidden)]
44    pub const fn __validate<T: IsUntypedSection>(_section: &T) {}
45}
46
47impl ::core::fmt::Debug for Section {
48    fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
49        f.debug_struct("Section")
50            .field("name", &self.name)
51            .field("start", &self.start_ptr())
52            .field("end", &self.end_ptr())
53            .field("byte_len", &self.byte_len())
54            .finish()
55    }
56}
57
58unsafe impl Sync for Section {}
59unsafe impl Send for Section {}
60
61// Waiting on Rust 1.78
62// #[diagnostic::on_unimplemented(message = "This is not an untyped section")]
63/// Marker: untyped [`Section`] handle.
64pub trait IsUntypedSection {}
65
66macro_rules! impl_section_new {
67    ($generic:ident) => {
68        /// # Safety
69        ///
70        /// For macro-generated use only. `bounds` must match the linker-resolved
71        /// (or WASM-initialized) section range for `name`.
72        #[doc(hidden)]
73        pub const unsafe fn new(name: &'static str, bounds: Bounds) -> Self {
74            assert!(
75                ::core::mem::size_of::<$generic>() > 0,
76                "Zero-sized types are not supported"
77            );
78            Self {
79                name,
80                bounds,
81                _phantom: ::core::marker::PhantomData,
82            }
83        }
84    };
85}
86
87macro_rules! impl_bounds_fns {
88    ($generic:ident) => {
89        /// The start address of the section.
90        #[inline(always)]
91        pub fn start_ptr(&self) -> *const T {
92            self.bounds.start_ptr() as *const T
93        }
94
95        /// The end address of the section.
96        #[inline(always)]
97        pub fn end_ptr(&self) -> *const T {
98            self.bounds.end_ptr() as *const T
99        }
100
101        /// The stride of the typed section.
102        #[inline(always)]
103        pub const fn stride(&self) -> usize {
104            assert!(
105                ::core::mem::size_of::<T>() > 0
106                    && ::core::mem::size_of::<T>() * 2 == ::core::mem::size_of::<[T; 2]>()
107            );
108            ::core::mem::size_of::<T>()
109        }
110
111        /// The byte length of the section.
112        #[inline]
113        pub fn byte_len(&self) -> usize {
114            self.bounds.byte_len()
115        }
116
117        /// The number of elements in the section.
118        #[inline]
119        pub fn len(&self) -> usize {
120            self.byte_len() / self.stride()
121        }
122
123        /// True if the section is empty.
124        #[inline]
125        pub fn is_empty(&self) -> bool {
126            self.len() == 0
127        }
128
129        /// The section as a slice.
130        #[inline]
131        pub fn as_slice(&self) -> &[T] {
132            if self.is_empty() {
133                &[]
134            } else {
135                unsafe { ::core::slice::from_raw_parts(self.start_ptr(), self.len()) }
136            }
137        }
138    };
139}
140
141macro_rules! impl_bounds_traits {
142    ($name:ident < $generic:ident >) => {
143        impl<'a, $generic> ::core::iter::IntoIterator for &'a $name<$generic> {
144            type Item = &'a $generic;
145            type IntoIter = ::core::slice::Iter<'a, $generic>;
146            fn into_iter(self) -> Self::IntoIter {
147                self.as_slice().iter()
148            }
149        }
150
151        impl<T> ::core::ops::Deref for $name<$generic> {
152            type Target = [$generic];
153            fn deref(&self) -> &Self::Target {
154                self.as_slice()
155            }
156        }
157
158        impl<T> ::core::fmt::Debug for $name<$generic> {
159            fn fmt(&self, f: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result {
160                f.debug_struct(stringify!($name))
161                    .field("name", &self.name)
162                    .field("start", &self.start_ptr())
163                    .field("end", &self.end_ptr())
164                    .field("len", &self.len())
165                    .field("stride", &self.stride())
166                    .finish()
167            }
168        }
169
170        impl<T> $crate::__support::SectionItemType for $name<$generic> {
171            type Item = $generic;
172        }
173
174        impl<T> $crate::__support::SectionItemTyped<$generic> for $name<$generic> {
175            type Item = $generic;
176        }
177
178        unsafe impl<$generic> Sync for $name<$generic> where $generic: Sync {}
179        unsafe impl<$generic> Send for $name<$generic> where $generic: Send {}
180    };
181}
182
183/// A typed link section that can be used to store any sized type. The
184/// underlying data is immutable and enumerable. `static` and `const` items are
185/// stored directly in the section.
186///
187/// `static` items are guaranteed to have a valid return from
188/// [`TypedSection::offset_of`] if they are in the section.
189///
190/// Platform note: WASM platforms require `const` items. Use
191/// [`TypedReferenceSection`] for cross-platform support for `static` items.
192#[repr(C)]
193pub struct TypedSection<T: 'static> {
194    name: &'static str,
195    bounds: Bounds,
196    _phantom: ::core::marker::PhantomData<T>,
197}
198
199impl<T: 'static> TypedSection<T> {
200    impl_section_new!(T);
201    impl_bounds_fns!(T);
202
203    /// The offset of the item in the section, if it is in the section.
204    #[inline]
205    pub fn offset_of(&self, item: &T) -> Option<usize> {
206        let ptr = item as *const T;
207        if ptr < self.start_ptr() || ptr >= self.end_ptr() {
208            None
209        } else {
210            Some(unsafe { ptr.offset_from(self.start_ptr()) as usize })
211        }
212    }
213}
214
215impl_bounds_traits!(TypedSection<T>);
216
217/// A mutable typed link section that can be used to store any sized type. The
218/// underlying data is (unsafely) mutable and enumerable.
219///
220/// Only `const` items may be submitted to a [`TypedMutableSection`].
221///
222/// Mutating the section (for example via [`TypedMutableSection::as_mut_slice`])
223/// requires exclusive access. See [Exclusive access](crate#exclusive-access) for
224/// more information.
225#[repr(C)]
226pub struct TypedMutableSection<T: 'static> {
227    name: &'static str,
228    bounds: Bounds,
229    _phantom: ::core::marker::PhantomData<T>,
230}
231
232impl<T: 'static> TypedMutableSection<T> {
233    impl_section_new!(T);
234    impl_bounds_fns!(T);
235
236    /// The offset of the item in the section, if it is in the section.
237    #[inline]
238    pub fn offset_of(&self, item: &T) -> Option<usize> {
239        let ptr = item as *const T;
240        if ptr < self.start_ptr() || ptr >= self.end_ptr() {
241            None
242        } else {
243            Some(unsafe { ptr.offset_from(self.start_ptr()) as usize })
244        }
245    }
246
247    /// The start address of the section.
248    #[inline]
249    pub fn start_ptr_mut(&self) -> *mut T {
250        self.bounds.start_ptr() as *mut T
251    }
252
253    /// The start address of the section.
254    #[inline]
255    pub fn end_ptr_mut(&self) -> *mut T {
256        self.bounds.end_ptr() as *mut T
257    }
258
259    /// The section as a mutable slice.
260    ///
261    /// # Safety
262    ///
263    /// Mutating the section requires exclusive access. See
264    /// [Exclusive access](crate#exclusive-access) for more information.
265    #[allow(clippy::mut_from_ref)]
266    #[inline]
267    pub unsafe fn as_mut_slice(&self) -> &mut [T] {
268        if self.is_empty() {
269            &mut []
270        } else {
271            unsafe { ::core::slice::from_raw_parts_mut(self.start_ptr() as *mut T, self.len()) }
272        }
273    }
274}
275
276impl_bounds_traits!(TypedMutableSection<T>);
277
278/// A movable typed link section that can be used to store any sized type. The
279/// underlying data is (unsafely) mutable, enumerable, and expected to be
280/// reordered during startup initialization. Each item is paired with a
281/// [`MovableBackref`] that updates a stable [`MovableRef`] slot when the
282/// section is sorted.
283///
284/// Only `static` items may be submitted to a [`TypedMovableSection`].
285///
286/// Mutating or reordering the section requires exclusive access. See
287/// [Exclusive access](crate#exclusive-access) for more information.
288/// [`TypedMovableSection::sort_unstable`] also updates every [`MovableRef`]; any
289/// `&T` obtained before sorting may be stale afterward.
290#[repr(C)]
291pub struct TypedMovableSection<T: 'static> {
292    name: &'static str,
293    bounds: MovableBounds,
294    #[cfg(not(target_family = "wasm"))]
295    backref_state: AtomicU8,
296    _phantom: ::core::marker::PhantomData<T>,
297}
298
299impl<T: 'static> TypedMovableSection<T> {
300    /// # Safety
301    ///
302    /// For macro-generated use only. `bounds` must describe the final layout of
303    /// the linker (or WASM runtime) section after all items are registered.
304    #[doc(hidden)]
305    pub const unsafe fn new(name: &'static str, bounds: MovableBounds) -> Self {
306        assert!(
307            ::core::mem::size_of::<T>() > 0,
308            "Zero-sized types are not supported"
309        );
310        Self {
311            name,
312            bounds,
313            #[cfg(not(target_family = "wasm"))]
314            backref_state: AtomicU8::new(0),
315            _phantom: ::core::marker::PhantomData,
316        }
317    }
318
319    impl_bounds_fns!(T);
320
321    /// The offset of the item in the section, if it is in the section.
322    #[inline]
323    pub fn offset_of(&self, item: &T) -> Option<usize> {
324        let ptr = item as *const T;
325        if ptr < self.start_ptr() || ptr >= self.end_ptr() {
326            None
327        } else {
328            Some(unsafe { ptr.offset_from(self.start_ptr()) as usize })
329        }
330    }
331
332    /// The section as a mutable slice.
333    ///
334    /// # Safety
335    ///
336    /// Mutating the section requires exclusive access. See
337    /// [Exclusive access](crate#exclusive-access) for more information.
338    #[allow(clippy::mut_from_ref)]
339    #[inline]
340    pub unsafe fn as_mut_slice(&self) -> &mut [T] {
341        if self.is_empty() {
342            &mut []
343        } else {
344            unsafe { ::core::slice::from_raw_parts_mut(self.start_ptr() as *mut T, self.len()) }
345        }
346    }
347
348    /// The backrefs as a mutable slice, ordered to match the current value
349    /// section addresses.
350    ///
351    /// # Safety
352    ///
353    /// Mutating the backref section requires exclusive access. See
354    /// [Exclusive access](crate#exclusive-access) for more information. Do not
355    /// call while any other slice into either the value or backref linker
356    /// sections is live.
357    #[allow(clippy::mut_from_ref)]
358    #[inline]
359    pub unsafe fn as_mut_backrefs(&self) -> &mut [MovableBackref<T>] {
360        let backrefs_len =
361            self.bounds.backrefs_byte_len() / ::core::mem::size_of::<MovableBackref<T>>();
362        let backrefs = if backrefs_len == 0 {
363            &mut []
364        } else {
365            unsafe {
366                ::core::slice::from_raw_parts_mut(
367                    self.bounds.backrefs_start_ptr() as *mut MovableBackref<T>,
368                    backrefs_len,
369                )
370            }
371        };
372        #[cfg(not(target_family = "wasm"))]
373        unsafe {
374            self.fixup_backrefs(backrefs)
375        };
376        backrefs
377    }
378
379    /// As we cannot guarantee that the linker placed the items and backrefs in
380    /// the same order, we need to sort the backrefs to match the items.
381    ///
382    /// The exception, of course, is WASM where we placed both of them
383    /// ourselves.
384    #[cfg(not(target_family = "wasm"))]
385    unsafe fn fixup_backrefs(&self, backrefs: &mut [MovableBackref<T>]) {
386        match self
387            .backref_state
388            .compare_exchange(0, 1, Ordering::Acquire, Ordering::Acquire)
389        {
390            Ok(_) => {}
391            Err(2) => return,
392            Err(_) => panic!("movable section backrefs already being initialized"),
393        }
394
395        if backrefs.len() != self.len() {
396            panic!("movable section backref count does not match item count");
397        }
398
399        backrefs.sort_unstable_by_key(|backref| backref.current_ptr());
400        self.backref_state.store(2, Ordering::Release);
401    }
402
403    /// Sort the section and backrefs in place.
404    ///
405    /// This algorithm is currently implemented as a quicksort.
406    ///
407    /// # Safety
408    ///
409    /// Reordering the section requires exclusive access. See
410    /// [Exclusive access](crate#exclusive-access) for more information. After
411    /// this returns, every [`MovableRef`] slot points at the new location of its
412    /// item; any `&T` obtained through [`MovableRef`] before the sort must not
413    /// be used.
414    #[allow(unsafe_code)]
415    pub unsafe fn sort_unstable(&self)
416    where
417        T: Ord,
418    {
419        // Trivial case.
420        let main = unsafe { self.as_mut_slice() };
421        if main.len() <= 1 {
422            return;
423        }
424
425        let refs = unsafe { self.as_mut_backrefs() };
426        debug_assert_eq!(main.len(), refs.len());
427
428        fn partition<T: Ord, R>(main: &mut [T], refs: &mut [R]) -> usize {
429            let n = main.len();
430            if n == 0 {
431                return 0;
432            }
433            let pivot = n - 1;
434            let mut i = 0;
435            for j in 0..pivot {
436                if main[j] <= main[pivot] {
437                    main.swap(i, j);
438                    refs.swap(i, j);
439                    i += 1;
440                }
441            }
442            main.swap(i, pivot);
443            refs.swap(i, pivot);
444            i
445        }
446
447        fn recurse<T: Ord, R>(main: &mut [T], refs: &mut [R]) {
448            let n = main.len();
449            if n <= 1 {
450                return;
451            }
452            let p = partition(main, refs);
453            let (ml, mr) = main.split_at_mut(p);
454            let (rl, rr) = refs.split_at_mut(p);
455            recurse(ml, rl);
456            if mr.len() > 1 {
457                recurse(&mut mr[1..], &mut rr[1..]);
458            }
459        }
460
461        recurse(main, refs);
462
463        // TODO: could we avoid the fixup if no changes are made?
464        for (item, backref) in main.iter().zip(refs.iter()) {
465            unsafe {
466                backref.set_current_ptr(item as *const T);
467            }
468        }
469    }
470}
471
472impl_bounds_traits!(TypedMovableSection<T>);
473
474/// A reference to a movable item through a stable pointer slot.
475///
476/// The slot is updated when a [`TypedMovableSection`] is reordered (for example
477/// by [`TypedMovableSection::sort_unstable`]). Do not keep an `&T` from
478/// dereferencing this handle across such an update.
479#[repr(transparent)]
480pub struct MovableRef<T: 'static> {
481    slot: SyncUnsafeCell<*const T>,
482}
483
484impl<T> MovableRef<T> {
485    #[doc(hidden)]
486    pub const fn new(ptr: *const T) -> Self {
487        Self {
488            slot: SyncUnsafeCell::new(ptr),
489        }
490    }
491
492    /// Get a raw pointer to the stable pointer slot inside this handle. Note
493    /// that both this and the SyncUnsafeCell are transparent.
494    #[doc(hidden)]
495    pub const fn slot_ptr(this: *const Self) -> *const UnsafeCell<*const T> {
496        this.cast::<UnsafeCell<*const T>>()
497    }
498
499    /// Raw pointer to the value currently referenced by this slot.
500    pub fn as_ptr(&self) -> *const T {
501        unsafe { *self.slot.get() }
502    }
503}
504
505impl<T> ::core::ops::Deref for MovableRef<T> {
506    type Target = T;
507    fn deref(&self) -> &Self::Target {
508        unsafe { self.as_ptr().as_ref().expect("MovableRef not initialized") }
509    }
510}
511
512unsafe impl<T> Send for MovableRef<T> where T: Send {}
513unsafe impl<T> Sync for MovableRef<T> where T: Sync {}
514
515/// A backref record submitted alongside each item in a [`TypedMovableSection`].
516/// This points to a [`MovableRef`] that lives outside of the section.
517#[repr(C)]
518pub struct MovableBackref<T: 'static> {
519    slot: *const UnsafeCell<*const T>,
520}
521
522impl<T> MovableBackref<T> {
523    #[doc(hidden)]
524    pub const fn new(slot: *const UnsafeCell<*const T>) -> Self {
525        Self { slot }
526    }
527
528    /// Current value of the stable pointer slot as a pointer.
529    pub fn current_ptr(&self) -> *const T {
530        unsafe { ptr::read(UnsafeCell::raw_get(self.slot)) }
531    }
532
533    /// Update the stable pointer slot.
534    ///
535    /// # Safety
536    ///
537    /// Updating the slot requires exclusive access. See
538    /// [Exclusive access](crate#exclusive-access) for more information. Any live
539    /// `&T` or [`MovableRef`] dereference may alias the old or new target.
540    pub unsafe fn set_current_ptr(&self, ptr: *const T) {
541        unsafe {
542            ptr::write(UnsafeCell::raw_get(self.slot), ptr);
543        }
544    }
545}
546
547unsafe impl<T> Send for MovableBackref<T> where T: Send {}
548unsafe impl<T> Sync for MovableBackref<T> where T: Sync {}
549
550/// A typed link section that can be used to store any sized type. The
551/// underlying data is enumerable.
552#[repr(C)]
553pub struct TypedReferenceSection<T: 'static> {
554    name: &'static str,
555    bounds: Bounds,
556    _phantom: ::core::marker::PhantomData<T>,
557}
558
559impl<T: 'static> TypedReferenceSection<T> {
560    impl_section_new!(T);
561    impl_bounds_fns!(T);
562
563    /// The offset of the item in the section, if it is in the section.
564    #[inline]
565    pub fn offset_of(&self, item: &Ref<T>) -> Option<usize> {
566        let ptr = item.as_ptr();
567        if ptr < self.start_ptr() || ptr >= self.end_ptr() {
568            None
569        } else {
570            Some(unsafe { ptr.offset_from(self.start_ptr()) as usize })
571        }
572    }
573}
574
575impl_bounds_traits!(TypedReferenceSection<T>);
576
577/// A reference to a value in a link section. This allows platforms like WASM
578/// to reference the value, even though the final location is not known until
579/// after initialization.
580#[repr(transparent)]
581pub struct Ref<T: 'static> {
582    #[cfg(target_family = "wasm")]
583    ptr: ::core::cell::UnsafeCell<*const T>,
584    #[cfg(not(target_family = "wasm"))]
585    t: T,
586}
587
588impl<T> Ref<T> {
589    #[cfg(not(target_family = "wasm"))]
590    #[doc(hidden)]
591    pub const fn new(t: T) -> Self {
592        Self { t }
593    }
594
595    #[cfg(target_family = "wasm")]
596    #[doc(hidden)]
597    pub const fn new() -> Self {
598        Self {
599            ptr: ::core::cell::UnsafeCell::new(::core::ptr::null()),
600        }
601    }
602
603    /// # Safety
604    ///
605    /// For macro/runtime registration only. `ptr` must refer to the item's final
606    /// location in the WASM link section. Requires exclusive access. See
607    /// [Exclusive access](crate#exclusive-access) for more information.
608    #[cfg(target_family = "wasm")]
609    #[doc(hidden)]
610    pub unsafe fn set(&self, ptr: *const T) {
611        *self.ptr.get() = ptr;
612    }
613
614    /// Raw pointer to the value (WASM: cell; otherwise `&T` as `*const T`).
615    pub fn as_ptr(&self) -> *const T {
616        #[cfg(target_family = "wasm")]
617        {
618            unsafe { *self.ptr.get() }
619        }
620        #[cfg(not(target_family = "wasm"))]
621        {
622            &self.t as *const T
623        }
624    }
625}
626
627impl<T> ::core::ops::Deref for Ref<T> {
628    type Target = T;
629    fn deref(&self) -> &Self::Target {
630        #[cfg(target_family = "wasm")]
631        unsafe {
632            ::core::ptr::read(self.ptr.get())
633                .as_ref()
634                .expect("Ref not initialized")
635        }
636        #[cfg(not(target_family = "wasm"))]
637        &self.t
638    }
639}
640
641unsafe impl<T> Send for Ref<T> where T: Send {}
642unsafe impl<T> Sync for Ref<T> where T: Sync {}
link_section/sections.rs

link_section/
sections.rs
