Skip to main content

libc_rs/
thread.rs

1//! Pure-Rust thread lifecycle (`spawn_thread`, `ThreadHandle`), per-thread `errno`,
2//! and 64-key thread-local storage with exit destructors.
3
4use core::cell::UnsafeCell;
5use core::mem::{ManuallyDrop, size_of};
6use core::ptr::null_mut;
7use core::sync::atomic::{AtomicU8, AtomicU32, AtomicUsize, Ordering};
8
9use crate::errno::{EDEADLK, EINVAL, ENOMEM, Errno, SysResult};
10use crate::heap::{GLOBAL_MMAP_HEAP, c_free};
11use crate::mm::{
12    MAP_STACK, PAGE_SIZE, PROT_NONE, PROT_READ, PROT_WRITE, mmap_anonymous, mprotect, munmap,
13};
14
15pub const MAX_TLS_KEYS: usize = 64;
16pub const DEFAULT_STACK_BYTES: usize = 512 * 1024;
17pub const MIN_STACK_BYTES: usize = 64 * 1024;
18
19pub const CLONE_VM: u64 = 0x0000_0100;
20pub const CLONE_FS: u64 = 0x0000_0200;
21pub const CLONE_FILES: u64 = 0x0000_0400;
22pub const CLONE_SIGHAND: u64 = 0x0000_0800;
23pub const CLONE_THREAD: u64 = 0x0001_0000;
24pub const CLONE_SYSVSEM: u64 = 0x0004_0000;
25pub const CLONE_SETTLS: u64 = 0x0008_0000;
26pub const CLONE_PARENT_SETTID: u64 = 0x0010_0000;
27pub const CLONE_CHILD_CLEARTID: u64 = 0x0020_0000;
28pub const CLONE_CHILD_SETTID: u64 = 0x0100_0000;
29
30const JOIN_JOINABLE: u32 = 0;
31const JOIN_DETACHED: u32 = 1;
32#[cfg(all(target_arch = "aarch64", target_os = "none"))]
33const JOIN_EXITING: u32 = 2;
34
35const _: () = assert!(MAX_TLS_KEYS == 64);
36
37const TLS_KEY_FREE: u8 = 0;
38const TLS_KEY_IN_USE: u8 = 1;
39
40#[repr(C, align(64))]
41pub(crate) struct ThreadControlBlock {
42    errno: UnsafeCell<i32>,
43    pub(crate) mag_len: UnsafeCell<[u8; 8]>,
44    pub(crate) mag_ptrs: UnsafeCell<[[*mut u8; 8]; 8]>,
45    tls_values: UnsafeCell<[*mut u8; 64]>,
46    child_tid: AtomicU32,
47    join_state: AtomicU32,
48    stack_map_base: *mut u8,
49    stack_map_len: usize,
50    entry_fn: Option<extern "C" fn(*mut u8)>,
51    entry_arg: *mut u8,
52}
53
54impl ThreadControlBlock {
55    const fn new() -> Self {
56        Self {
57            errno: UnsafeCell::new(0),
58            mag_len: UnsafeCell::new([0; 8]),
59            mag_ptrs: UnsafeCell::new([[null_mut(); 8]; 8]),
60            tls_values: UnsafeCell::new([null_mut(); 64]),
61            child_tid: AtomicU32::new(0),
62            join_state: AtomicU32::new(JOIN_JOINABLE),
63            stack_map_base: null_mut(),
64            stack_map_len: 0,
65            entry_fn: None,
66            entry_arg: null_mut(),
67        }
68    }
69}
70
71#[repr(transparent)]
72struct SyncTcb(ThreadControlBlock);
73// SAFETY: `MAIN_TCB` is only accessed per-thread via `current_tcb_ptr()`.
74unsafe impl Sync for SyncTcb {}
75
76static MAIN_TCB: SyncTcb = SyncTcb(ThreadControlBlock::new());
77
78#[inline]
79pub(crate) fn current_tcb_ptr() -> *mut ThreadControlBlock {
80    let main_addr = core::ptr::addr_of!(MAIN_TCB.0) as usize;
81    let tp = crate::raw::read_thread_pointer();
82    let addr = if tp == 0 { main_addr } else { tp };
83    addr as *mut ThreadControlBlock
84}
85
86#[must_use]
87#[inline]
88pub fn errno_cell() -> *mut i32 {
89    let tcb = current_tcb_ptr();
90    // SAFETY: `tcb` points to the calling thread's live `ThreadControlBlock`.
91    unsafe { (*tcb).errno.get() }
92}
93
94#[must_use]
95#[inline]
96pub fn current_thread_id() -> u64 {
97    current_tcb_ptr() as usize as u64
98}
99
100static TLS_KEY_STATE: [AtomicU8; 64] = [const { AtomicU8::new(TLS_KEY_FREE) }; 64];
101static TLS_DTORS: [AtomicUsize; 64] = [const { AtomicUsize::new(0) }; 64];
102
103#[must_use]
104pub fn tls_key_create(dtor: Option<unsafe extern "C" fn(*mut u8)>) -> Option<usize> {
105    let dtor_addr = dtor.map_or(0, |f| f as usize);
106    let tcb = current_tcb_ptr();
107    for key in 0..MAX_TLS_KEYS {
108        if TLS_KEY_STATE[key]
109            .compare_exchange(
110                TLS_KEY_FREE,
111                TLS_KEY_IN_USE,
112                Ordering::AcqRel,
113                Ordering::Acquire,
114            )
115            .is_ok()
116        {
117            TLS_DTORS[key].store(dtor_addr, Ordering::Release);
118            // SAFETY: Clearing the calling thread's own TLS slot for the newly allocated key.
119            unsafe { (*(*tcb).tls_values.get())[key] = null_mut() };
120            return Some(key);
121        }
122    }
123    None
124}
125
126pub fn tls_key_delete(key: usize) {
127    if key >= MAX_TLS_KEYS {
128        return;
129    }
130    let tcb = current_tcb_ptr();
131    TLS_DTORS[key].store(0, Ordering::Release);
132    // SAFETY: Clearing the calling thread's own TLS slot.
133    unsafe { (*(*tcb).tls_values.get())[key] = null_mut() };
134    TLS_KEY_STATE[key].store(TLS_KEY_FREE, Ordering::Release);
135}
136
137#[must_use]
138#[inline]
139pub fn tls_get(key: usize) -> *mut u8 {
140    if key >= MAX_TLS_KEYS || TLS_KEY_STATE[key].load(Ordering::Acquire) != TLS_KEY_IN_USE {
141        return null_mut();
142    }
143    let tcb = current_tcb_ptr();
144    // SAFETY: Reading the calling thread's own TLS slot.
145    unsafe { (*(*tcb).tls_values.get())[key] }
146}
147
148#[inline]
149pub fn tls_set(key: usize, val: *mut u8) -> bool {
150    if key >= MAX_TLS_KEYS || TLS_KEY_STATE[key].load(Ordering::Acquire) != TLS_KEY_IN_USE {
151        return false;
152    }
153    let tcb = current_tcb_ptr();
154    // SAFETY: Writing the calling thread's own TLS slot.
155    unsafe { (*(*tcb).tls_values.get())[key] = val };
156    true
157}
158
159pub fn run_current_thread_tls_destructors() {
160    let tcb = current_tcb_ptr();
161    for key in 0..MAX_TLS_KEYS {
162        // SAFETY: Accessing the calling thread's own TLS slot.
163        let val = unsafe { (*(*tcb).tls_values.get())[key] };
164        if !val.is_null() {
165            // SAFETY: Clearing the calling thread's own TLS slot before invoking the destructor.
166            unsafe { (*(*tcb).tls_values.get())[key] = null_mut() };
167            if TLS_KEY_STATE[key].load(Ordering::Acquire) == TLS_KEY_IN_USE {
168                let addr = TLS_DTORS[key].load(Ordering::Acquire);
169                if addr != 0 {
170                    // SAFETY: `addr` was stored from an `unsafe extern "C" fn(*mut u8)` in `tls_key_create`.
171                    let dtor: unsafe extern "C" fn(*mut u8) = unsafe { core::mem::transmute(addr) };
172                    // SAFETY: `dtor` accepts `val` for this TLS key.
173                    unsafe { dtor(val) };
174                }
175            }
176        }
177    }
178    #[cfg(feature = "alloc")]
179    crate::heap::flush_current_thread_magazine();
180}
181
182pub struct ThreadHandle {
183    tcb: *mut ThreadControlBlock,
184}
185
186// SAFETY: `ThreadHandle` owns a thread lifecycle reference synchronized via atomics and futexes.
187unsafe impl Send for ThreadHandle {}
188// SAFETY: Shared references to `ThreadHandle` only inspect immutable pointer identity.
189unsafe impl Sync for ThreadHandle {}
190
191impl ThreadHandle {
192    #[must_use]
193    pub fn id(&self) -> u64 {
194        self.tcb as usize as u64
195    }
196
197    #[must_use]
198    pub fn is_current(&self) -> bool {
199        current_thread_id() == self.id()
200    }
201
202    pub fn join(self) -> SysResult<()> {
203        if self.is_current() {
204            let _ = ManuallyDrop::new(self);
205            return Err(Errno(EDEADLK));
206        }
207        let this = ManuallyDrop::new(self);
208        let tcb = this.tcb;
209        // SAFETY: `tcb` is valid for the lifetime of this joinable `ThreadHandle`.
210        unsafe {
211            loop {
212                let tid = (*tcb).child_tid.load(Ordering::Acquire);
213                if tid == 0 {
214                    break;
215                }
216                crate::sync::futex_wait(&(*tcb).child_tid, tid, None);
217            }
218            if !(*tcb).stack_map_base.is_null() && (*tcb).stack_map_len > 0 {
219                let _ = munmap((*tcb).stack_map_base, (*tcb).stack_map_len);
220            }
221            c_free(tcb.cast::<u8>());
222        }
223        Ok(())
224    }
225
226    pub fn detach(self) -> SysResult<()> {
227        let mut this = ManuallyDrop::new(self);
228        this.detach_inner();
229        Ok(())
230    }
231
232    fn detach_inner(&mut self) {
233        let tcb = self.tcb;
234        if tcb.is_null() {
235            return;
236        }
237        self.tcb = null_mut();
238        // SAFETY: `tcb` is valid until either the worker sees `JOIN_DETACHED` or we wait for `child_tid == 0`.
239        unsafe {
240            if (*tcb)
241                .join_state
242                .compare_exchange(
243                    JOIN_JOINABLE,
244                    JOIN_DETACHED,
245                    Ordering::AcqRel,
246                    Ordering::Acquire,
247                )
248                .is_err()
249            {
250                loop {
251                    let tid = (*tcb).child_tid.load(Ordering::Acquire);
252                    if tid == 0 {
253                        break;
254                    }
255                    crate::sync::futex_wait(&(*tcb).child_tid, tid, None);
256                }
257                if !(*tcb).stack_map_base.is_null() && (*tcb).stack_map_len > 0 {
258                    let _ = munmap((*tcb).stack_map_base, (*tcb).stack_map_len);
259                }
260                c_free(tcb.cast::<u8>());
261            }
262        }
263    }
264}
265
266impl Drop for ThreadHandle {
267    fn drop(&mut self) {
268        self.detach_inner();
269    }
270}
271
272#[cfg(all(target_arch = "aarch64", target_os = "none"))]
273unsafe extern "C" fn child_thread_start() -> ! {
274    let tcb_ptr = current_tcb_ptr();
275    // SAFETY: `CLONE_SETTLS` installed this child's valid `ThreadControlBlock` in `TPIDR_EL0`.
276    unsafe {
277        if let Some(entry) = (*tcb_ptr).entry_fn {
278            entry((*tcb_ptr).entry_arg);
279        }
280    }
281    run_current_thread_tls_destructors();
282    // SAFETY: Synchronizing exit disposition with `ThreadHandle::join` / `ThreadHandle::detach`.
283    unsafe {
284        if (*tcb_ptr).join_state.swap(JOIN_EXITING, Ordering::AcqRel) == JOIN_JOINABLE {
285            crate::process::exit_thread(0);
286        }
287        let map_base = (*tcb_ptr).stack_map_base as usize;
288        let map_len = (*tcb_ptr).stack_map_len;
289        crate::raw::unmap_stack_and_exit(map_base, map_len)
290    }
291}
292
293/// Spawns a new thread executing `entry(arg)` on a guarded stack.
294///
295/// # Safety
296///
297/// `entry` and `arg` must satisfy thread-safety invariants for the lifetime of the spawned thread.
298pub unsafe fn spawn_thread(
299    stack_bytes: usize,
300    entry: extern "C" fn(*mut u8),
301    arg: *mut u8,
302    detached: bool,
303) -> SysResult<Option<ThreadHandle>> {
304    let usable_stack = stack_bytes
305        .max(MIN_STACK_BYTES)
306        .checked_add(PAGE_SIZE - 1)
307        .map(|v| v & !(PAGE_SIZE - 1))
308        .ok_or(Errno(EINVAL))?;
309    let total_map = usable_stack.checked_add(PAGE_SIZE).ok_or(Errno(EINVAL))?;
310    let map_base = mmap_anonymous(total_map, PROT_READ | PROT_WRITE, MAP_STACK)?;
311    // SAFETY: `map_base` is page-aligned and `[map_base, map_base + PAGE_SIZE)` is within the mapping.
312    if let Err(e) = unsafe { mprotect(map_base, PAGE_SIZE, PROT_NONE) } {
313        // SAFETY: Unmapping the region just returned by `mmap_anonymous`.
314        let _ = unsafe { munmap(map_base, total_map) };
315        return Err(e);
316    }
317
318    // SAFETY: Allocating a 64-byte aligned `ThreadControlBlock` from `GLOBAL_MMAP_HEAP`.
319    let raw_tcb = unsafe { GLOBAL_MMAP_HEAP.alloc_c(size_of::<ThreadControlBlock>(), 64) };
320    if raw_tcb.is_null() {
321        // SAFETY: Unmapping the stack region on TCB allocation failure.
322        let _ = unsafe { munmap(map_base, total_map) };
323        return Err(Errno(ENOMEM));
324    }
325    let tcb = raw_tcb.cast::<ThreadControlBlock>();
326    // SAFETY: `tcb` points to a writable, aligned `ThreadControlBlock` allocation.
327    unsafe {
328        tcb.write(ThreadControlBlock::new());
329        (*tcb)
330            .child_tid
331            .store(if detached { 0 } else { u32::MAX }, Ordering::Relaxed);
332        (*tcb).join_state.store(
333            if detached {
334                JOIN_DETACHED
335            } else {
336                JOIN_JOINABLE
337            },
338            Ordering::Relaxed,
339        );
340        (*tcb).stack_map_base = map_base;
341        (*tcb).stack_map_len = total_map;
342        (*tcb).entry_fn = Some(entry);
343        (*tcb).entry_arg = arg;
344    }
345
346    let child_sp = (((map_base as usize) + total_map) & !15usize) - 64;
347    // SAFETY: `child_sp` lies near the top of the writable stack region above the guard page with 64B headroom.
348    unsafe {
349        let frame = child_sp as *mut usize;
350        frame.write(0);
351        frame.add(1).write(0);
352    }
353
354    let mut flags = CLONE_VM
355        | CLONE_FS
356        | CLONE_FILES
357        | CLONE_SIGHAND
358        | CLONE_THREAD
359        | CLONE_SYSVSEM
360        | CLONE_SETTLS;
361    let (parent_tid_ptr, child_tid_ptr) = if detached {
362        (0usize, 0usize)
363    } else {
364        flags |= CLONE_PARENT_SETTID | CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID;
365        // SAFETY: `tcb` is a valid aligned `ThreadControlBlock`.
366        let tid_addr = unsafe { core::ptr::addr_of!((*tcb).child_tid) as usize };
367        (tid_addr, tid_addr)
368    };
369
370    #[cfg(all(target_arch = "aarch64", target_os = "none"))]
371    // SAFETY: `child_sp`, `tcb`, and TID pointers are valid for `CLONE`.
372    let clone_res = unsafe {
373        crate::raw::clone_thread(
374            flags,
375            child_sp,
376            parent_tid_ptr,
377            tcb as usize,
378            child_tid_ptr,
379            child_thread_start,
380        )
381    };
382    #[cfg(not(all(target_arch = "aarch64", target_os = "none")))]
383    // SAFETY: Invoking mocked/host `CLONE` syscall with validated stack and TCB pointers.
384    let clone_res = unsafe {
385        crate::raw::syscall5(
386            crate::raw::nr::CLONE,
387            flags as usize,
388            child_sp,
389            parent_tid_ptr,
390            tcb as usize,
391            child_tid_ptr,
392        )
393    };
394
395    match clone_res {
396        Ok(_) if detached => {
397            #[cfg(not(all(target_arch = "aarch64", target_os = "none")))]
398            // SAFETY: Under host mock syscalls no child thread runs, so reclaim simulated stack & TCB cleanly.
399            unsafe {
400                let _ = munmap(map_base, total_map);
401                c_free(tcb.cast::<u8>());
402            }
403            Ok(None)
404        }
405        Ok(_) => Ok(Some(ThreadHandle { tcb })),
406        Err(e) => {
407            // SAFETY: `CLONE` failed before any child thread started; reclaim stack and TCB.
408            unsafe {
409                let _ = munmap(map_base, total_map);
410                c_free(tcb.cast::<u8>());
411            }
412            Err(e)
413        }
414    }
415}
416
417#[cfg(test)]
418mod tests {
419    use super::*;
420    use crate::errno::EAGAIN;
421    use crate::raw::{nr, with_mock_syscall};
422    use core::sync::atomic::AtomicBool;
423
424    #[test]
425    fn errno_and_tls_destructors() {
426        assert_ne!(current_thread_id(), 0);
427        let ptr = errno_cell();
428        unsafe {
429            let prev = *ptr;
430            *ptr = 42;
431            assert_eq!(*errno_cell(), 42);
432            *ptr = prev;
433        }
434        static DTOR_CALLED: AtomicUsize = AtomicUsize::new(0);
435        unsafe extern "C" fn dtor(val: *mut u8) {
436            DTOR_CALLED.store(val as usize, Ordering::SeqCst);
437        }
438        let k = tls_key_create(Some(dtor)).expect("tls key");
439        assert!(tls_set(k, core::ptr::dangling_mut::<u8>()));
440        assert!(!tls_get(k).is_null());
441        run_current_thread_tls_destructors();
442        assert_ne!(DTOR_CALLED.load(Ordering::SeqCst), 0);
443        assert!(tls_get(k).is_null());
444        tls_key_delete(k);
445    }
446
447    #[test]
448    fn spawn_join_and_clone_error_cleanup() {
449        #[repr(align(4096))]
450        struct AlignedArena([u8; 256 * 1024]);
451
452        extern "C" fn dummy(_: *mut u8) {}
453        static mut ARENA: AlignedArena = AlignedArena([0; 256 * 1024]);
454        static OFFSET: AtomicUsize = AtomicUsize::new(0);
455        static FAIL_CLONE: AtomicBool = AtomicBool::new(false);
456        static MUNMAPS: AtomicUsize = AtomicUsize::new(0);
457
458        OFFSET.store(0, Ordering::SeqCst);
459        FAIL_CLONE.store(false, Ordering::SeqCst);
460        MUNMAPS.store(0, Ordering::SeqCst);
461
462        with_mock_syscall(
463            |sys_nr, args| match sys_nr {
464                nr::MMAP => {
465                    let len = (args[1] + 4095) & !4095;
466                    let off = OFFSET.fetch_add(len, Ordering::SeqCst);
467                    if off + len > 256 * 1024 {
468                        return Err(Errno(ENOMEM));
469                    }
470                    // SAFETY: `off + len` lies within `ARENA` while `MOCK_LOCK` serializes mock tests.
471                    let ptr = unsafe { core::ptr::addr_of_mut!(ARENA.0).cast::<u8>().add(off) };
472                    Ok(ptr as usize)
473                }
474                nr::MPROTECT => Ok(0),
475                nr::MUNMAP => {
476                    MUNMAPS.fetch_add(1, Ordering::SeqCst);
477                    Ok(0)
478                }
479                nr::CLONE if FAIL_CLONE.load(Ordering::SeqCst) => Err(Errno(EAGAIN)),
480                nr::CLONE => {
481                    let tid = args[4] as *const AtomicU32;
482                    if !tid.is_null() {
483                        unsafe { (*tid).store(0, Ordering::Release) };
484                    }
485                    Ok(777)
486                }
487                _ => Err(Errno(crate::errno::ENOSYS)),
488            },
489            || {
490                let h = unsafe { spawn_thread(MIN_STACK_BYTES, dummy, null_mut(), false) }
491                    .expect("spawn")
492                    .expect("handle");
493                let before = MUNMAPS.load(Ordering::SeqCst);
494                h.join().expect("join");
495                assert!(MUNMAPS.load(Ordering::SeqCst) > before);
496
497                FAIL_CLONE.store(true, Ordering::SeqCst);
498                let before_fail = MUNMAPS.load(Ordering::SeqCst);
499                let err = unsafe { spawn_thread(MIN_STACK_BYTES, dummy, null_mut(), false) };
500                assert_eq!(err.err(), Some(Errno(EAGAIN)));
501                assert!(MUNMAPS.load(Ordering::SeqCst) > before_fail);
502            },
503        );
504    }
505}