Skip to main content

libc_rs/
thread.rs

1//! Pure-Rust thread lifecycle (`spawn_thread`, `ThreadHandle`), per-thread `errno`,
2//! and 64-key thread-local storage with exit destructors.
3
4use core::cell::UnsafeCell;
5use core::mem::{ManuallyDrop, size_of};
6use core::ptr::null_mut;
7use core::sync::atomic::{AtomicU8, AtomicU32, AtomicUsize, Ordering};
8
9use crate::errno::{EDEADLK, EINVAL, ENOMEM, Errno, SysResult};
10use crate::heap::{GLOBAL_MMAP_HEAP, c_free};
11use crate::mm::{
12    MAP_STACK, PAGE_SIZE, PROT_NONE, PROT_READ, PROT_WRITE, mmap_anonymous, mprotect, munmap,
13};
14
15pub const MAX_TLS_KEYS: usize = 64;
16pub const DEFAULT_STACK_BYTES: usize = 512 * 1024;
17pub const MIN_STACK_BYTES: usize = 64 * 1024;
18
19pub const CLONE_VM: u64 = 0x0000_0100;
20pub const CLONE_FS: u64 = 0x0000_0200;
21pub const CLONE_FILES: u64 = 0x0000_0400;
22pub const CLONE_SIGHAND: u64 = 0x0000_0800;
23pub const CLONE_THREAD: u64 = 0x0001_0000;
24pub const CLONE_SYSVSEM: u64 = 0x0004_0000;
25pub const CLONE_SETTLS: u64 = 0x0008_0000;
26pub const CLONE_PARENT_SETTID: u64 = 0x0010_0000;
27pub const CLONE_CHILD_CLEARTID: u64 = 0x0020_0000;
28pub const CLONE_CHILD_SETTID: u64 = 0x0100_0000;
29
30const JOIN_JOINABLE: u32 = 0;
31const JOIN_DETACHED: u32 = 1;
32#[cfg(all(target_arch = "aarch64", target_os = "none"))]
33const JOIN_EXITING: u32 = 2;
34
35const TLS_KEY_FREE: u8 = 0;
36const TLS_KEY_IN_USE: u8 = 1;
37
38#[repr(C, align(64))]
39struct ThreadControlBlock {
40    errno: UnsafeCell<i32>,
41    tls_values: UnsafeCell<[*mut u8; MAX_TLS_KEYS]>,
42    child_tid: AtomicU32,
43    join_state: AtomicU32,
44    stack_map_base: *mut u8,
45    stack_map_len: usize,
46    entry_fn: Option<extern "C" fn(*mut u8)>,
47    entry_arg: *mut u8,
48}
49
50impl ThreadControlBlock {
51    const fn new() -> Self {
52        Self {
53            errno: UnsafeCell::new(0),
54            tls_values: UnsafeCell::new([null_mut(); MAX_TLS_KEYS]),
55            child_tid: AtomicU32::new(0),
56            join_state: AtomicU32::new(JOIN_JOINABLE),
57            stack_map_base: null_mut(),
58            stack_map_len: 0,
59            entry_fn: None,
60            entry_arg: null_mut(),
61        }
62    }
63}
64
65#[repr(transparent)]
66struct SyncTcb(ThreadControlBlock);
67// SAFETY: `MAIN_TCB` is only accessed per-thread via `current_tcb()`.
68unsafe impl Sync for SyncTcb {}
69
70static MAIN_TCB: SyncTcb = SyncTcb(ThreadControlBlock::new());
71
72#[inline]
73fn current_tcb() -> &'static ThreadControlBlock {
74    #[cfg(all(target_arch = "aarch64", target_os = "none"))]
75    {
76        let mut tp: usize;
77        // SAFETY: `TPIDR_EL0` holds the thread's `ThreadControlBlock` pointer when set.
78        unsafe {
79            core::arch::asm!("mrs {}, tpidr_el0", out(reg) tp, options(nomem, nostack, preserves_flags));
80        }
81        if tp != 0 {
82            // SAFETY: Non-zero `TPIDR_EL0` points to a live `ThreadControlBlock`.
83            return unsafe { &*(tp as *const ThreadControlBlock) };
84        }
85    }
86    &MAIN_TCB.0
87}
88
89#[must_use]
90pub fn errno_cell() -> *mut i32 {
91    current_tcb().errno.get()
92}
93
94#[must_use]
95pub fn current_thread_id() -> u64 {
96    core::ptr::from_ref(current_tcb()) as usize as u64
97}
98
99static TLS_KEY_STATE: [AtomicU8; MAX_TLS_KEYS] =
100    [const { AtomicU8::new(TLS_KEY_FREE) }; MAX_TLS_KEYS];
101static TLS_DTORS: [AtomicUsize; MAX_TLS_KEYS] = [const { AtomicUsize::new(0) }; MAX_TLS_KEYS];
102
103#[must_use]
104pub fn tls_key_create(dtor: Option<unsafe extern "C" fn(*mut u8)>) -> Option<usize> {
105    let dtor_addr = dtor.map_or(0, |f| f as usize);
106    for key in 0..MAX_TLS_KEYS {
107        if TLS_KEY_STATE[key]
108            .compare_exchange(
109                TLS_KEY_FREE,
110                TLS_KEY_IN_USE,
111                Ordering::AcqRel,
112                Ordering::Acquire,
113            )
114            .is_ok()
115        {
116            TLS_DTORS[key].store(dtor_addr, Ordering::Release);
117            // SAFETY: Clearing the calling thread's own TLS slot for the newly allocated key.
118            unsafe { (*current_tcb().tls_values.get())[key] = null_mut() };
119            return Some(key);
120        }
121    }
122    None
123}
124
125pub fn tls_key_delete(key: usize) {
126    if key >= MAX_TLS_KEYS {
127        return;
128    }
129    TLS_DTORS[key].store(0, Ordering::Release);
130    // SAFETY: Clearing the calling thread's own TLS slot.
131    unsafe { (*current_tcb().tls_values.get())[key] = null_mut() };
132    TLS_KEY_STATE[key].store(TLS_KEY_FREE, Ordering::Release);
133}
134
135#[must_use]
136pub fn tls_get(key: usize) -> *mut u8 {
137    if key >= MAX_TLS_KEYS || TLS_KEY_STATE[key].load(Ordering::Acquire) != TLS_KEY_IN_USE {
138        return null_mut();
139    }
140    // SAFETY: Reading the calling thread's own TLS slot.
141    unsafe { (*current_tcb().tls_values.get())[key] }
142}
143
144pub fn tls_set(key: usize, val: *mut u8) -> bool {
145    if key >= MAX_TLS_KEYS || TLS_KEY_STATE[key].load(Ordering::Acquire) != TLS_KEY_IN_USE {
146        return false;
147    }
148    // SAFETY: Writing the calling thread's own TLS slot.
149    unsafe { (*current_tcb().tls_values.get())[key] = val };
150    true
151}
152
153pub fn run_current_thread_tls_destructors() {
154    let tcb = current_tcb();
155    for key in 0..MAX_TLS_KEYS {
156        // SAFETY: Accessing the calling thread's own TLS slot.
157        let val = unsafe { (*tcb.tls_values.get())[key] };
158        if !val.is_null() {
159            // SAFETY: Clearing the calling thread's own TLS slot before invoking the destructor.
160            unsafe { (*tcb.tls_values.get())[key] = null_mut() };
161            if TLS_KEY_STATE[key].load(Ordering::Acquire) == TLS_KEY_IN_USE {
162                let addr = TLS_DTORS[key].load(Ordering::Acquire);
163                if addr != 0 {
164                    // SAFETY: `addr` was stored from an `unsafe extern "C" fn(*mut u8)` in `tls_key_create`.
165                    let dtor: unsafe extern "C" fn(*mut u8) = unsafe { core::mem::transmute(addr) };
166                    // SAFETY: `dtor` accepts `val` for this TLS key.
167                    unsafe { dtor(val) };
168                }
169            }
170        }
171    }
172}
173
174pub struct ThreadHandle {
175    tcb: *mut ThreadControlBlock,
176}
177
178// SAFETY: `ThreadHandle` owns a thread lifecycle reference synchronized via atomics and futexes.
179unsafe impl Send for ThreadHandle {}
180// SAFETY: Shared references to `ThreadHandle` only inspect immutable pointer identity.
181unsafe impl Sync for ThreadHandle {}
182
183impl ThreadHandle {
184    #[must_use]
185    pub fn id(&self) -> u64 {
186        self.tcb as usize as u64
187    }
188
189    #[must_use]
190    pub fn is_current(&self) -> bool {
191        current_thread_id() == self.id()
192    }
193
194    pub fn join(self) -> SysResult<()> {
195        if self.is_current() {
196            let _ = ManuallyDrop::new(self);
197            return Err(Errno(EDEADLK));
198        }
199        let this = ManuallyDrop::new(self);
200        let tcb = this.tcb;
201        // SAFETY: `tcb` is valid for the lifetime of this joinable `ThreadHandle`.
202        unsafe {
203            loop {
204                let tid = (*tcb).child_tid.load(Ordering::Acquire);
205                if tid == 0 {
206                    break;
207                }
208                crate::sync::futex_wait(&(*tcb).child_tid, tid, None);
209            }
210            if !(*tcb).stack_map_base.is_null() && (*tcb).stack_map_len > 0 {
211                let _ = munmap((*tcb).stack_map_base, (*tcb).stack_map_len);
212            }
213            c_free(tcb.cast::<u8>());
214        }
215        Ok(())
216    }
217
218    pub fn detach(self) -> SysResult<()> {
219        let mut this = ManuallyDrop::new(self);
220        this.detach_inner();
221        Ok(())
222    }
223
224    fn detach_inner(&mut self) {
225        let tcb = self.tcb;
226        if tcb.is_null() {
227            return;
228        }
229        self.tcb = null_mut();
230        // SAFETY: `tcb` is valid until either the worker sees `JOIN_DETACHED` or we wait for `child_tid == 0`.
231        unsafe {
232            if (*tcb)
233                .join_state
234                .compare_exchange(
235                    JOIN_JOINABLE,
236                    JOIN_DETACHED,
237                    Ordering::AcqRel,
238                    Ordering::Acquire,
239                )
240                .is_err()
241            {
242                loop {
243                    let tid = (*tcb).child_tid.load(Ordering::Acquire);
244                    if tid == 0 {
245                        break;
246                    }
247                    crate::sync::futex_wait(&(*tcb).child_tid, tid, None);
248                }
249                if !(*tcb).stack_map_base.is_null() && (*tcb).stack_map_len > 0 {
250                    let _ = munmap((*tcb).stack_map_base, (*tcb).stack_map_len);
251                }
252                c_free(tcb.cast::<u8>());
253            }
254        }
255    }
256}
257
258impl Drop for ThreadHandle {
259    fn drop(&mut self) {
260        self.detach_inner();
261    }
262}
263
264#[cfg(all(target_arch = "aarch64", target_os = "none"))]
265extern "C" fn child_thread_start(tcb_ptr: *mut ThreadControlBlock) -> ! {
266    // SAFETY: `tcb_ptr` is the valid `ThreadControlBlock` allocated by `spawn_thread`.
267    unsafe {
268        if let Some(entry) = (*tcb_ptr).entry_fn {
269            entry((*tcb_ptr).entry_arg);
270        }
271    }
272    run_current_thread_tls_destructors();
273    // SAFETY: Synchronizing exit disposition with `ThreadHandle::join` / `ThreadHandle::detach`.
274    unsafe {
275        if (*tcb_ptr).join_state.swap(JOIN_EXITING, Ordering::AcqRel) == JOIN_JOINABLE {
276            crate::process::exit_thread(0);
277        }
278        let map_base = (*tcb_ptr).stack_map_base as usize;
279        let map_len = (*tcb_ptr).stack_map_len;
280        core::arch::asm!(
281            "msr tpidr_el0, xzr",
282            "mov x8, {munmap_nr}",
283            "svc #0",
284            "mov x0, #0",
285            "mov x8, {exit_nr}",
286            "svc #0",
287            "brk #0",
288            munmap_nr = const crate::raw::nr::MUNMAP,
289            exit_nr = const crate::raw::nr::EXIT,
290            in("x0") map_base,
291            in("x1") map_len,
292            options(noreturn, nostack),
293        );
294    }
295}
296
297#[cfg(all(target_arch = "aarch64", target_os = "none"))]
298#[inline]
299unsafe fn raw_clone_thread(
300    flags: u64,
301    child_sp: usize,
302    parent_tid: usize,
303    tls: usize,
304    child_tid: usize,
305) -> SysResult<usize> {
306    let ret: usize;
307    // SAFETY: Caller sets up a valid `child_sp` with `tcb` at `[child_sp]` and valid TID/TLS pointers.
308    unsafe {
309        core::arch::asm!(
310            "svc #0",
311            "cbnz x0, 2f",
312            "ldr x0, [sp]",
313            "add sp, sp, #16",
314            "mov x29, #0",
315            "mov x30, #0",
316            "bl {child_start}",
317            "brk #0",
318            "2:",
319            child_start = sym child_thread_start,
320            in("x8") crate::raw::nr::CLONE,
321            inlateout("x0") flags as usize => ret,
322            in("x1") child_sp,
323            in("x2") parent_tid,
324            in("x3") tls,
325            in("x4") child_tid,
326            options(nostack),
327        );
328    }
329    crate::raw::decode_raw(ret)
330}
331
332/// Spawns a new thread executing `entry(arg)` on a guarded stack.
333///
334/// # Safety
335///
336/// `entry` and `arg` must satisfy thread-safety invariants for the lifetime of the spawned thread.
337pub unsafe fn spawn_thread(
338    stack_bytes: usize,
339    entry: extern "C" fn(*mut u8),
340    arg: *mut u8,
341    detached: bool,
342) -> SysResult<Option<ThreadHandle>> {
343    let usable_stack = stack_bytes
344        .max(MIN_STACK_BYTES)
345        .checked_add(PAGE_SIZE - 1)
346        .map(|v| v & !(PAGE_SIZE - 1))
347        .ok_or(Errno(EINVAL))?;
348    let total_map = usable_stack.checked_add(PAGE_SIZE).ok_or(Errno(EINVAL))?;
349    let map_base = mmap_anonymous(total_map, PROT_READ | PROT_WRITE, MAP_STACK)?;
350    // SAFETY: `map_base` is page-aligned and `[map_base, map_base + PAGE_SIZE)` is within the mapping.
351    if let Err(e) = unsafe { mprotect(map_base, PAGE_SIZE, PROT_NONE) } {
352        // SAFETY: Unmapping the region just returned by `mmap_anonymous`.
353        let _ = unsafe { munmap(map_base, total_map) };
354        return Err(e);
355    }
356
357    // SAFETY: Allocating a 64-byte aligned `ThreadControlBlock` from `GLOBAL_MMAP_HEAP`.
358    let raw_tcb = unsafe { GLOBAL_MMAP_HEAP.alloc_c(size_of::<ThreadControlBlock>(), 64) };
359    if raw_tcb.is_null() {
360        // SAFETY: Unmapping the stack region on TCB allocation failure.
361        let _ = unsafe { munmap(map_base, total_map) };
362        return Err(Errno(ENOMEM));
363    }
364    let tcb = raw_tcb.cast::<ThreadControlBlock>();
365    // SAFETY: `tcb` points to a writable, aligned `ThreadControlBlock` allocation.
366    unsafe {
367        tcb.write(ThreadControlBlock::new());
368        (*tcb)
369            .child_tid
370            .store(if detached { 0 } else { u32::MAX }, Ordering::Relaxed);
371        (*tcb).join_state.store(
372            if detached {
373                JOIN_DETACHED
374            } else {
375                JOIN_JOINABLE
376            },
377            Ordering::Relaxed,
378        );
379        (*tcb).stack_map_base = map_base;
380        (*tcb).stack_map_len = total_map;
381        (*tcb).entry_fn = Some(entry);
382        (*tcb).entry_arg = arg;
383    }
384
385    let child_sp = (((map_base as usize) + total_map) & !15usize) - 16;
386    // SAFETY: `child_sp` lies at the top of the writable stack region above the guard page.
387    unsafe {
388        let frame = child_sp as *mut usize;
389        frame.write(tcb as usize);
390        frame.add(1).write(0);
391    }
392
393    let mut flags = CLONE_VM
394        | CLONE_FS
395        | CLONE_FILES
396        | CLONE_SIGHAND
397        | CLONE_THREAD
398        | CLONE_SYSVSEM
399        | CLONE_SETTLS;
400    let (parent_tid_ptr, child_tid_ptr) = if detached {
401        (0usize, 0usize)
402    } else {
403        flags |= CLONE_PARENT_SETTID | CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID;
404        // SAFETY: `tcb` is a valid aligned `ThreadControlBlock`.
405        let tid_addr = unsafe { core::ptr::addr_of!((*tcb).child_tid) as usize };
406        (tid_addr, tid_addr)
407    };
408
409    #[cfg(all(target_arch = "aarch64", target_os = "none"))]
410    // SAFETY: `child_sp`, `tcb`, and TID pointers are valid for `CLONE`.
411    let clone_res =
412        unsafe { raw_clone_thread(flags, child_sp, parent_tid_ptr, tcb as usize, child_tid_ptr) };
413    #[cfg(not(all(target_arch = "aarch64", target_os = "none")))]
414    // SAFETY: Invoking mocked/host `CLONE` syscall with validated stack and TCB pointers.
415    let clone_res = unsafe {
416        crate::raw::syscall5(
417            crate::raw::nr::CLONE,
418            flags as usize,
419            child_sp,
420            parent_tid_ptr,
421            tcb as usize,
422            child_tid_ptr,
423        )
424    };
425
426    match clone_res {
427        Ok(_) if detached => {
428            #[cfg(not(all(target_arch = "aarch64", target_os = "none")))]
429            // SAFETY: Under host mock syscalls no child thread runs, so reclaim simulated stack & TCB cleanly.
430            unsafe {
431                let _ = munmap(map_base, total_map);
432                c_free(tcb.cast::<u8>());
433            }
434            Ok(None)
435        }
436        Ok(_) => Ok(Some(ThreadHandle { tcb })),
437        Err(e) => {
438            // SAFETY: `CLONE` failed before any child thread started; reclaim stack and TCB.
439            unsafe {
440                let _ = munmap(map_base, total_map);
441                c_free(tcb.cast::<u8>());
442            }
443            Err(e)
444        }
445    }
446}
447
448#[cfg(test)]
449mod tests {
450    use super::*;
451    use crate::errno::EAGAIN;
452    use crate::raw::{nr, with_mock_syscall};
453    use core::sync::atomic::AtomicBool;
454
455    #[test]
456    fn errno_and_tls_destructors() {
457        assert_ne!(current_thread_id(), 0);
458        let ptr = errno_cell();
459        unsafe {
460            let prev = *ptr;
461            *ptr = 42;
462            assert_eq!(*errno_cell(), 42);
463            *ptr = prev;
464        }
465        static DTOR_CALLED: AtomicUsize = AtomicUsize::new(0);
466        unsafe extern "C" fn dtor(val: *mut u8) {
467            DTOR_CALLED.store(val as usize, Ordering::SeqCst);
468        }
469        let k = tls_key_create(Some(dtor)).expect("tls key");
470        assert!(tls_set(k, core::ptr::dangling_mut::<u8>()));
471        assert!(!tls_get(k).is_null());
472        run_current_thread_tls_destructors();
473        assert_ne!(DTOR_CALLED.load(Ordering::SeqCst), 0);
474        assert!(tls_get(k).is_null());
475        tls_key_delete(k);
476    }
477
478    #[test]
479    fn spawn_join_and_clone_error_cleanup() {
480        #[repr(align(4096))]
481        struct AlignedArena([u8; 256 * 1024]);
482
483        extern "C" fn dummy(_: *mut u8) {}
484        static mut ARENA: AlignedArena = AlignedArena([0; 256 * 1024]);
485        static OFFSET: AtomicUsize = AtomicUsize::new(0);
486        static FAIL_CLONE: AtomicBool = AtomicBool::new(false);
487        static MUNMAPS: AtomicUsize = AtomicUsize::new(0);
488
489        OFFSET.store(0, Ordering::SeqCst);
490        FAIL_CLONE.store(false, Ordering::SeqCst);
491        MUNMAPS.store(0, Ordering::SeqCst);
492
493        with_mock_syscall(
494            |sys_nr, args| match sys_nr {
495                nr::MMAP => {
496                    let len = (args[1] + 4095) & !4095;
497                    let off = OFFSET.fetch_add(len, Ordering::SeqCst);
498                    if off + len > 256 * 1024 {
499                        return Err(Errno(ENOMEM));
500                    }
501                    // SAFETY: `off + len` lies within `ARENA` while `MOCK_LOCK` serializes mock tests.
502                    let ptr = unsafe { core::ptr::addr_of_mut!(ARENA.0).cast::<u8>().add(off) };
503                    Ok(ptr as usize)
504                }
505                nr::MPROTECT => Ok(0),
506                nr::MUNMAP => {
507                    MUNMAPS.fetch_add(1, Ordering::SeqCst);
508                    Ok(0)
509                }
510                nr::CLONE if FAIL_CLONE.load(Ordering::SeqCst) => Err(Errno(EAGAIN)),
511                nr::CLONE => {
512                    let tid = args[4] as *const AtomicU32;
513                    if !tid.is_null() {
514                        unsafe { (*tid).store(0, Ordering::Release) };
515                    }
516                    Ok(777)
517                }
518                _ => Err(Errno(crate::errno::ENOSYS)),
519            },
520            || {
521                let h = unsafe { spawn_thread(MIN_STACK_BYTES, dummy, null_mut(), false) }
522                    .expect("spawn")
523                    .expect("handle");
524                let before = MUNMAPS.load(Ordering::SeqCst);
525                h.join().expect("join");
526                assert!(MUNMAPS.load(Ordering::SeqCst) > before);
527
528                FAIL_CLONE.store(true, Ordering::SeqCst);
529                let before_fail = MUNMAPS.load(Ordering::SeqCst);
530                let err = unsafe { spawn_thread(MIN_STACK_BYTES, dummy, null_mut(), false) };
531                assert_eq!(err.err(), Some(Errno(EAGAIN)));
532                assert!(MUNMAPS.load(Ordering::SeqCst) > before_fail);
533            },
534        );
535    }
536}