Skip to main content

libc_rs/
thread.rs

1//! Pure-Rust thread lifecycle (`spawn_thread`, `ThreadHandle`), per-thread `errno`,
2//! and 64-key thread-local storage with exit destructors.
3
4use core::cell::UnsafeCell;
5use core::mem::{ManuallyDrop, size_of};
6use core::ptr::null_mut;
7use core::sync::atomic::{AtomicU8, AtomicU32, AtomicUsize, Ordering};
8
9use crate::errno::{EDEADLK, EINVAL, ENOMEM, Errno, SysResult};
10use crate::heap::{GLOBAL_MMAP_HEAP, c_free};
11use crate::mm::{
12    MAP_STACK, PAGE_SIZE, PROT_NONE, PROT_READ, PROT_WRITE, mmap_anonymous, mprotect, munmap,
13};
14
15pub const MAX_TLS_KEYS: usize = 64;
16pub const DEFAULT_STACK_BYTES: usize = 512 * 1024;
17pub const MIN_STACK_BYTES: usize = 64 * 1024;
18
19pub const CLONE_VM: u64 = 0x0000_0100;
20pub const CLONE_FS: u64 = 0x0000_0200;
21pub const CLONE_FILES: u64 = 0x0000_0400;
22pub const CLONE_SIGHAND: u64 = 0x0000_0800;
23pub const CLONE_THREAD: u64 = 0x0001_0000;
24pub const CLONE_SYSVSEM: u64 = 0x0004_0000;
25pub const CLONE_SETTLS: u64 = 0x0008_0000;
26pub const CLONE_PARENT_SETTID: u64 = 0x0010_0000;
27pub const CLONE_CHILD_CLEARTID: u64 = 0x0020_0000;
28pub const CLONE_CHILD_SETTID: u64 = 0x0100_0000;
29
30const JOIN_JOINABLE: u32 = 0;
31const JOIN_DETACHED: u32 = 1;
32#[cfg(all(target_arch = "aarch64", target_os = "none"))]
33const JOIN_EXITING: u32 = 2;
34
35const _: () = assert!(MAX_TLS_KEYS == 64);
36
37const TLS_KEY_FREE: u8 = 0;
38const TLS_KEY_IN_USE: u8 = 1;
39
40#[repr(C, align(64))]
41struct ThreadControlBlock {
42    errno: UnsafeCell<i32>,
43    tls_values: UnsafeCell<[*mut u8; 64]>,
44    child_tid: AtomicU32,
45    join_state: AtomicU32,
46    stack_map_base: *mut u8,
47    stack_map_len: usize,
48    entry_fn: Option<extern "C" fn(*mut u8)>,
49    entry_arg: *mut u8,
50}
51
52impl ThreadControlBlock {
53    const fn new() -> Self {
54        Self {
55            errno: UnsafeCell::new(0),
56            tls_values: UnsafeCell::new([null_mut(); 64]),
57            child_tid: AtomicU32::new(0),
58            join_state: AtomicU32::new(JOIN_JOINABLE),
59            stack_map_base: null_mut(),
60            stack_map_len: 0,
61            entry_fn: None,
62            entry_arg: null_mut(),
63        }
64    }
65}
66
67#[repr(transparent)]
68struct SyncTcb(ThreadControlBlock);
69// SAFETY: `MAIN_TCB` is only accessed per-thread via `current_tcb_ptr()`.
70unsafe impl Sync for SyncTcb {}
71
72static MAIN_TCB: SyncTcb = SyncTcb(ThreadControlBlock::new());
73
74#[inline]
75fn current_tcb_ptr() -> *mut ThreadControlBlock {
76    let main_addr = core::ptr::addr_of!(MAIN_TCB.0) as usize;
77    let tp = crate::raw::read_thread_pointer();
78    let addr = if tp == 0 { main_addr } else { tp };
79    addr as *mut ThreadControlBlock
80}
81
82#[must_use]
83pub fn errno_cell() -> *mut i32 {
84    let tcb = current_tcb_ptr();
85    // SAFETY: `tcb` points to the calling thread's live `ThreadControlBlock`.
86    unsafe { (*tcb).errno.get() }
87}
88
89#[must_use]
90pub fn current_thread_id() -> u64 {
91    current_tcb_ptr() as usize as u64
92}
93
94static TLS_KEY_STATE: [AtomicU8; 64] = [const { AtomicU8::new(TLS_KEY_FREE) }; 64];
95static TLS_DTORS: [AtomicUsize; 64] = [const { AtomicUsize::new(0) }; 64];
96
97#[must_use]
98pub fn tls_key_create(dtor: Option<unsafe extern "C" fn(*mut u8)>) -> Option<usize> {
99    let dtor_addr = dtor.map_or(0, |f| f as usize);
100    let tcb = current_tcb_ptr();
101    for key in 0..MAX_TLS_KEYS {
102        if TLS_KEY_STATE[key]
103            .compare_exchange(
104                TLS_KEY_FREE,
105                TLS_KEY_IN_USE,
106                Ordering::AcqRel,
107                Ordering::Acquire,
108            )
109            .is_ok()
110        {
111            TLS_DTORS[key].store(dtor_addr, Ordering::Release);
112            // SAFETY: Clearing the calling thread's own TLS slot for the newly allocated key.
113            unsafe { (*(*tcb).tls_values.get())[key] = null_mut() };
114            return Some(key);
115        }
116    }
117    None
118}
119
120pub fn tls_key_delete(key: usize) {
121    if key >= MAX_TLS_KEYS {
122        return;
123    }
124    let tcb = current_tcb_ptr();
125    TLS_DTORS[key].store(0, Ordering::Release);
126    // SAFETY: Clearing the calling thread's own TLS slot.
127    unsafe { (*(*tcb).tls_values.get())[key] = null_mut() };
128    TLS_KEY_STATE[key].store(TLS_KEY_FREE, Ordering::Release);
129}
130
131#[must_use]
132pub fn tls_get(key: usize) -> *mut u8 {
133    if key >= MAX_TLS_KEYS || TLS_KEY_STATE[key].load(Ordering::Acquire) != TLS_KEY_IN_USE {
134        return null_mut();
135    }
136    let tcb = current_tcb_ptr();
137    // SAFETY: Reading the calling thread's own TLS slot.
138    unsafe { (*(*tcb).tls_values.get())[key] }
139}
140
141pub fn tls_set(key: usize, val: *mut u8) -> bool {
142    if key >= MAX_TLS_KEYS || TLS_KEY_STATE[key].load(Ordering::Acquire) != TLS_KEY_IN_USE {
143        return false;
144    }
145    let tcb = current_tcb_ptr();
146    // SAFETY: Writing the calling thread's own TLS slot.
147    unsafe { (*(*tcb).tls_values.get())[key] = val };
148    true
149}
150
151pub fn run_current_thread_tls_destructors() {
152    let tcb = current_tcb_ptr();
153    for key in 0..MAX_TLS_KEYS {
154        // SAFETY: Accessing the calling thread's own TLS slot.
155        let val = unsafe { (*(*tcb).tls_values.get())[key] };
156        if !val.is_null() {
157            // SAFETY: Clearing the calling thread's own TLS slot before invoking the destructor.
158            unsafe { (*(*tcb).tls_values.get())[key] = null_mut() };
159            if TLS_KEY_STATE[key].load(Ordering::Acquire) == TLS_KEY_IN_USE {
160                let addr = TLS_DTORS[key].load(Ordering::Acquire);
161                if addr != 0 {
162                    // SAFETY: `addr` was stored from an `unsafe extern "C" fn(*mut u8)` in `tls_key_create`.
163                    let dtor: unsafe extern "C" fn(*mut u8) = unsafe { core::mem::transmute(addr) };
164                    // SAFETY: `dtor` accepts `val` for this TLS key.
165                    unsafe { dtor(val) };
166                }
167            }
168        }
169    }
170}
171
172pub struct ThreadHandle {
173    tcb: *mut ThreadControlBlock,
174}
175
176// SAFETY: `ThreadHandle` owns a thread lifecycle reference synchronized via atomics and futexes.
177unsafe impl Send for ThreadHandle {}
178// SAFETY: Shared references to `ThreadHandle` only inspect immutable pointer identity.
179unsafe impl Sync for ThreadHandle {}
180
181impl ThreadHandle {
182    #[must_use]
183    pub fn id(&self) -> u64 {
184        self.tcb as usize as u64
185    }
186
187    #[must_use]
188    pub fn is_current(&self) -> bool {
189        current_thread_id() == self.id()
190    }
191
192    pub fn join(self) -> SysResult<()> {
193        if self.is_current() {
194            let _ = ManuallyDrop::new(self);
195            return Err(Errno(EDEADLK));
196        }
197        let this = ManuallyDrop::new(self);
198        let tcb = this.tcb;
199        // SAFETY: `tcb` is valid for the lifetime of this joinable `ThreadHandle`.
200        unsafe {
201            loop {
202                let tid = (*tcb).child_tid.load(Ordering::Acquire);
203                if tid == 0 {
204                    break;
205                }
206                crate::sync::futex_wait(&(*tcb).child_tid, tid, None);
207            }
208            if !(*tcb).stack_map_base.is_null() && (*tcb).stack_map_len > 0 {
209                let _ = munmap((*tcb).stack_map_base, (*tcb).stack_map_len);
210            }
211            c_free(tcb.cast::<u8>());
212        }
213        Ok(())
214    }
215
216    pub fn detach(self) -> SysResult<()> {
217        let mut this = ManuallyDrop::new(self);
218        this.detach_inner();
219        Ok(())
220    }
221
222    fn detach_inner(&mut self) {
223        let tcb = self.tcb;
224        if tcb.is_null() {
225            return;
226        }
227        self.tcb = null_mut();
228        // SAFETY: `tcb` is valid until either the worker sees `JOIN_DETACHED` or we wait for `child_tid == 0`.
229        unsafe {
230            if (*tcb)
231                .join_state
232                .compare_exchange(
233                    JOIN_JOINABLE,
234                    JOIN_DETACHED,
235                    Ordering::AcqRel,
236                    Ordering::Acquire,
237                )
238                .is_err()
239            {
240                loop {
241                    let tid = (*tcb).child_tid.load(Ordering::Acquire);
242                    if tid == 0 {
243                        break;
244                    }
245                    crate::sync::futex_wait(&(*tcb).child_tid, tid, None);
246                }
247                if !(*tcb).stack_map_base.is_null() && (*tcb).stack_map_len > 0 {
248                    let _ = munmap((*tcb).stack_map_base, (*tcb).stack_map_len);
249                }
250                c_free(tcb.cast::<u8>());
251            }
252        }
253    }
254}
255
256impl Drop for ThreadHandle {
257    fn drop(&mut self) {
258        self.detach_inner();
259    }
260}
261
262#[cfg(all(target_arch = "aarch64", target_os = "none"))]
263unsafe extern "C" fn child_thread_start() -> ! {
264    let tcb_ptr = current_tcb_ptr();
265    // SAFETY: `CLONE_SETTLS` installed this child's valid `ThreadControlBlock` in `TPIDR_EL0`.
266    unsafe {
267        if let Some(entry) = (*tcb_ptr).entry_fn {
268            entry((*tcb_ptr).entry_arg);
269        }
270    }
271    run_current_thread_tls_destructors();
272    // SAFETY: Synchronizing exit disposition with `ThreadHandle::join` / `ThreadHandle::detach`.
273    unsafe {
274        if (*tcb_ptr).join_state.swap(JOIN_EXITING, Ordering::AcqRel) == JOIN_JOINABLE {
275            crate::process::exit_thread(0);
276        }
277        let map_base = (*tcb_ptr).stack_map_base as usize;
278        let map_len = (*tcb_ptr).stack_map_len;
279        crate::raw::unmap_stack_and_exit(map_base, map_len)
280    }
281}
282
283/// Spawns a new thread executing `entry(arg)` on a guarded stack.
284///
285/// # Safety
286///
287/// `entry` and `arg` must satisfy thread-safety invariants for the lifetime of the spawned thread.
288pub unsafe fn spawn_thread(
289    stack_bytes: usize,
290    entry: extern "C" fn(*mut u8),
291    arg: *mut u8,
292    detached: bool,
293) -> SysResult<Option<ThreadHandle>> {
294    let usable_stack = stack_bytes
295        .max(MIN_STACK_BYTES)
296        .checked_add(PAGE_SIZE - 1)
297        .map(|v| v & !(PAGE_SIZE - 1))
298        .ok_or(Errno(EINVAL))?;
299    let total_map = usable_stack.checked_add(PAGE_SIZE).ok_or(Errno(EINVAL))?;
300    let map_base = mmap_anonymous(total_map, PROT_READ | PROT_WRITE, MAP_STACK)?;
301    // SAFETY: `map_base` is page-aligned and `[map_base, map_base + PAGE_SIZE)` is within the mapping.
302    if let Err(e) = unsafe { mprotect(map_base, PAGE_SIZE, PROT_NONE) } {
303        // SAFETY: Unmapping the region just returned by `mmap_anonymous`.
304        let _ = unsafe { munmap(map_base, total_map) };
305        return Err(e);
306    }
307
308    // SAFETY: Allocating a 64-byte aligned `ThreadControlBlock` from `GLOBAL_MMAP_HEAP`.
309    let raw_tcb = unsafe { GLOBAL_MMAP_HEAP.alloc_c(size_of::<ThreadControlBlock>(), 64) };
310    if raw_tcb.is_null() {
311        // SAFETY: Unmapping the stack region on TCB allocation failure.
312        let _ = unsafe { munmap(map_base, total_map) };
313        return Err(Errno(ENOMEM));
314    }
315    let tcb = raw_tcb.cast::<ThreadControlBlock>();
316    // SAFETY: `tcb` points to a writable, aligned `ThreadControlBlock` allocation.
317    unsafe {
318        tcb.write(ThreadControlBlock::new());
319        (*tcb)
320            .child_tid
321            .store(if detached { 0 } else { u32::MAX }, Ordering::Relaxed);
322        (*tcb).join_state.store(
323            if detached {
324                JOIN_DETACHED
325            } else {
326                JOIN_JOINABLE
327            },
328            Ordering::Relaxed,
329        );
330        (*tcb).stack_map_base = map_base;
331        (*tcb).stack_map_len = total_map;
332        (*tcb).entry_fn = Some(entry);
333        (*tcb).entry_arg = arg;
334    }
335
336    let child_sp = (((map_base as usize) + total_map) & !15usize) - 64;
337    // SAFETY: `child_sp` lies near the top of the writable stack region above the guard page with 64B headroom.
338    unsafe {
339        let frame = child_sp as *mut usize;
340        frame.write(0);
341        frame.add(1).write(0);
342    }
343
344    let mut flags = CLONE_VM
345        | CLONE_FS
346        | CLONE_FILES
347        | CLONE_SIGHAND
348        | CLONE_THREAD
349        | CLONE_SYSVSEM
350        | CLONE_SETTLS;
351    let (parent_tid_ptr, child_tid_ptr) = if detached {
352        (0usize, 0usize)
353    } else {
354        flags |= CLONE_PARENT_SETTID | CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID;
355        // SAFETY: `tcb` is a valid aligned `ThreadControlBlock`.
356        let tid_addr = unsafe { core::ptr::addr_of!((*tcb).child_tid) as usize };
357        (tid_addr, tid_addr)
358    };
359
360    #[cfg(all(target_arch = "aarch64", target_os = "none"))]
361    // SAFETY: `child_sp`, `tcb`, and TID pointers are valid for `CLONE`.
362    let clone_res = unsafe {
363        crate::raw::clone_thread(
364            flags,
365            child_sp,
366            parent_tid_ptr,
367            tcb as usize,
368            child_tid_ptr,
369            child_thread_start,
370        )
371    };
372    #[cfg(not(all(target_arch = "aarch64", target_os = "none")))]
373    // SAFETY: Invoking mocked/host `CLONE` syscall with validated stack and TCB pointers.
374    let clone_res = unsafe {
375        crate::raw::syscall5(
376            crate::raw::nr::CLONE,
377            flags as usize,
378            child_sp,
379            parent_tid_ptr,
380            tcb as usize,
381            child_tid_ptr,
382        )
383    };
384
385    match clone_res {
386        Ok(_) if detached => {
387            #[cfg(not(all(target_arch = "aarch64", target_os = "none")))]
388            // SAFETY: Under host mock syscalls no child thread runs, so reclaim simulated stack & TCB cleanly.
389            unsafe {
390                let _ = munmap(map_base, total_map);
391                c_free(tcb.cast::<u8>());
392            }
393            Ok(None)
394        }
395        Ok(_) => Ok(Some(ThreadHandle { tcb })),
396        Err(e) => {
397            // SAFETY: `CLONE` failed before any child thread started; reclaim stack and TCB.
398            unsafe {
399                let _ = munmap(map_base, total_map);
400                c_free(tcb.cast::<u8>());
401            }
402            Err(e)
403        }
404    }
405}
406
407#[cfg(test)]
408mod tests {
409    use super::*;
410    use crate::errno::EAGAIN;
411    use crate::raw::{nr, with_mock_syscall};
412    use core::sync::atomic::AtomicBool;
413
414    #[test]
415    fn errno_and_tls_destructors() {
416        assert_ne!(current_thread_id(), 0);
417        let ptr = errno_cell();
418        unsafe {
419            let prev = *ptr;
420            *ptr = 42;
421            assert_eq!(*errno_cell(), 42);
422            *ptr = prev;
423        }
424        static DTOR_CALLED: AtomicUsize = AtomicUsize::new(0);
425        unsafe extern "C" fn dtor(val: *mut u8) {
426            DTOR_CALLED.store(val as usize, Ordering::SeqCst);
427        }
428        let k = tls_key_create(Some(dtor)).expect("tls key");
429        assert!(tls_set(k, core::ptr::dangling_mut::<u8>()));
430        assert!(!tls_get(k).is_null());
431        run_current_thread_tls_destructors();
432        assert_ne!(DTOR_CALLED.load(Ordering::SeqCst), 0);
433        assert!(tls_get(k).is_null());
434        tls_key_delete(k);
435    }
436
437    #[test]
438    fn spawn_join_and_clone_error_cleanup() {
439        #[repr(align(4096))]
440        struct AlignedArena([u8; 256 * 1024]);
441
442        extern "C" fn dummy(_: *mut u8) {}
443        static mut ARENA: AlignedArena = AlignedArena([0; 256 * 1024]);
444        static OFFSET: AtomicUsize = AtomicUsize::new(0);
445        static FAIL_CLONE: AtomicBool = AtomicBool::new(false);
446        static MUNMAPS: AtomicUsize = AtomicUsize::new(0);
447
448        OFFSET.store(0, Ordering::SeqCst);
449        FAIL_CLONE.store(false, Ordering::SeqCst);
450        MUNMAPS.store(0, Ordering::SeqCst);
451
452        with_mock_syscall(
453            |sys_nr, args| match sys_nr {
454                nr::MMAP => {
455                    let len = (args[1] + 4095) & !4095;
456                    let off = OFFSET.fetch_add(len, Ordering::SeqCst);
457                    if off + len > 256 * 1024 {
458                        return Err(Errno(ENOMEM));
459                    }
460                    // SAFETY: `off + len` lies within `ARENA` while `MOCK_LOCK` serializes mock tests.
461                    let ptr = unsafe { core::ptr::addr_of_mut!(ARENA.0).cast::<u8>().add(off) };
462                    Ok(ptr as usize)
463                }
464                nr::MPROTECT => Ok(0),
465                nr::MUNMAP => {
466                    MUNMAPS.fetch_add(1, Ordering::SeqCst);
467                    Ok(0)
468                }
469                nr::CLONE if FAIL_CLONE.load(Ordering::SeqCst) => Err(Errno(EAGAIN)),
470                nr::CLONE => {
471                    let tid = args[4] as *const AtomicU32;
472                    if !tid.is_null() {
473                        unsafe { (*tid).store(0, Ordering::Release) };
474                    }
475                    Ok(777)
476                }
477                _ => Err(Errno(crate::errno::ENOSYS)),
478            },
479            || {
480                let h = unsafe { spawn_thread(MIN_STACK_BYTES, dummy, null_mut(), false) }
481                    .expect("spawn")
482                    .expect("handle");
483                let before = MUNMAPS.load(Ordering::SeqCst);
484                h.join().expect("join");
485                assert!(MUNMAPS.load(Ordering::SeqCst) > before);
486
487                FAIL_CLONE.store(true, Ordering::SeqCst);
488                let before_fail = MUNMAPS.load(Ordering::SeqCst);
489                let err = unsafe { spawn_thread(MIN_STACK_BYTES, dummy, null_mut(), false) };
490                assert_eq!(err.err(), Some(Errno(EAGAIN)));
491                assert!(MUNMAPS.load(Ordering::SeqCst) > before_fail);
492            },
493        );
494    }
495}