1use core::cell::UnsafeCell;
5use core::mem::{ManuallyDrop, size_of};
6use core::ptr::null_mut;
7use core::sync::atomic::{AtomicU8, AtomicU32, AtomicUsize, Ordering};
8
9use crate::errno::{EDEADLK, EINVAL, ENOMEM, Errno, SysResult};
10use crate::heap::{GLOBAL_MMAP_HEAP, c_free};
11use crate::mm::{
12 MAP_STACK, PAGE_SIZE, PROT_NONE, PROT_READ, PROT_WRITE, mmap_anonymous, mprotect, munmap,
13};
14
15pub const MAX_TLS_KEYS: usize = 64;
16pub const DEFAULT_STACK_BYTES: usize = 512 * 1024;
17pub const MIN_STACK_BYTES: usize = 64 * 1024;
18
19pub const CLONE_VM: u64 = 0x0000_0100;
20pub const CLONE_FS: u64 = 0x0000_0200;
21pub const CLONE_FILES: u64 = 0x0000_0400;
22pub const CLONE_SIGHAND: u64 = 0x0000_0800;
23pub const CLONE_THREAD: u64 = 0x0001_0000;
24pub const CLONE_SYSVSEM: u64 = 0x0004_0000;
25pub const CLONE_SETTLS: u64 = 0x0008_0000;
26pub const CLONE_PARENT_SETTID: u64 = 0x0010_0000;
27pub const CLONE_CHILD_CLEARTID: u64 = 0x0020_0000;
28pub const CLONE_CHILD_SETTID: u64 = 0x0100_0000;
29
30const JOIN_JOINABLE: u32 = 0;
31const JOIN_DETACHED: u32 = 1;
32#[cfg(all(target_arch = "aarch64", target_os = "none"))]
33const JOIN_EXITING: u32 = 2;
34
35const _: () = assert!(MAX_TLS_KEYS == 64);
36
37const TLS_KEY_FREE: u8 = 0;
38const TLS_KEY_IN_USE: u8 = 1;
39
40#[repr(C, align(64))]
41pub(crate) struct ThreadControlBlock {
42 errno: UnsafeCell<i32>,
43 pub(crate) mag_len: UnsafeCell<[u8; 8]>,
44 pub(crate) mag_ptrs: UnsafeCell<[[*mut u8; 8]; 8]>,
45 tls_values: UnsafeCell<[*mut u8; 64]>,
46 child_tid: AtomicU32,
47 join_state: AtomicU32,
48 stack_map_base: *mut u8,
49 stack_map_len: usize,
50 entry_fn: Option<extern "C" fn(*mut u8)>,
51 entry_arg: *mut u8,
52}
53
54impl ThreadControlBlock {
55 const fn new() -> Self {
56 Self {
57 errno: UnsafeCell::new(0),
58 mag_len: UnsafeCell::new([0; 8]),
59 mag_ptrs: UnsafeCell::new([[null_mut(); 8]; 8]),
60 tls_values: UnsafeCell::new([null_mut(); 64]),
61 child_tid: AtomicU32::new(0),
62 join_state: AtomicU32::new(JOIN_JOINABLE),
63 stack_map_base: null_mut(),
64 stack_map_len: 0,
65 entry_fn: None,
66 entry_arg: null_mut(),
67 }
68 }
69}
70
71#[repr(transparent)]
72struct SyncTcb(ThreadControlBlock);
73unsafe impl Sync for SyncTcb {}
75
76static MAIN_TCB: SyncTcb = SyncTcb(ThreadControlBlock::new());
77
78#[inline]
79pub(crate) fn current_tcb_ptr() -> *mut ThreadControlBlock {
80 let main_addr = core::ptr::addr_of!(MAIN_TCB.0) as usize;
81 let tp = crate::raw::read_thread_pointer();
82 let addr = if tp == 0 { main_addr } else { tp };
83 addr as *mut ThreadControlBlock
84}
85
86#[must_use]
87#[inline]
88pub fn errno_cell() -> *mut i32 {
89 let tcb = current_tcb_ptr();
90 unsafe { (*tcb).errno.get() }
92}
93
94#[must_use]
95#[inline]
96pub fn current_thread_id() -> u64 {
97 current_tcb_ptr() as usize as u64
98}
99
100static TLS_KEY_STATE: [AtomicU8; 64] = [const { AtomicU8::new(TLS_KEY_FREE) }; 64];
101static TLS_DTORS: [AtomicUsize; 64] = [const { AtomicUsize::new(0) }; 64];
102
103#[must_use]
104pub fn tls_key_create(dtor: Option<unsafe extern "C" fn(*mut u8)>) -> Option<usize> {
105 let dtor_addr = dtor.map_or(0, |f| f as usize);
106 let tcb = current_tcb_ptr();
107 for key in 0..MAX_TLS_KEYS {
108 if TLS_KEY_STATE[key]
109 .compare_exchange(
110 TLS_KEY_FREE,
111 TLS_KEY_IN_USE,
112 Ordering::AcqRel,
113 Ordering::Acquire,
114 )
115 .is_ok()
116 {
117 TLS_DTORS[key].store(dtor_addr, Ordering::Release);
118 unsafe { (*(*tcb).tls_values.get())[key] = null_mut() };
120 return Some(key);
121 }
122 }
123 None
124}
125
126pub fn tls_key_delete(key: usize) {
127 if key >= MAX_TLS_KEYS {
128 return;
129 }
130 let tcb = current_tcb_ptr();
131 TLS_DTORS[key].store(0, Ordering::Release);
132 unsafe { (*(*tcb).tls_values.get())[key] = null_mut() };
134 TLS_KEY_STATE[key].store(TLS_KEY_FREE, Ordering::Release);
135}
136
137#[must_use]
138#[inline]
139pub fn tls_get(key: usize) -> *mut u8 {
140 if key >= MAX_TLS_KEYS || TLS_KEY_STATE[key].load(Ordering::Acquire) != TLS_KEY_IN_USE {
141 return null_mut();
142 }
143 let tcb = current_tcb_ptr();
144 unsafe { (*(*tcb).tls_values.get())[key] }
146}
147
148#[inline]
149pub fn tls_set(key: usize, val: *mut u8) -> bool {
150 if key >= MAX_TLS_KEYS || TLS_KEY_STATE[key].load(Ordering::Acquire) != TLS_KEY_IN_USE {
151 return false;
152 }
153 let tcb = current_tcb_ptr();
154 unsafe { (*(*tcb).tls_values.get())[key] = val };
156 true
157}
158
159pub fn run_current_thread_tls_destructors() {
160 let tcb = current_tcb_ptr();
161 for key in 0..MAX_TLS_KEYS {
162 let val = unsafe { (*(*tcb).tls_values.get())[key] };
164 if !val.is_null() {
165 unsafe { (*(*tcb).tls_values.get())[key] = null_mut() };
167 if TLS_KEY_STATE[key].load(Ordering::Acquire) == TLS_KEY_IN_USE {
168 let addr = TLS_DTORS[key].load(Ordering::Acquire);
169 if addr != 0 {
170 let dtor: unsafe extern "C" fn(*mut u8) = unsafe { core::mem::transmute(addr) };
172 unsafe { dtor(val) };
174 }
175 }
176 }
177 }
178 #[cfg(feature = "alloc")]
179 crate::heap::flush_current_thread_magazine();
180}
181
182pub struct ThreadHandle {
183 tcb: *mut ThreadControlBlock,
184}
185
186unsafe impl Send for ThreadHandle {}
188unsafe impl Sync for ThreadHandle {}
190
191impl ThreadHandle {
192 #[must_use]
193 pub fn id(&self) -> u64 {
194 self.tcb as usize as u64
195 }
196
197 #[must_use]
198 pub fn is_current(&self) -> bool {
199 current_thread_id() == self.id()
200 }
201
202 pub fn join(self) -> SysResult<()> {
203 if self.is_current() {
204 let _ = ManuallyDrop::new(self);
205 return Err(Errno(EDEADLK));
206 }
207 let this = ManuallyDrop::new(self);
208 let tcb = this.tcb;
209 unsafe {
211 loop {
212 let tid = (*tcb).child_tid.load(Ordering::Acquire);
213 if tid == 0 {
214 break;
215 }
216 crate::sync::futex_wait(&(*tcb).child_tid, tid, None);
217 }
218 if !(*tcb).stack_map_base.is_null() && (*tcb).stack_map_len > 0 {
219 let _ = munmap((*tcb).stack_map_base, (*tcb).stack_map_len);
220 }
221 c_free(tcb.cast::<u8>());
222 }
223 Ok(())
224 }
225
226 pub fn detach(self) -> SysResult<()> {
227 let mut this = ManuallyDrop::new(self);
228 this.detach_inner();
229 Ok(())
230 }
231
232 fn detach_inner(&mut self) {
233 let tcb = self.tcb;
234 if tcb.is_null() {
235 return;
236 }
237 self.tcb = null_mut();
238 unsafe {
240 if (*tcb)
241 .join_state
242 .compare_exchange(
243 JOIN_JOINABLE,
244 JOIN_DETACHED,
245 Ordering::AcqRel,
246 Ordering::Acquire,
247 )
248 .is_err()
249 {
250 loop {
251 let tid = (*tcb).child_tid.load(Ordering::Acquire);
252 if tid == 0 {
253 break;
254 }
255 crate::sync::futex_wait(&(*tcb).child_tid, tid, None);
256 }
257 if !(*tcb).stack_map_base.is_null() && (*tcb).stack_map_len > 0 {
258 let _ = munmap((*tcb).stack_map_base, (*tcb).stack_map_len);
259 }
260 c_free(tcb.cast::<u8>());
261 }
262 }
263 }
264}
265
266impl Drop for ThreadHandle {
267 fn drop(&mut self) {
268 self.detach_inner();
269 }
270}
271
272#[cfg(all(target_arch = "aarch64", target_os = "none"))]
273unsafe extern "C" fn child_thread_start() -> ! {
274 let tcb_ptr = current_tcb_ptr();
275 unsafe {
277 if let Some(entry) = (*tcb_ptr).entry_fn {
278 entry((*tcb_ptr).entry_arg);
279 }
280 }
281 run_current_thread_tls_destructors();
282 unsafe {
284 if (*tcb_ptr).join_state.swap(JOIN_EXITING, Ordering::AcqRel) == JOIN_JOINABLE {
285 crate::process::exit_thread(0);
286 }
287 let map_base = (*tcb_ptr).stack_map_base as usize;
288 let map_len = (*tcb_ptr).stack_map_len;
289 crate::raw::unmap_stack_and_exit(map_base, map_len)
290 }
291}
292
293pub unsafe fn spawn_thread(
299 stack_bytes: usize,
300 entry: extern "C" fn(*mut u8),
301 arg: *mut u8,
302 detached: bool,
303) -> SysResult<Option<ThreadHandle>> {
304 let usable_stack = stack_bytes
305 .max(MIN_STACK_BYTES)
306 .checked_add(PAGE_SIZE - 1)
307 .map(|v| v & !(PAGE_SIZE - 1))
308 .ok_or(Errno(EINVAL))?;
309 let total_map = usable_stack.checked_add(PAGE_SIZE).ok_or(Errno(EINVAL))?;
310 let map_base = mmap_anonymous(total_map, PROT_READ | PROT_WRITE, MAP_STACK)?;
311 if let Err(e) = unsafe { mprotect(map_base, PAGE_SIZE, PROT_NONE) } {
313 let _ = unsafe { munmap(map_base, total_map) };
315 return Err(e);
316 }
317
318 let raw_tcb = unsafe { GLOBAL_MMAP_HEAP.alloc_c(size_of::<ThreadControlBlock>(), 64) };
320 if raw_tcb.is_null() {
321 let _ = unsafe { munmap(map_base, total_map) };
323 return Err(Errno(ENOMEM));
324 }
325 let tcb = raw_tcb.cast::<ThreadControlBlock>();
326 unsafe {
328 tcb.write(ThreadControlBlock::new());
329 (*tcb)
330 .child_tid
331 .store(if detached { 0 } else { u32::MAX }, Ordering::Relaxed);
332 (*tcb).join_state.store(
333 if detached {
334 JOIN_DETACHED
335 } else {
336 JOIN_JOINABLE
337 },
338 Ordering::Relaxed,
339 );
340 (*tcb).stack_map_base = map_base;
341 (*tcb).stack_map_len = total_map;
342 (*tcb).entry_fn = Some(entry);
343 (*tcb).entry_arg = arg;
344 }
345
346 let child_sp = (((map_base as usize) + total_map) & !15usize) - 64;
347 unsafe {
349 let frame = child_sp as *mut usize;
350 frame.write(0);
351 frame.add(1).write(0);
352 }
353
354 let mut flags = CLONE_VM
355 | CLONE_FS
356 | CLONE_FILES
357 | CLONE_SIGHAND
358 | CLONE_THREAD
359 | CLONE_SYSVSEM
360 | CLONE_SETTLS;
361 let (parent_tid_ptr, child_tid_ptr) = if detached {
362 (0usize, 0usize)
363 } else {
364 flags |= CLONE_PARENT_SETTID | CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID;
365 let tid_addr = unsafe { core::ptr::addr_of!((*tcb).child_tid) as usize };
367 (tid_addr, tid_addr)
368 };
369
370 #[cfg(all(target_arch = "aarch64", target_os = "none"))]
371 let clone_res = unsafe {
373 crate::raw::clone_thread(
374 flags,
375 child_sp,
376 parent_tid_ptr,
377 tcb as usize,
378 child_tid_ptr,
379 child_thread_start,
380 )
381 };
382 #[cfg(not(all(target_arch = "aarch64", target_os = "none")))]
383 let clone_res = unsafe {
385 crate::raw::syscall5(
386 crate::raw::nr::CLONE,
387 flags as usize,
388 child_sp,
389 parent_tid_ptr,
390 tcb as usize,
391 child_tid_ptr,
392 )
393 };
394
395 match clone_res {
396 Ok(_) if detached => {
397 #[cfg(not(all(target_arch = "aarch64", target_os = "none")))]
398 unsafe {
400 let _ = munmap(map_base, total_map);
401 c_free(tcb.cast::<u8>());
402 }
403 Ok(None)
404 }
405 Ok(_) => Ok(Some(ThreadHandle { tcb })),
406 Err(e) => {
407 unsafe {
409 let _ = munmap(map_base, total_map);
410 c_free(tcb.cast::<u8>());
411 }
412 Err(e)
413 }
414 }
415}
416
417#[cfg(test)]
418mod tests {
419 use super::*;
420 use crate::errno::EAGAIN;
421 use crate::raw::{nr, with_mock_syscall};
422 use core::sync::atomic::AtomicBool;
423
424 #[test]
425 fn errno_and_tls_destructors() {
426 assert_ne!(current_thread_id(), 0);
427 let ptr = errno_cell();
428 unsafe {
429 let prev = *ptr;
430 *ptr = 42;
431 assert_eq!(*errno_cell(), 42);
432 *ptr = prev;
433 }
434 static DTOR_CALLED: AtomicUsize = AtomicUsize::new(0);
435 unsafe extern "C" fn dtor(val: *mut u8) {
436 DTOR_CALLED.store(val as usize, Ordering::SeqCst);
437 }
438 let k = tls_key_create(Some(dtor)).expect("tls key");
439 assert!(tls_set(k, core::ptr::dangling_mut::<u8>()));
440 assert!(!tls_get(k).is_null());
441 run_current_thread_tls_destructors();
442 assert_ne!(DTOR_CALLED.load(Ordering::SeqCst), 0);
443 assert!(tls_get(k).is_null());
444 tls_key_delete(k);
445 }
446
447 #[test]
448 fn spawn_join_and_clone_error_cleanup() {
449 #[repr(align(4096))]
450 struct AlignedArena([u8; 256 * 1024]);
451
452 extern "C" fn dummy(_: *mut u8) {}
453 static mut ARENA: AlignedArena = AlignedArena([0; 256 * 1024]);
454 static OFFSET: AtomicUsize = AtomicUsize::new(0);
455 static FAIL_CLONE: AtomicBool = AtomicBool::new(false);
456 static MUNMAPS: AtomicUsize = AtomicUsize::new(0);
457
458 OFFSET.store(0, Ordering::SeqCst);
459 FAIL_CLONE.store(false, Ordering::SeqCst);
460 MUNMAPS.store(0, Ordering::SeqCst);
461
462 with_mock_syscall(
463 |sys_nr, args| match sys_nr {
464 nr::MMAP => {
465 let len = (args[1] + 4095) & !4095;
466 let off = OFFSET.fetch_add(len, Ordering::SeqCst);
467 if off + len > 256 * 1024 {
468 return Err(Errno(ENOMEM));
469 }
470 let ptr = unsafe { core::ptr::addr_of_mut!(ARENA.0).cast::<u8>().add(off) };
472 Ok(ptr as usize)
473 }
474 nr::MPROTECT => Ok(0),
475 nr::MUNMAP => {
476 MUNMAPS.fetch_add(1, Ordering::SeqCst);
477 Ok(0)
478 }
479 nr::CLONE if FAIL_CLONE.load(Ordering::SeqCst) => Err(Errno(EAGAIN)),
480 nr::CLONE => {
481 let tid = args[4] as *const AtomicU32;
482 if !tid.is_null() {
483 unsafe { (*tid).store(0, Ordering::Release) };
484 }
485 Ok(777)
486 }
487 _ => Err(Errno(crate::errno::ENOSYS)),
488 },
489 || {
490 let h = unsafe { spawn_thread(MIN_STACK_BYTES, dummy, null_mut(), false) }
491 .expect("spawn")
492 .expect("handle");
493 let before = MUNMAPS.load(Ordering::SeqCst);
494 h.join().expect("join");
495 assert!(MUNMAPS.load(Ordering::SeqCst) > before);
496
497 FAIL_CLONE.store(true, Ordering::SeqCst);
498 let before_fail = MUNMAPS.load(Ordering::SeqCst);
499 let err = unsafe { spawn_thread(MIN_STACK_BYTES, dummy, null_mut(), false) };
500 assert_eq!(err.err(), Some(Errno(EAGAIN)));
501 assert!(MUNMAPS.load(Ordering::SeqCst) > before_fail);
502 },
503 );
504 }
505}