1use core::cell::UnsafeCell;
5use core::mem::{ManuallyDrop, size_of};
6use core::ptr::null_mut;
7use core::sync::atomic::{AtomicU8, AtomicU32, AtomicUsize, Ordering};
8
9use crate::errno::{EDEADLK, EINVAL, ENOMEM, Errno, SysResult};
10use crate::heap::{GLOBAL_MMAP_HEAP, c_free};
11use crate::mm::{
12 MAP_STACK, PAGE_SIZE, PROT_NONE, PROT_READ, PROT_WRITE, mmap_anonymous, mprotect, munmap,
13};
14
15pub const MAX_TLS_KEYS: usize = 64;
16pub const DEFAULT_STACK_BYTES: usize = 512 * 1024;
17pub const MIN_STACK_BYTES: usize = 64 * 1024;
18
19pub const CLONE_VM: u64 = 0x0000_0100;
20pub const CLONE_FS: u64 = 0x0000_0200;
21pub const CLONE_FILES: u64 = 0x0000_0400;
22pub const CLONE_SIGHAND: u64 = 0x0000_0800;
23pub const CLONE_THREAD: u64 = 0x0001_0000;
24pub const CLONE_SYSVSEM: u64 = 0x0004_0000;
25pub const CLONE_SETTLS: u64 = 0x0008_0000;
26pub const CLONE_PARENT_SETTID: u64 = 0x0010_0000;
27pub const CLONE_CHILD_CLEARTID: u64 = 0x0020_0000;
28pub const CLONE_CHILD_SETTID: u64 = 0x0100_0000;
29
30const JOIN_JOINABLE: u32 = 0;
31const JOIN_DETACHED: u32 = 1;
32#[cfg(all(target_arch = "aarch64", target_os = "none"))]
33const JOIN_EXITING: u32 = 2;
34
35const _: () = assert!(MAX_TLS_KEYS == 64);
36
37const TLS_KEY_FREE: u8 = 0;
38const TLS_KEY_IN_USE: u8 = 1;
39
40#[repr(C, align(64))]
41struct ThreadControlBlock {
42 errno: UnsafeCell<i32>,
43 tls_values: UnsafeCell<[*mut u8; 64]>,
44 child_tid: AtomicU32,
45 join_state: AtomicU32,
46 stack_map_base: *mut u8,
47 stack_map_len: usize,
48 entry_fn: Option<extern "C" fn(*mut u8)>,
49 entry_arg: *mut u8,
50}
51
52impl ThreadControlBlock {
53 const fn new() -> Self {
54 Self {
55 errno: UnsafeCell::new(0),
56 tls_values: UnsafeCell::new([null_mut(); 64]),
57 child_tid: AtomicU32::new(0),
58 join_state: AtomicU32::new(JOIN_JOINABLE),
59 stack_map_base: null_mut(),
60 stack_map_len: 0,
61 entry_fn: None,
62 entry_arg: null_mut(),
63 }
64 }
65}
66
67#[repr(transparent)]
68struct SyncTcb(ThreadControlBlock);
69unsafe impl Sync for SyncTcb {}
71
72static MAIN_TCB: SyncTcb = SyncTcb(ThreadControlBlock::new());
73
74#[inline]
75fn current_tcb_ptr() -> *mut ThreadControlBlock {
76 let main_addr = core::ptr::addr_of!(MAIN_TCB.0) as usize;
77 let tp = crate::raw::read_thread_pointer();
78 let addr = if tp == 0 { main_addr } else { tp };
79 addr as *mut ThreadControlBlock
80}
81
82#[must_use]
83pub fn errno_cell() -> *mut i32 {
84 let tcb = current_tcb_ptr();
85 unsafe { (*tcb).errno.get() }
87}
88
89#[must_use]
90pub fn current_thread_id() -> u64 {
91 current_tcb_ptr() as usize as u64
92}
93
94static TLS_KEY_STATE: [AtomicU8; 64] = [const { AtomicU8::new(TLS_KEY_FREE) }; 64];
95static TLS_DTORS: [AtomicUsize; 64] = [const { AtomicUsize::new(0) }; 64];
96
97#[must_use]
98pub fn tls_key_create(dtor: Option<unsafe extern "C" fn(*mut u8)>) -> Option<usize> {
99 let dtor_addr = dtor.map_or(0, |f| f as usize);
100 let tcb = current_tcb_ptr();
101 for key in 0..MAX_TLS_KEYS {
102 if TLS_KEY_STATE[key]
103 .compare_exchange(
104 TLS_KEY_FREE,
105 TLS_KEY_IN_USE,
106 Ordering::AcqRel,
107 Ordering::Acquire,
108 )
109 .is_ok()
110 {
111 TLS_DTORS[key].store(dtor_addr, Ordering::Release);
112 unsafe { (*(*tcb).tls_values.get())[key] = null_mut() };
114 return Some(key);
115 }
116 }
117 None
118}
119
120pub fn tls_key_delete(key: usize) {
121 if key >= MAX_TLS_KEYS {
122 return;
123 }
124 let tcb = current_tcb_ptr();
125 TLS_DTORS[key].store(0, Ordering::Release);
126 unsafe { (*(*tcb).tls_values.get())[key] = null_mut() };
128 TLS_KEY_STATE[key].store(TLS_KEY_FREE, Ordering::Release);
129}
130
131#[must_use]
132pub fn tls_get(key: usize) -> *mut u8 {
133 if key >= MAX_TLS_KEYS || TLS_KEY_STATE[key].load(Ordering::Acquire) != TLS_KEY_IN_USE {
134 return null_mut();
135 }
136 let tcb = current_tcb_ptr();
137 unsafe { (*(*tcb).tls_values.get())[key] }
139}
140
141pub fn tls_set(key: usize, val: *mut u8) -> bool {
142 if key >= MAX_TLS_KEYS || TLS_KEY_STATE[key].load(Ordering::Acquire) != TLS_KEY_IN_USE {
143 return false;
144 }
145 let tcb = current_tcb_ptr();
146 unsafe { (*(*tcb).tls_values.get())[key] = val };
148 true
149}
150
151pub fn run_current_thread_tls_destructors() {
152 let tcb = current_tcb_ptr();
153 for key in 0..MAX_TLS_KEYS {
154 let val = unsafe { (*(*tcb).tls_values.get())[key] };
156 if !val.is_null() {
157 unsafe { (*(*tcb).tls_values.get())[key] = null_mut() };
159 if TLS_KEY_STATE[key].load(Ordering::Acquire) == TLS_KEY_IN_USE {
160 let addr = TLS_DTORS[key].load(Ordering::Acquire);
161 if addr != 0 {
162 let dtor: unsafe extern "C" fn(*mut u8) = unsafe { core::mem::transmute(addr) };
164 unsafe { dtor(val) };
166 }
167 }
168 }
169 }
170}
171
172pub struct ThreadHandle {
173 tcb: *mut ThreadControlBlock,
174}
175
176unsafe impl Send for ThreadHandle {}
178unsafe impl Sync for ThreadHandle {}
180
181impl ThreadHandle {
182 #[must_use]
183 pub fn id(&self) -> u64 {
184 self.tcb as usize as u64
185 }
186
187 #[must_use]
188 pub fn is_current(&self) -> bool {
189 current_thread_id() == self.id()
190 }
191
192 pub fn join(self) -> SysResult<()> {
193 if self.is_current() {
194 let _ = ManuallyDrop::new(self);
195 return Err(Errno(EDEADLK));
196 }
197 let this = ManuallyDrop::new(self);
198 let tcb = this.tcb;
199 unsafe {
201 loop {
202 let tid = (*tcb).child_tid.load(Ordering::Acquire);
203 if tid == 0 {
204 break;
205 }
206 crate::sync::futex_wait(&(*tcb).child_tid, tid, None);
207 }
208 if !(*tcb).stack_map_base.is_null() && (*tcb).stack_map_len > 0 {
209 let _ = munmap((*tcb).stack_map_base, (*tcb).stack_map_len);
210 }
211 c_free(tcb.cast::<u8>());
212 }
213 Ok(())
214 }
215
216 pub fn detach(self) -> SysResult<()> {
217 let mut this = ManuallyDrop::new(self);
218 this.detach_inner();
219 Ok(())
220 }
221
222 fn detach_inner(&mut self) {
223 let tcb = self.tcb;
224 if tcb.is_null() {
225 return;
226 }
227 self.tcb = null_mut();
228 unsafe {
230 if (*tcb)
231 .join_state
232 .compare_exchange(
233 JOIN_JOINABLE,
234 JOIN_DETACHED,
235 Ordering::AcqRel,
236 Ordering::Acquire,
237 )
238 .is_err()
239 {
240 loop {
241 let tid = (*tcb).child_tid.load(Ordering::Acquire);
242 if tid == 0 {
243 break;
244 }
245 crate::sync::futex_wait(&(*tcb).child_tid, tid, None);
246 }
247 if !(*tcb).stack_map_base.is_null() && (*tcb).stack_map_len > 0 {
248 let _ = munmap((*tcb).stack_map_base, (*tcb).stack_map_len);
249 }
250 c_free(tcb.cast::<u8>());
251 }
252 }
253 }
254}
255
256impl Drop for ThreadHandle {
257 fn drop(&mut self) {
258 self.detach_inner();
259 }
260}
261
262#[cfg(all(target_arch = "aarch64", target_os = "none"))]
263unsafe extern "C" fn child_thread_start() -> ! {
264 let tcb_ptr = current_tcb_ptr();
265 unsafe {
267 if let Some(entry) = (*tcb_ptr).entry_fn {
268 entry((*tcb_ptr).entry_arg);
269 }
270 }
271 run_current_thread_tls_destructors();
272 unsafe {
274 if (*tcb_ptr).join_state.swap(JOIN_EXITING, Ordering::AcqRel) == JOIN_JOINABLE {
275 crate::process::exit_thread(0);
276 }
277 let map_base = (*tcb_ptr).stack_map_base as usize;
278 let map_len = (*tcb_ptr).stack_map_len;
279 crate::raw::unmap_stack_and_exit(map_base, map_len)
280 }
281}
282
283pub unsafe fn spawn_thread(
289 stack_bytes: usize,
290 entry: extern "C" fn(*mut u8),
291 arg: *mut u8,
292 detached: bool,
293) -> SysResult<Option<ThreadHandle>> {
294 let usable_stack = stack_bytes
295 .max(MIN_STACK_BYTES)
296 .checked_add(PAGE_SIZE - 1)
297 .map(|v| v & !(PAGE_SIZE - 1))
298 .ok_or(Errno(EINVAL))?;
299 let total_map = usable_stack.checked_add(PAGE_SIZE).ok_or(Errno(EINVAL))?;
300 let map_base = mmap_anonymous(total_map, PROT_READ | PROT_WRITE, MAP_STACK)?;
301 if let Err(e) = unsafe { mprotect(map_base, PAGE_SIZE, PROT_NONE) } {
303 let _ = unsafe { munmap(map_base, total_map) };
305 return Err(e);
306 }
307
308 let raw_tcb = unsafe { GLOBAL_MMAP_HEAP.alloc_c(size_of::<ThreadControlBlock>(), 64) };
310 if raw_tcb.is_null() {
311 let _ = unsafe { munmap(map_base, total_map) };
313 return Err(Errno(ENOMEM));
314 }
315 let tcb = raw_tcb.cast::<ThreadControlBlock>();
316 unsafe {
318 tcb.write(ThreadControlBlock::new());
319 (*tcb)
320 .child_tid
321 .store(if detached { 0 } else { u32::MAX }, Ordering::Relaxed);
322 (*tcb).join_state.store(
323 if detached {
324 JOIN_DETACHED
325 } else {
326 JOIN_JOINABLE
327 },
328 Ordering::Relaxed,
329 );
330 (*tcb).stack_map_base = map_base;
331 (*tcb).stack_map_len = total_map;
332 (*tcb).entry_fn = Some(entry);
333 (*tcb).entry_arg = arg;
334 }
335
336 let child_sp = (((map_base as usize) + total_map) & !15usize) - 64;
337 unsafe {
339 let frame = child_sp as *mut usize;
340 frame.write(0);
341 frame.add(1).write(0);
342 }
343
344 let mut flags = CLONE_VM
345 | CLONE_FS
346 | CLONE_FILES
347 | CLONE_SIGHAND
348 | CLONE_THREAD
349 | CLONE_SYSVSEM
350 | CLONE_SETTLS;
351 let (parent_tid_ptr, child_tid_ptr) = if detached {
352 (0usize, 0usize)
353 } else {
354 flags |= CLONE_PARENT_SETTID | CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID;
355 let tid_addr = unsafe { core::ptr::addr_of!((*tcb).child_tid) as usize };
357 (tid_addr, tid_addr)
358 };
359
360 #[cfg(all(target_arch = "aarch64", target_os = "none"))]
361 let clone_res = unsafe {
363 crate::raw::clone_thread(
364 flags,
365 child_sp,
366 parent_tid_ptr,
367 tcb as usize,
368 child_tid_ptr,
369 child_thread_start,
370 )
371 };
372 #[cfg(not(all(target_arch = "aarch64", target_os = "none")))]
373 let clone_res = unsafe {
375 crate::raw::syscall5(
376 crate::raw::nr::CLONE,
377 flags as usize,
378 child_sp,
379 parent_tid_ptr,
380 tcb as usize,
381 child_tid_ptr,
382 )
383 };
384
385 match clone_res {
386 Ok(_) if detached => {
387 #[cfg(not(all(target_arch = "aarch64", target_os = "none")))]
388 unsafe {
390 let _ = munmap(map_base, total_map);
391 c_free(tcb.cast::<u8>());
392 }
393 Ok(None)
394 }
395 Ok(_) => Ok(Some(ThreadHandle { tcb })),
396 Err(e) => {
397 unsafe {
399 let _ = munmap(map_base, total_map);
400 c_free(tcb.cast::<u8>());
401 }
402 Err(e)
403 }
404 }
405}
406
407#[cfg(test)]
408mod tests {
409 use super::*;
410 use crate::errno::EAGAIN;
411 use crate::raw::{nr, with_mock_syscall};
412 use core::sync::atomic::AtomicBool;
413
414 #[test]
415 fn errno_and_tls_destructors() {
416 assert_ne!(current_thread_id(), 0);
417 let ptr = errno_cell();
418 unsafe {
419 let prev = *ptr;
420 *ptr = 42;
421 assert_eq!(*errno_cell(), 42);
422 *ptr = prev;
423 }
424 static DTOR_CALLED: AtomicUsize = AtomicUsize::new(0);
425 unsafe extern "C" fn dtor(val: *mut u8) {
426 DTOR_CALLED.store(val as usize, Ordering::SeqCst);
427 }
428 let k = tls_key_create(Some(dtor)).expect("tls key");
429 assert!(tls_set(k, core::ptr::dangling_mut::<u8>()));
430 assert!(!tls_get(k).is_null());
431 run_current_thread_tls_destructors();
432 assert_ne!(DTOR_CALLED.load(Ordering::SeqCst), 0);
433 assert!(tls_get(k).is_null());
434 tls_key_delete(k);
435 }
436
437 #[test]
438 fn spawn_join_and_clone_error_cleanup() {
439 #[repr(align(4096))]
440 struct AlignedArena([u8; 256 * 1024]);
441
442 extern "C" fn dummy(_: *mut u8) {}
443 static mut ARENA: AlignedArena = AlignedArena([0; 256 * 1024]);
444 static OFFSET: AtomicUsize = AtomicUsize::new(0);
445 static FAIL_CLONE: AtomicBool = AtomicBool::new(false);
446 static MUNMAPS: AtomicUsize = AtomicUsize::new(0);
447
448 OFFSET.store(0, Ordering::SeqCst);
449 FAIL_CLONE.store(false, Ordering::SeqCst);
450 MUNMAPS.store(0, Ordering::SeqCst);
451
452 with_mock_syscall(
453 |sys_nr, args| match sys_nr {
454 nr::MMAP => {
455 let len = (args[1] + 4095) & !4095;
456 let off = OFFSET.fetch_add(len, Ordering::SeqCst);
457 if off + len > 256 * 1024 {
458 return Err(Errno(ENOMEM));
459 }
460 let ptr = unsafe { core::ptr::addr_of_mut!(ARENA.0).cast::<u8>().add(off) };
462 Ok(ptr as usize)
463 }
464 nr::MPROTECT => Ok(0),
465 nr::MUNMAP => {
466 MUNMAPS.fetch_add(1, Ordering::SeqCst);
467 Ok(0)
468 }
469 nr::CLONE if FAIL_CLONE.load(Ordering::SeqCst) => Err(Errno(EAGAIN)),
470 nr::CLONE => {
471 let tid = args[4] as *const AtomicU32;
472 if !tid.is_null() {
473 unsafe { (*tid).store(0, Ordering::Release) };
474 }
475 Ok(777)
476 }
477 _ => Err(Errno(crate::errno::ENOSYS)),
478 },
479 || {
480 let h = unsafe { spawn_thread(MIN_STACK_BYTES, dummy, null_mut(), false) }
481 .expect("spawn")
482 .expect("handle");
483 let before = MUNMAPS.load(Ordering::SeqCst);
484 h.join().expect("join");
485 assert!(MUNMAPS.load(Ordering::SeqCst) > before);
486
487 FAIL_CLONE.store(true, Ordering::SeqCst);
488 let before_fail = MUNMAPS.load(Ordering::SeqCst);
489 let err = unsafe { spawn_thread(MIN_STACK_BYTES, dummy, null_mut(), false) };
490 assert_eq!(err.err(), Some(Errno(EAGAIN)));
491 assert!(MUNMAPS.load(Ordering::SeqCst) > before_fail);
492 },
493 );
494 }
495}