1use core::cell::UnsafeCell;
5use core::mem::{ManuallyDrop, size_of};
6use core::ptr::null_mut;
7use core::sync::atomic::{AtomicU8, AtomicU32, AtomicUsize, Ordering};
8
9use crate::errno::{EDEADLK, EINVAL, ENOMEM, Errno, SysResult};
10use crate::heap::{GLOBAL_MMAP_HEAP, c_free};
11use crate::mm::{
12 MAP_STACK, PAGE_SIZE, PROT_NONE, PROT_READ, PROT_WRITE, mmap_anonymous, mprotect, munmap,
13};
14
15pub const MAX_TLS_KEYS: usize = 64;
16pub const DEFAULT_STACK_BYTES: usize = 512 * 1024;
17pub const MIN_STACK_BYTES: usize = 64 * 1024;
18
19pub const CLONE_VM: u64 = 0x0000_0100;
20pub const CLONE_FS: u64 = 0x0000_0200;
21pub const CLONE_FILES: u64 = 0x0000_0400;
22pub const CLONE_SIGHAND: u64 = 0x0000_0800;
23pub const CLONE_THREAD: u64 = 0x0001_0000;
24pub const CLONE_SYSVSEM: u64 = 0x0004_0000;
25pub const CLONE_SETTLS: u64 = 0x0008_0000;
26pub const CLONE_PARENT_SETTID: u64 = 0x0010_0000;
27pub const CLONE_CHILD_CLEARTID: u64 = 0x0020_0000;
28pub const CLONE_CHILD_SETTID: u64 = 0x0100_0000;
29
30const JOIN_JOINABLE: u32 = 0;
31const JOIN_DETACHED: u32 = 1;
32#[cfg(all(target_arch = "aarch64", target_os = "none"))]
33const JOIN_EXITING: u32 = 2;
34
35const TLS_KEY_FREE: u8 = 0;
36const TLS_KEY_IN_USE: u8 = 1;
37
38#[repr(C, align(64))]
39struct ThreadControlBlock {
40 errno: UnsafeCell<i32>,
41 tls_values: UnsafeCell<[*mut u8; MAX_TLS_KEYS]>,
42 child_tid: AtomicU32,
43 join_state: AtomicU32,
44 stack_map_base: *mut u8,
45 stack_map_len: usize,
46 entry_fn: Option<extern "C" fn(*mut u8)>,
47 entry_arg: *mut u8,
48}
49
50impl ThreadControlBlock {
51 const fn new() -> Self {
52 Self {
53 errno: UnsafeCell::new(0),
54 tls_values: UnsafeCell::new([null_mut(); MAX_TLS_KEYS]),
55 child_tid: AtomicU32::new(0),
56 join_state: AtomicU32::new(JOIN_JOINABLE),
57 stack_map_base: null_mut(),
58 stack_map_len: 0,
59 entry_fn: None,
60 entry_arg: null_mut(),
61 }
62 }
63}
64
65#[repr(transparent)]
66struct SyncTcb(ThreadControlBlock);
67unsafe impl Sync for SyncTcb {}
69
70static MAIN_TCB: SyncTcb = SyncTcb(ThreadControlBlock::new());
71
72#[inline]
73fn current_tcb() -> &'static ThreadControlBlock {
74 #[cfg(all(target_arch = "aarch64", target_os = "none"))]
75 {
76 let mut tp: usize;
77 unsafe {
79 core::arch::asm!("mrs {}, tpidr_el0", out(reg) tp, options(nomem, nostack, preserves_flags));
80 }
81 if tp != 0 {
82 return unsafe { &*(tp as *const ThreadControlBlock) };
84 }
85 }
86 &MAIN_TCB.0
87}
88
89#[must_use]
90pub fn errno_cell() -> *mut i32 {
91 current_tcb().errno.get()
92}
93
94#[must_use]
95pub fn current_thread_id() -> u64 {
96 core::ptr::from_ref(current_tcb()) as usize as u64
97}
98
99static TLS_KEY_STATE: [AtomicU8; MAX_TLS_KEYS] =
100 [const { AtomicU8::new(TLS_KEY_FREE) }; MAX_TLS_KEYS];
101static TLS_DTORS: [AtomicUsize; MAX_TLS_KEYS] = [const { AtomicUsize::new(0) }; MAX_TLS_KEYS];
102
103#[must_use]
104pub fn tls_key_create(dtor: Option<unsafe extern "C" fn(*mut u8)>) -> Option<usize> {
105 let dtor_addr = dtor.map_or(0, |f| f as usize);
106 for key in 0..MAX_TLS_KEYS {
107 if TLS_KEY_STATE[key]
108 .compare_exchange(
109 TLS_KEY_FREE,
110 TLS_KEY_IN_USE,
111 Ordering::AcqRel,
112 Ordering::Acquire,
113 )
114 .is_ok()
115 {
116 TLS_DTORS[key].store(dtor_addr, Ordering::Release);
117 unsafe { (*current_tcb().tls_values.get())[key] = null_mut() };
119 return Some(key);
120 }
121 }
122 None
123}
124
125pub fn tls_key_delete(key: usize) {
126 if key >= MAX_TLS_KEYS {
127 return;
128 }
129 TLS_DTORS[key].store(0, Ordering::Release);
130 unsafe { (*current_tcb().tls_values.get())[key] = null_mut() };
132 TLS_KEY_STATE[key].store(TLS_KEY_FREE, Ordering::Release);
133}
134
135#[must_use]
136pub fn tls_get(key: usize) -> *mut u8 {
137 if key >= MAX_TLS_KEYS || TLS_KEY_STATE[key].load(Ordering::Acquire) != TLS_KEY_IN_USE {
138 return null_mut();
139 }
140 unsafe { (*current_tcb().tls_values.get())[key] }
142}
143
144pub fn tls_set(key: usize, val: *mut u8) -> bool {
145 if key >= MAX_TLS_KEYS || TLS_KEY_STATE[key].load(Ordering::Acquire) != TLS_KEY_IN_USE {
146 return false;
147 }
148 unsafe { (*current_tcb().tls_values.get())[key] = val };
150 true
151}
152
153pub fn run_current_thread_tls_destructors() {
154 let tcb = current_tcb();
155 for key in 0..MAX_TLS_KEYS {
156 let val = unsafe { (*tcb.tls_values.get())[key] };
158 if !val.is_null() {
159 unsafe { (*tcb.tls_values.get())[key] = null_mut() };
161 if TLS_KEY_STATE[key].load(Ordering::Acquire) == TLS_KEY_IN_USE {
162 let addr = TLS_DTORS[key].load(Ordering::Acquire);
163 if addr != 0 {
164 let dtor: unsafe extern "C" fn(*mut u8) = unsafe { core::mem::transmute(addr) };
166 unsafe { dtor(val) };
168 }
169 }
170 }
171 }
172}
173
174pub struct ThreadHandle {
175 tcb: *mut ThreadControlBlock,
176}
177
178unsafe impl Send for ThreadHandle {}
180unsafe impl Sync for ThreadHandle {}
182
183impl ThreadHandle {
184 #[must_use]
185 pub fn id(&self) -> u64 {
186 self.tcb as usize as u64
187 }
188
189 #[must_use]
190 pub fn is_current(&self) -> bool {
191 current_thread_id() == self.id()
192 }
193
194 pub fn join(self) -> SysResult<()> {
195 if self.is_current() {
196 let _ = ManuallyDrop::new(self);
197 return Err(Errno(EDEADLK));
198 }
199 let this = ManuallyDrop::new(self);
200 let tcb = this.tcb;
201 unsafe {
203 loop {
204 let tid = (*tcb).child_tid.load(Ordering::Acquire);
205 if tid == 0 {
206 break;
207 }
208 crate::sync::futex_wait(&(*tcb).child_tid, tid, None);
209 }
210 if !(*tcb).stack_map_base.is_null() && (*tcb).stack_map_len > 0 {
211 let _ = munmap((*tcb).stack_map_base, (*tcb).stack_map_len);
212 }
213 c_free(tcb.cast::<u8>());
214 }
215 Ok(())
216 }
217
218 pub fn detach(self) -> SysResult<()> {
219 let mut this = ManuallyDrop::new(self);
220 this.detach_inner();
221 Ok(())
222 }
223
224 fn detach_inner(&mut self) {
225 let tcb = self.tcb;
226 if tcb.is_null() {
227 return;
228 }
229 self.tcb = null_mut();
230 unsafe {
232 if (*tcb)
233 .join_state
234 .compare_exchange(
235 JOIN_JOINABLE,
236 JOIN_DETACHED,
237 Ordering::AcqRel,
238 Ordering::Acquire,
239 )
240 .is_err()
241 {
242 loop {
243 let tid = (*tcb).child_tid.load(Ordering::Acquire);
244 if tid == 0 {
245 break;
246 }
247 crate::sync::futex_wait(&(*tcb).child_tid, tid, None);
248 }
249 if !(*tcb).stack_map_base.is_null() && (*tcb).stack_map_len > 0 {
250 let _ = munmap((*tcb).stack_map_base, (*tcb).stack_map_len);
251 }
252 c_free(tcb.cast::<u8>());
253 }
254 }
255 }
256}
257
258impl Drop for ThreadHandle {
259 fn drop(&mut self) {
260 self.detach_inner();
261 }
262}
263
264#[cfg(all(target_arch = "aarch64", target_os = "none"))]
265extern "C" fn child_thread_start(tcb_ptr: *mut ThreadControlBlock) -> ! {
266 unsafe {
268 if let Some(entry) = (*tcb_ptr).entry_fn {
269 entry((*tcb_ptr).entry_arg);
270 }
271 }
272 run_current_thread_tls_destructors();
273 unsafe {
275 if (*tcb_ptr).join_state.swap(JOIN_EXITING, Ordering::AcqRel) == JOIN_JOINABLE {
276 crate::process::exit_thread(0);
277 }
278 let map_base = (*tcb_ptr).stack_map_base as usize;
279 let map_len = (*tcb_ptr).stack_map_len;
280 core::arch::asm!(
281 "msr tpidr_el0, xzr",
282 "mov x8, {munmap_nr}",
283 "svc #0",
284 "mov x0, #0",
285 "mov x8, {exit_nr}",
286 "svc #0",
287 "brk #0",
288 munmap_nr = const crate::raw::nr::MUNMAP,
289 exit_nr = const crate::raw::nr::EXIT,
290 in("x0") map_base,
291 in("x1") map_len,
292 options(noreturn, nostack),
293 );
294 }
295}
296
297#[cfg(all(target_arch = "aarch64", target_os = "none"))]
298#[inline]
299unsafe fn raw_clone_thread(
300 flags: u64,
301 child_sp: usize,
302 parent_tid: usize,
303 tls: usize,
304 child_tid: usize,
305) -> SysResult<usize> {
306 let ret: usize;
307 unsafe {
309 core::arch::asm!(
310 "svc #0",
311 "cbnz x0, 2f",
312 "ldr x0, [sp]",
313 "add sp, sp, #16",
314 "mov x29, #0",
315 "mov x30, #0",
316 "bl {child_start}",
317 "brk #0",
318 "2:",
319 child_start = sym child_thread_start,
320 in("x8") crate::raw::nr::CLONE,
321 inlateout("x0") flags as usize => ret,
322 in("x1") child_sp,
323 in("x2") parent_tid,
324 in("x3") tls,
325 in("x4") child_tid,
326 options(nostack),
327 );
328 }
329 crate::raw::decode_raw(ret)
330}
331
332pub unsafe fn spawn_thread(
338 stack_bytes: usize,
339 entry: extern "C" fn(*mut u8),
340 arg: *mut u8,
341 detached: bool,
342) -> SysResult<Option<ThreadHandle>> {
343 let usable_stack = stack_bytes
344 .max(MIN_STACK_BYTES)
345 .checked_add(PAGE_SIZE - 1)
346 .map(|v| v & !(PAGE_SIZE - 1))
347 .ok_or(Errno(EINVAL))?;
348 let total_map = usable_stack.checked_add(PAGE_SIZE).ok_or(Errno(EINVAL))?;
349 let map_base = mmap_anonymous(total_map, PROT_READ | PROT_WRITE, MAP_STACK)?;
350 if let Err(e) = unsafe { mprotect(map_base, PAGE_SIZE, PROT_NONE) } {
352 let _ = unsafe { munmap(map_base, total_map) };
354 return Err(e);
355 }
356
357 let raw_tcb = unsafe { GLOBAL_MMAP_HEAP.alloc_c(size_of::<ThreadControlBlock>(), 64) };
359 if raw_tcb.is_null() {
360 let _ = unsafe { munmap(map_base, total_map) };
362 return Err(Errno(ENOMEM));
363 }
364 let tcb = raw_tcb.cast::<ThreadControlBlock>();
365 unsafe {
367 tcb.write(ThreadControlBlock::new());
368 (*tcb)
369 .child_tid
370 .store(if detached { 0 } else { u32::MAX }, Ordering::Relaxed);
371 (*tcb).join_state.store(
372 if detached {
373 JOIN_DETACHED
374 } else {
375 JOIN_JOINABLE
376 },
377 Ordering::Relaxed,
378 );
379 (*tcb).stack_map_base = map_base;
380 (*tcb).stack_map_len = total_map;
381 (*tcb).entry_fn = Some(entry);
382 (*tcb).entry_arg = arg;
383 }
384
385 let child_sp = (((map_base as usize) + total_map) & !15usize) - 16;
386 unsafe {
388 let frame = child_sp as *mut usize;
389 frame.write(tcb as usize);
390 frame.add(1).write(0);
391 }
392
393 let mut flags = CLONE_VM
394 | CLONE_FS
395 | CLONE_FILES
396 | CLONE_SIGHAND
397 | CLONE_THREAD
398 | CLONE_SYSVSEM
399 | CLONE_SETTLS;
400 let (parent_tid_ptr, child_tid_ptr) = if detached {
401 (0usize, 0usize)
402 } else {
403 flags |= CLONE_PARENT_SETTID | CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID;
404 let tid_addr = unsafe { core::ptr::addr_of!((*tcb).child_tid) as usize };
406 (tid_addr, tid_addr)
407 };
408
409 #[cfg(all(target_arch = "aarch64", target_os = "none"))]
410 let clone_res =
412 unsafe { raw_clone_thread(flags, child_sp, parent_tid_ptr, tcb as usize, child_tid_ptr) };
413 #[cfg(not(all(target_arch = "aarch64", target_os = "none")))]
414 let clone_res = unsafe {
416 crate::raw::syscall5(
417 crate::raw::nr::CLONE,
418 flags as usize,
419 child_sp,
420 parent_tid_ptr,
421 tcb as usize,
422 child_tid_ptr,
423 )
424 };
425
426 match clone_res {
427 Ok(_) if detached => {
428 #[cfg(not(all(target_arch = "aarch64", target_os = "none")))]
429 unsafe {
431 let _ = munmap(map_base, total_map);
432 c_free(tcb.cast::<u8>());
433 }
434 Ok(None)
435 }
436 Ok(_) => Ok(Some(ThreadHandle { tcb })),
437 Err(e) => {
438 unsafe {
440 let _ = munmap(map_base, total_map);
441 c_free(tcb.cast::<u8>());
442 }
443 Err(e)
444 }
445 }
446}
447
448#[cfg(test)]
449mod tests {
450 use super::*;
451 use crate::errno::EAGAIN;
452 use crate::raw::{nr, with_mock_syscall};
453 use core::sync::atomic::AtomicBool;
454
455 #[test]
456 fn errno_and_tls_destructors() {
457 assert_ne!(current_thread_id(), 0);
458 let ptr = errno_cell();
459 unsafe {
460 let prev = *ptr;
461 *ptr = 42;
462 assert_eq!(*errno_cell(), 42);
463 *ptr = prev;
464 }
465 static DTOR_CALLED: AtomicUsize = AtomicUsize::new(0);
466 unsafe extern "C" fn dtor(val: *mut u8) {
467 DTOR_CALLED.store(val as usize, Ordering::SeqCst);
468 }
469 let k = tls_key_create(Some(dtor)).expect("tls key");
470 assert!(tls_set(k, core::ptr::dangling_mut::<u8>()));
471 assert!(!tls_get(k).is_null());
472 run_current_thread_tls_destructors();
473 assert_ne!(DTOR_CALLED.load(Ordering::SeqCst), 0);
474 assert!(tls_get(k).is_null());
475 tls_key_delete(k);
476 }
477
478 #[test]
479 fn spawn_join_and_clone_error_cleanup() {
480 #[repr(align(4096))]
481 struct AlignedArena([u8; 256 * 1024]);
482
483 extern "C" fn dummy(_: *mut u8) {}
484 static mut ARENA: AlignedArena = AlignedArena([0; 256 * 1024]);
485 static OFFSET: AtomicUsize = AtomicUsize::new(0);
486 static FAIL_CLONE: AtomicBool = AtomicBool::new(false);
487 static MUNMAPS: AtomicUsize = AtomicUsize::new(0);
488
489 OFFSET.store(0, Ordering::SeqCst);
490 FAIL_CLONE.store(false, Ordering::SeqCst);
491 MUNMAPS.store(0, Ordering::SeqCst);
492
493 with_mock_syscall(
494 |sys_nr, args| match sys_nr {
495 nr::MMAP => {
496 let len = (args[1] + 4095) & !4095;
497 let off = OFFSET.fetch_add(len, Ordering::SeqCst);
498 if off + len > 256 * 1024 {
499 return Err(Errno(ENOMEM));
500 }
501 let ptr = unsafe { core::ptr::addr_of_mut!(ARENA.0).cast::<u8>().add(off) };
503 Ok(ptr as usize)
504 }
505 nr::MPROTECT => Ok(0),
506 nr::MUNMAP => {
507 MUNMAPS.fetch_add(1, Ordering::SeqCst);
508 Ok(0)
509 }
510 nr::CLONE if FAIL_CLONE.load(Ordering::SeqCst) => Err(Errno(EAGAIN)),
511 nr::CLONE => {
512 let tid = args[4] as *const AtomicU32;
513 if !tid.is_null() {
514 unsafe { (*tid).store(0, Ordering::Release) };
515 }
516 Ok(777)
517 }
518 _ => Err(Errno(crate::errno::ENOSYS)),
519 },
520 || {
521 let h = unsafe { spawn_thread(MIN_STACK_BYTES, dummy, null_mut(), false) }
522 .expect("spawn")
523 .expect("handle");
524 let before = MUNMAPS.load(Ordering::SeqCst);
525 h.join().expect("join");
526 assert!(MUNMAPS.load(Ordering::SeqCst) > before);
527
528 FAIL_CLONE.store(true, Ordering::SeqCst);
529 let before_fail = MUNMAPS.load(Ordering::SeqCst);
530 let err = unsafe { spawn_thread(MIN_STACK_BYTES, dummy, null_mut(), false) };
531 assert_eq!(err.err(), Some(Errno(EAGAIN)));
532 assert!(MUNMAPS.load(Ordering::SeqCst) > before_fail);
533 },
534 );
535 }
536}