kh_runtime/thread.rs
1//! Guest thread support: host pthread spawn + `bsdthread_*` helpers.
2//!
3//! Model (micro / clean-room):
4//! - `bsdthread_register` stores the libpthread start trampoline VA.
5//! - `bsdthread_create` spawns a **host** thread that jumps into that trampoline
6//! with Darwin `_pthread_start` register convention (`x0`–`x3`).
7//! - `bsdthread_terminate` switches back to a saved **host** stack then ends
8//! this host thread only (never exit on the guest stack).
9//! - Guest `pthread_join` completion (`done` + futex wake) is published from
10//! the host stack so joiners never `munmap` a stack still in use.
11//! - Worker teardown uses raw `SYS_exit` (not `pthread_exit`): glibc
12//! `pthread_exit` runs `_Unwind_ForcedUnwind`, which walks `x29` into guest /
13//! hypercall frames and SEGV's in `libgcc_s` under 7zz MT.
14//! - Live spawn requires **Linux aarch64** (same as the trap backend).
15
16#![allow(unsafe_code)]
17
18use std::sync::atomic::{AtomicU64, Ordering};
19
20use crate::host_slot;
21
22/// Monotonic guest-visible thread id counter (`thread_selfid`).
23static NEXT_TID: AtomicU64 = AtomicU64::new(1);
24
25/// Size of the per-thread host hypercall stack (grows down from the top).
26#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
27const HOST_ALT_STACK_SIZE: usize = 512 * 1024;
28
29/// Returns a 16-byte-aligned host SP for freestanding hypercall dispatch.
30///
31/// Prefer calling [`ensure_host_alt_stack`] while host `TPIDR_EL0` is live
32/// (worker prepare / main TLS install). The hypercall entry also restores host
33/// TLS before invoking this so a cold first map is safe.
34///
35/// Uses [`host_slot`] — never Rust `thread_local!`.
36///
37/// # Safety
38///
39/// Caller must restore guest SP after the call returns. The buffer is
40/// valid until [`drop_host_alt_stack`] / process exit.
41#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
42#[unsafe(no_mangle)]
43pub unsafe extern "C" fn kh_host_alt_sp() -> u64 {
44 ensure_host_alt_stack()
45}
46
47/// Map this OS thread's hypercall alt stack if missing; return its top SP.
48///
49/// Safe to call repeatedly. Must run with **host** `TPIDR_EL0` for the first
50/// map (glibc `mmap` / map mutex).
51#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
52#[must_use]
53pub fn ensure_host_alt_stack() -> u64 {
54 let top = host_slot::with_alt_mut(|cell| {
55 if let Some(s) = *cell {
56 return s.top;
57 }
58 match map_host_alt_stack() {
59 Some(s) => {
60 let t = s.top;
61 *cell = Some(s);
62 t
63 }
64 None => 0,
65 }
66 });
67 // A1: keep guest-TLS alt_top mirror current for gettid-free enter.
68 if top != 0 {
69 crate::tls::publish_alt_top_to_current_guest(top);
70 }
71 top
72}
73
74#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
75#[must_use]
76pub fn ensure_host_alt_stack() -> u64 {
77 0
78}
79
80#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
81fn map_host_alt_stack() -> Option<host_slot::AltStackSlot> {
82 let len = HOST_ALT_STACK_SIZE;
83 let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
84 let prot = libc::PROT_READ | libc::PROT_WRITE;
85 // Prefer raw syscall so a mistaken guest-TPIDR call does not enter glibc
86 // arena code that depends on host TLS.
87 let ptr = mmap_anon_raw(len, prot, flags)?;
88 let base_u = crate::host::ptr_addr_u64(ptr);
89 let top = base_u.saturating_add(u64::try_from(len).unwrap_or(0)) & !0xF;
90 if top <= base_u {
91 let _ = crate::host::munmap(ptr, len);
92 return None;
93 }
94 Some(host_slot::AltStackSlot {
95 base: ptr,
96 len,
97 top,
98 })
99}
100
101/// Anonymous `mmap` via raw syscall (no glibc TLS dependency).
102#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
103fn mmap_anon_raw(len: usize, prot: libc::c_int, flags: libc::c_int) -> Option<*mut u8> {
104 // SAFETY: anonymous map; fd/offset unused with MAP_ANONYMOUS.
105 let raw = unsafe {
106 libc::syscall(
107 libc::SYS_mmap,
108 core::ptr::null_mut::<libc::c_void>(),
109 len,
110 prot,
111 flags,
112 -1_i32,
113 0_i64,
114 )
115 };
116 if raw < 0 {
117 return None;
118 }
119 let addr = usize::try_from(raw).ok()?;
120 Some(core::ptr::with_exposed_provenance_mut(addr))
121}
122
123/// Unmaps this thread's hypercall alt stack (worker exit / tests).
124#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
125pub(crate) fn drop_host_alt_stack() {
126 host_slot::with_alt_mut(|cell| {
127 if let Some(s) = cell.take() {
128 let _ = crate::host::munmap(s.base, s.len);
129 }
130 });
131}
132
133#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
134#[allow(dead_code)] // Linux aarch64 only; stub keeps call sites compiling.
135pub(crate) fn drop_host_alt_stack() {}
136
137/// Live guest worker threads (not including the main `LC_MAIN` thread).
138static LIVE_WORKERS: AtomicU64 = AtomicU64::new(0);
139
140/// Darwin `EAGAIN`.
141#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
142const EAGAIN: i64 = 35;
143
144/// Resets thread counters for a new guest run.
145pub fn reset_thread_runtime() {
146 NEXT_TID.store(1, Ordering::SeqCst);
147 LIVE_WORKERS.store(0, Ordering::SeqCst);
148}
149
150/// Allocates a new guest thread id (unique within the process run).
151#[must_use]
152pub fn alloc_tid() -> u64 {
153 NEXT_TID.fetch_add(1, Ordering::SeqCst)
154}
155
156/// Current host thread's guest tid (stable for the host thread lifetime).
157///
158/// Uses [`host_slot`] so it remains safe under guest `TPIDR_EL0`.
159#[must_use]
160pub fn thread_selfid() -> u64 {
161 host_slot::with_worker_mut(|w| {
162 if w.guest_tid == 0 {
163 w.guest_tid = alloc_tid();
164 }
165 w.guest_tid
166 })
167}
168
169/// Number of live worker threads spawned via `bsdthread_create`.
170#[must_use]
171pub fn live_workers() -> u64 {
172 LIVE_WORKERS.load(Ordering::SeqCst)
173}
174
175/// Arguments for a newly created guest worker.
176#[derive(Debug, Clone, Copy)]
177pub struct GuestThreadStart {
178 /// Registered `threadstart` trampoline VA.
179 pub entry: u64,
180 /// Guest stack pointer (16-byte aligned top of stack).
181 pub sp: u64,
182 /// `x0` — pthread structure VA.
183 pub pthread: u64,
184 /// `x1` — Mach thread port (stubbed to 0).
185 pub port: u64,
186 /// `x2` — user start routine VA.
187 pub func: u64,
188 /// `x3` — user argument.
189 pub func_arg: u64,
190}
191
192/// Spawns a host thread that enters guest code at `start.entry`.
193pub fn spawn_guest_thread(start: GuestThreadStart) -> Result<(), i64> {
194 #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
195 {
196 let _ = start;
197 Err(crate::syscall::ENOSYS)
198 }
199
200 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
201 {
202 linux_spawn(start)
203 }
204}
205
206#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
207mod linux {
208 use super::{EAGAIN, GuestThreadStart, LIVE_WORKERS, thread_selfid};
209 use crate::host_slot;
210 use std::sync::atomic::{AtomicU32, Ordering};
211
212 /// Must match freestanding `kh-libsystem` `KhThread` / `MAGIC`.
213 ///
214 /// Layout (`repr(C, align(16))` on aarch64):
215 /// ```text
216 /// offset 0: magic: u64
217 /// offset 8: done: AtomicU32 ← join wait / host publish
218 /// offset 12: detached: AtomicU32
219 /// offset 16: result: AtomicUsize
220 /// … stack fields follow; host only touches magic + done.
221 /// offset 56: tsd
222 /// ```
223 const KH_THREAD_MAGIC: u64 = 0x4B48_5054_4852_4401;
224 const KH_THREAD_DONE_OFF: usize = 8;
225
226 pub(super) fn linux_spawn(start: GuestThreadStart) -> Result<(), i64> {
227 if start.entry == 0 || start.sp == 0 || !start.sp.is_multiple_of(16) {
228 return Err(crate::syscall::EINVAL);
229 }
230
231 let builder = std::thread::Builder::new().name("kh-guest".into());
232 match builder.spawn(move || guest_worker_main(start)) {
233 Ok(_handle) => {
234 LIVE_WORKERS.fetch_add(1, Ordering::SeqCst);
235 Ok(())
236 }
237 Err(_) => Err(EAGAIN),
238 }
239 }
240
241 fn guest_worker_main(start: GuestThreadStart) {
242 let _ = thread_selfid();
243 host_slot::with_worker_mut(|w| {
244 w.guest_pthread = start.pthread;
245 });
246
247 // Capture host TPIDR *before* guest code may msr guest TSD into TPIDR_EL0.
248 crate::tls::prepare_host_meta();
249 // Map hypercall alt stack while host TLS is still live (workers hit
250 // freestanding hypercall immediately under 7zz `-mmt>1`).
251 let _ = super::ensure_host_alt_stack();
252
253 // `bsdthread_create` runs inside the SIGTRAP handler, where SIGTRAP is
254 // blocked. `std::thread::spawn` copies that mask, so the worker would
255 // die on the first guest `brk` unless we unblock here.
256 unblock_sigtrap();
257
258 let host_sp = crate::cpu::read_sp();
259 // Leave room for host exit / join-publish frames (not just 512 B).
260 let frame_sp = host_sp.saturating_sub(4096) & !0xF;
261 set_host_exit_frame(host_thread_exit_pc(), frame_sp);
262
263 // Prefer freestanding KhThread.tsd (offset 56) when present; else guest
264 // trampoline will install TLS itself and boundary will mrs it.
265 if let Some(tsd) = guest_tsd_from_pthread(start.pthread) {
266 crate::tls::enter_guest_tls(tsd);
267 }
268
269 // SAFETY: trap handlers installed by `run_micro` before create; entry/stack
270 // are guest `bsdthread_create` arguments (mapped stack region).
271 let result = unsafe {
272 crate::entry::jump_to_guest_args(
273 start.entry,
274 start.sp,
275 start.pthread,
276 start.port,
277 start.func,
278 start.func_arg,
279 )
280 };
281 if let Err(err) = result {
282 tracing::error!(?err, "guest worker failed to enter");
283 }
284 // jump_to_guest is noreturn on success; if we return, publish join + exit.
285 finish_worker_on_host();
286 }
287
288 /// Freestanding `KhThread` layout: `tsd: *mut GuestTls` at offset 56.
289 const KH_THREAD_TSD_OFF: usize = 56;
290
291 fn guest_tsd_from_pthread(pthread: u64) -> Option<u64> {
292 let base = usize::try_from(pthread).ok()?;
293 if base == 0 {
294 return None;
295 }
296 // SAFETY: identity-mapped freestanding KhThread from pthread_create.
297 let magic_ptr = std::ptr::with_exposed_provenance::<u64>(base);
298 let magic = unsafe { core::ptr::read_volatile(magic_ptr) };
299 if magic != KH_THREAD_MAGIC {
300 return None;
301 }
302 let tsd_ptr =
303 std::ptr::with_exposed_provenance::<u64>(base.saturating_add(KH_THREAD_TSD_OFF));
304 let tsd = unsafe { core::ptr::read_volatile(tsd_ptr) };
305 if tsd == 0 {
306 return None;
307 }
308 Some(tsd)
309 }
310
311 fn unblock_sigtrap() {
312 // SAFETY: only clears the per-thread SIGTRAP block bit inherited from
313 // the spawning signal handler; disposition is already installed.
314 unsafe {
315 let mut set: libc::sigset_t = std::mem::zeroed();
316 libc::sigemptyset(std::ptr::addr_of_mut!(set));
317 libc::sigaddset(std::ptr::addr_of_mut!(set), libc::SIGTRAP);
318 libc::pthread_sigmask(
319 libc::SIG_UNBLOCK,
320 std::ptr::addr_of!(set),
321 std::ptr::null_mut(),
322 );
323 }
324 }
325
326 #[allow(clippy::as_conversions, function_casts_as_integer)]
327 fn host_thread_exit_pc() -> u64 {
328 u64::try_from(host_thread_exit as usize).unwrap_or(0)
329 }
330
331 /// Landing pad on the **host** stack after `bsdthread_terminate`.
332 unsafe extern "C" fn host_thread_exit() -> ! {
333 finish_worker_on_host();
334 }
335
336 /// Publish guest join state, drop live-worker count, end this host thread.
337 ///
338 /// Must run with SP on the host stack (never on the guest worker stack that
339 /// `pthread_join` may munmap once `done` is visible).
340 fn finish_worker_on_host() -> ! {
341 clear_host_exit_frame();
342 // Host TLS before any further host libc (join publish, munmap, exit).
343 crate::tls::clear_guest_tls_on_exit();
344 publish_guest_join_done();
345 host_slot::with_worker_mut(|w| {
346 w.guest_pthread = 0;
347 });
348 super::drop_host_alt_stack();
349 host_slot::clear_current();
350 worker_finished();
351 // End **this** OS thread only. Do **not** call `pthread_exit`:
352 // glibc runs `_Unwind_ForcedUnwind` for cleanup handlers; after a
353 // hypercall/guest jump the FP chain still points into guest memory
354 // and the DWARF walker SEGV's in `libgcc_s` (seen as pc=libgcc+0xe320
355 // under 7zz `-mmt>1`). Raw `SYS_exit` skips forced unwind.
356 // SAFETY: Linux `exit` (not `exit_group`) terminates only the caller.
357 unsafe {
358 libc::syscall(libc::SYS_exit, 0);
359 }
360 // syscall exit is noreturn; keep the type checker happy.
361 loop {
362 core::hint::spin_loop();
363 }
364 }
365
366 /// Set freestanding `KhThread.done` and futex-wake joiners.
367 ///
368 /// Guest `kh_pthread_start` stores `result` before `bsdthread_terminate` but
369 /// **must not** set `done` early: that allowed `pthread_join` to munmap the
370 /// guest stack while hypercall terminate / Rust still ran on it (intermittent
371 /// SEGV under 7zz `-mmt>1` with freestanding hypercall).
372 fn publish_guest_join_done() {
373 let pthread = host_slot::worker_get().guest_pthread;
374 if pthread == 0 {
375 return;
376 }
377 let base = usize::try_from(pthread).unwrap_or(0);
378 if base == 0 {
379 return;
380 }
381 // SAFETY: identity-mapped guest heap block from freestanding pthread_create.
382 let magic_ptr = std::ptr::with_exposed_provenance::<u64>(base);
383 let magic = unsafe { core::ptr::read_volatile(magic_ptr) };
384 if magic != KH_THREAD_MAGIC {
385 tracing::warn!(
386 pthread = format_args!("{pthread:#x}"),
387 magic = format_args!("{magic:#x}"),
388 "guest pthread magic mismatch; skip join publish"
389 );
390 return;
391 }
392 let done_addr = base.saturating_add(KH_THREAD_DONE_OFF);
393 let done_ptr = std::ptr::with_exposed_provenance_mut::<AtomicU32>(done_addr);
394 // SAFETY: aligned AtomicU32 at known freestanding layout offset.
395 unsafe {
396 (*done_ptr).store(1, Ordering::Release);
397 }
398 futex_wake(done_ptr.cast::<u32>(), i32::MAX);
399 }
400
401 fn futex_wake(addr: *mut u32, n: i32) {
402 // SAFETY: same identity-mapped word guests park on via KH_HELPER_PARK.
403 // FUTEX_WAKE_PRIVATE (129) must match `helpers::wake_u32` (park uses PRIVATE).
404 let _ = unsafe {
405 libc::syscall(
406 libc::SYS_futex,
407 addr,
408 129_i32, // FUTEX_WAKE_PRIVATE
409 n,
410 core::ptr::null::<libc::timespec>(),
411 core::ptr::null_mut::<u32>(),
412 0_i32,
413 )
414 };
415 }
416
417 fn worker_finished() {
418 let _ = LIVE_WORKERS.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| {
419 Some(n.saturating_sub(1))
420 });
421 }
422
423 /// Hypercall / trap path: leave the guest stack, then end this host worker.
424 ///
425 /// Switching SP (and clearing FP) before teardown is mandatory: freestanding
426 /// hypercall may have left `x29` pointing at a guest frame that joiners
427 /// munmap once `done` is published; forced unwind must never see that chain.
428 pub(crate) fn exit_worker_now() -> ! {
429 let (pc, sp, has) = host_slot::with_worker_mut(|w| {
430 let has = w.has_exit;
431 let pc = w.exit_pc;
432 let sp = w.exit_sp;
433 w.has_exit = false;
434 (pc, sp, has)
435 });
436 if has {
437 // SAFETY: frame was recorded on this host thread before guest entry;
438 // `pc` is `host_thread_exit`, `sp` is 16-byte aligned host stack.
439 // Zero `x29` so any accidental libc unwind cannot follow guest FP.
440 unsafe {
441 std::arch::asm!(
442 "mov sp, {sp}",
443 "mov x29, xzr",
444 "br {pc}",
445 sp = in(reg) sp,
446 pc = in(reg) pc,
447 options(noreturn),
448 );
449 }
450 }
451 // No frame (should not happen for workers): best-effort host exit.
452 finish_worker_on_host();
453 }
454
455 fn set_host_exit_frame(pc: u64, sp: u64) {
456 host_slot::with_worker_mut(|w| {
457 w.exit_pc = pc;
458 w.exit_sp = sp;
459 w.has_exit = true;
460 });
461 }
462
463 fn clear_host_exit_frame() {
464 host_slot::with_worker_mut(|w| {
465 w.has_exit = false;
466 });
467 }
468
469 /// Rewrite `mcontext` so sigreturn jumps to the host worker-exit trampoline.
470 pub(crate) fn redirect_ucontext_to_host_exit(m: &mut libc::mcontext_t) -> bool {
471 host_slot::with_worker_mut(|w| {
472 if !w.has_exit {
473 return false;
474 }
475 // Consume so a second terminate does not reuse a stale frame.
476 w.has_exit = false;
477 m.pc = w.exit_pc;
478 m.sp = w.exit_sp;
479 // Drop guest FP chain (same rationale as `exit_worker_now`).
480 m.regs[29] = 0;
481 true
482 })
483 }
484}
485
486#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
487use linux::linux_spawn;
488
489/// Rewrite `mcontext` so sigreturn jumps to the host worker-exit trampoline.
490///
491/// Returns `false` when no frame is installed (main thread / non-worker).
492#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
493pub(crate) use linux::redirect_ucontext_to_host_exit;
494
495/// End the current guest worker from hypercall/trap (not signal ucontext).
496///
497/// Switches to the saved host stack, publishes join `done`, then ends the
498/// host OS thread (`SYS_exit` — see `finish_worker_on_host`).
499#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
500pub(crate) fn exit_current_guest_worker() -> ! {
501 linux::exit_worker_now();
502}
503
504#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
505#[allow(dead_code)] // live only via Linux aarch64 trap/hypercall path
506pub(crate) fn redirect_ucontext_to_host_exit() -> bool {
507 false
508}
509
510#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
511#[allow(dead_code)] // live only via Linux aarch64 trap/hypercall path
512pub(crate) fn exit_current_guest_worker() -> ! {
513 loop {
514 core::hint::spin_loop();
515 }
516}
517
518#[cfg(test)]
519mod tests {
520 use super::*;
521
522 #[test]
523 fn selfid_stable_on_thread() {
524 let a = thread_selfid();
525 let b = thread_selfid();
526 assert_eq!(a, b);
527 assert!(a > 0);
528 }
529
530 #[test]
531 fn alloc_tid_monotonic() {
532 let a = alloc_tid();
533 let b = alloc_tid();
534 assert!(b > a);
535 }
536}