Skip to main content

kh_runtime/
thread.rs

1//! Guest thread support: host pthread spawn + `bsdthread_*` helpers.
2//!
3//! Model (micro / clean-room):
4//! - `bsdthread_register` stores the libpthread start trampoline VA.
5//! - `bsdthread_create` spawns a **host** thread that jumps into that trampoline
6//!   with Darwin `_pthread_start` register convention (`x0`–`x3`).
7//! - `bsdthread_terminate` switches back to a saved **host** stack then ends
8//!   this host thread only (never exit on the guest stack).
9//! - Guest `pthread_join` completion (`done` + futex wake) is published from
10//!   the host stack so joiners never `munmap` a stack still in use.
11//! - Worker teardown uses raw `SYS_exit` (not `pthread_exit`): glibc
12//!   `pthread_exit` runs `_Unwind_ForcedUnwind`, which walks `x29` into guest /
13//!   hypercall frames and SEGV's in `libgcc_s` under 7zz MT.
14//! - Live spawn requires **Linux aarch64** (same as the trap backend).
15
16#![allow(unsafe_code)]
17
18use std::sync::atomic::{AtomicU64, Ordering};
19
20use crate::host_slot;
21
22/// Monotonic guest-visible thread id counter (`thread_selfid`).
23static NEXT_TID: AtomicU64 = AtomicU64::new(1);
24
25/// Size of the per-thread host hypercall stack (grows down from the top).
26#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
27const HOST_ALT_STACK_SIZE: usize = 512 * 1024;
28
29/// Returns a 16-byte-aligned host SP for freestanding hypercall dispatch.
30///
31/// Prefer calling [`ensure_host_alt_stack`] while host `TPIDR_EL0` is live
32/// (worker prepare / main TLS install). The hypercall entry also restores host
33/// TLS before invoking this so a cold first map is safe.
34///
35/// Uses [`host_slot`] — never Rust `thread_local!`.
36///
37/// # Safety
38///
39/// Caller must restore guest SP after the call returns. The buffer is
40/// valid until [`drop_host_alt_stack`] / process exit.
41#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
42#[unsafe(no_mangle)]
43pub unsafe extern "C" fn kh_host_alt_sp() -> u64 {
44    ensure_host_alt_stack()
45}
46
47/// Map this OS thread's hypercall alt stack if missing; return its top SP.
48///
49/// Safe to call repeatedly. Must run with **host** `TPIDR_EL0` for the first
50/// map (glibc `mmap` / map mutex).
51#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
52#[must_use]
53pub fn ensure_host_alt_stack() -> u64 {
54    let top = host_slot::with_alt_mut(|cell| {
55        if let Some(s) = *cell {
56            return s.top;
57        }
58        match map_host_alt_stack() {
59            Some(s) => {
60                let t = s.top;
61                *cell = Some(s);
62                t
63            }
64            None => 0,
65        }
66    });
67    // A1: keep guest-TLS alt_top mirror current for gettid-free enter.
68    if top != 0 {
69        crate::tls::publish_alt_top_to_current_guest(top);
70    }
71    top
72}
73
74#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
75#[must_use]
76pub fn ensure_host_alt_stack() -> u64 {
77    0
78}
79
80#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
81fn map_host_alt_stack() -> Option<host_slot::AltStackSlot> {
82    let len = HOST_ALT_STACK_SIZE;
83    let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
84    let prot = libc::PROT_READ | libc::PROT_WRITE;
85    // Prefer raw syscall so a mistaken guest-TPIDR call does not enter glibc
86    // arena code that depends on host TLS.
87    let ptr = mmap_anon_raw(len, prot, flags)?;
88    let base_u = crate::host::ptr_addr_u64(ptr);
89    let top = base_u.saturating_add(u64::try_from(len).unwrap_or(0)) & !0xF;
90    if top <= base_u {
91        let _ = crate::host::munmap(ptr, len);
92        return None;
93    }
94    Some(host_slot::AltStackSlot {
95        base: ptr,
96        len,
97        top,
98    })
99}
100
101/// Anonymous `mmap` via raw syscall (no glibc TLS dependency).
102#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
103fn mmap_anon_raw(len: usize, prot: libc::c_int, flags: libc::c_int) -> Option<*mut u8> {
104    // SAFETY: anonymous map; fd/offset unused with MAP_ANONYMOUS.
105    let raw = unsafe {
106        libc::syscall(
107            libc::SYS_mmap,
108            core::ptr::null_mut::<libc::c_void>(),
109            len,
110            prot,
111            flags,
112            -1_i32,
113            0_i64,
114        )
115    };
116    if raw < 0 {
117        return None;
118    }
119    let addr = usize::try_from(raw).ok()?;
120    Some(core::ptr::with_exposed_provenance_mut(addr))
121}
122
123/// Unmaps this thread's hypercall alt stack (worker exit / tests).
124#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
125pub(crate) fn drop_host_alt_stack() {
126    host_slot::with_alt_mut(|cell| {
127        if let Some(s) = cell.take() {
128            let _ = crate::host::munmap(s.base, s.len);
129        }
130    });
131}
132
133#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
134#[allow(dead_code)] // Linux aarch64 only; stub keeps call sites compiling.
135pub(crate) fn drop_host_alt_stack() {}
136
137/// Live guest worker threads (not including the main `LC_MAIN` thread).
138static LIVE_WORKERS: AtomicU64 = AtomicU64::new(0);
139
140/// Darwin `EAGAIN`.
141#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
142const EAGAIN: i64 = 35;
143
144/// Resets thread counters for a new guest run.
145pub fn reset_thread_runtime() {
146    NEXT_TID.store(1, Ordering::SeqCst);
147    LIVE_WORKERS.store(0, Ordering::SeqCst);
148}
149
150/// Allocates a new guest thread id (unique within the process run).
151#[must_use]
152pub fn alloc_tid() -> u64 {
153    NEXT_TID.fetch_add(1, Ordering::SeqCst)
154}
155
156/// Current host thread's guest tid (stable for the host thread lifetime).
157///
158/// Uses [`host_slot`] so it remains safe under guest `TPIDR_EL0`.
159#[must_use]
160pub fn thread_selfid() -> u64 {
161    host_slot::with_worker_mut(|w| {
162        if w.guest_tid == 0 {
163            w.guest_tid = alloc_tid();
164        }
165        w.guest_tid
166    })
167}
168
169/// Number of live worker threads spawned via `bsdthread_create`.
170#[must_use]
171pub fn live_workers() -> u64 {
172    LIVE_WORKERS.load(Ordering::SeqCst)
173}
174
175/// Arguments for a newly created guest worker.
176#[derive(Debug, Clone, Copy)]
177pub struct GuestThreadStart {
178    /// Registered `threadstart` trampoline VA.
179    pub entry: u64,
180    /// Guest stack pointer (16-byte aligned top of stack).
181    pub sp: u64,
182    /// `x0` — pthread structure VA.
183    pub pthread: u64,
184    /// `x1` — Mach thread port (stubbed to 0).
185    pub port: u64,
186    /// `x2` — user start routine VA.
187    pub func: u64,
188    /// `x3` — user argument.
189    pub func_arg: u64,
190}
191
192/// Spawns a host thread that enters guest code at `start.entry`.
193pub fn spawn_guest_thread(start: GuestThreadStart) -> Result<(), i64> {
194    #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
195    {
196        let _ = start;
197        Err(crate::syscall::ENOSYS)
198    }
199
200    #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
201    {
202        linux_spawn(start)
203    }
204}
205
206#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
207mod linux {
208    use super::{EAGAIN, GuestThreadStart, LIVE_WORKERS, thread_selfid};
209    use crate::host_slot;
210    use std::sync::atomic::{AtomicU32, Ordering};
211
212    /// Must match freestanding `kh-libsystem` `KhThread` / `MAGIC`.
213    ///
214    /// Layout (`repr(C, align(16))` on aarch64):
215    /// ```text
216    /// offset 0:  magic: u64
217    /// offset 8:  done: AtomicU32   ← join wait / host publish
218    /// offset 12: detached: AtomicU32
219    /// offset 16: result: AtomicUsize
220    /// … stack fields follow; host only touches magic + done.
221    /// offset 56: tsd
222    /// ```
223    const KH_THREAD_MAGIC: u64 = 0x4B48_5054_4852_4401;
224    const KH_THREAD_DONE_OFF: usize = 8;
225
226    pub(super) fn linux_spawn(start: GuestThreadStart) -> Result<(), i64> {
227        if start.entry == 0 || start.sp == 0 || !start.sp.is_multiple_of(16) {
228            return Err(crate::syscall::EINVAL);
229        }
230
231        let builder = std::thread::Builder::new().name("kh-guest".into());
232        match builder.spawn(move || guest_worker_main(start)) {
233            Ok(_handle) => {
234                LIVE_WORKERS.fetch_add(1, Ordering::SeqCst);
235                Ok(())
236            }
237            Err(_) => Err(EAGAIN),
238        }
239    }
240
241    fn guest_worker_main(start: GuestThreadStart) {
242        let _ = thread_selfid();
243        host_slot::with_worker_mut(|w| {
244            w.guest_pthread = start.pthread;
245        });
246
247        // Capture host TPIDR *before* guest code may msr guest TSD into TPIDR_EL0.
248        crate::tls::prepare_host_meta();
249        // Map hypercall alt stack while host TLS is still live (workers hit
250        // freestanding hypercall immediately under 7zz `-mmt>1`).
251        let _ = super::ensure_host_alt_stack();
252
253        // `bsdthread_create` runs inside the SIGTRAP handler, where SIGTRAP is
254        // blocked. `std::thread::spawn` copies that mask, so the worker would
255        // die on the first guest `brk` unless we unblock here.
256        unblock_sigtrap();
257
258        let host_sp = crate::cpu::read_sp();
259        // Leave room for host exit / join-publish frames (not just 512 B).
260        let frame_sp = host_sp.saturating_sub(4096) & !0xF;
261        set_host_exit_frame(host_thread_exit_pc(), frame_sp);
262
263        // Prefer freestanding KhThread.tsd (offset 56) when present; else guest
264        // trampoline will install TLS itself and boundary will mrs it.
265        if let Some(tsd) = guest_tsd_from_pthread(start.pthread) {
266            crate::tls::enter_guest_tls(tsd);
267        }
268
269        // SAFETY: trap handlers installed by `run_micro` before create; entry/stack
270        // are guest `bsdthread_create` arguments (mapped stack region).
271        let result = unsafe {
272            crate::entry::jump_to_guest_args(
273                start.entry,
274                start.sp,
275                start.pthread,
276                start.port,
277                start.func,
278                start.func_arg,
279            )
280        };
281        if let Err(err) = result {
282            tracing::error!(?err, "guest worker failed to enter");
283        }
284        // jump_to_guest is noreturn on success; if we return, publish join + exit.
285        finish_worker_on_host();
286    }
287
288    /// Freestanding `KhThread` layout: `tsd: *mut GuestTls` at offset 56.
289    const KH_THREAD_TSD_OFF: usize = 56;
290
291    fn guest_tsd_from_pthread(pthread: u64) -> Option<u64> {
292        let base = usize::try_from(pthread).ok()?;
293        if base == 0 {
294            return None;
295        }
296        // SAFETY: identity-mapped freestanding KhThread from pthread_create.
297        let magic_ptr = std::ptr::with_exposed_provenance::<u64>(base);
298        let magic = unsafe { core::ptr::read_volatile(magic_ptr) };
299        if magic != KH_THREAD_MAGIC {
300            return None;
301        }
302        let tsd_ptr =
303            std::ptr::with_exposed_provenance::<u64>(base.saturating_add(KH_THREAD_TSD_OFF));
304        let tsd = unsafe { core::ptr::read_volatile(tsd_ptr) };
305        if tsd == 0 {
306            return None;
307        }
308        Some(tsd)
309    }
310
311    fn unblock_sigtrap() {
312        // SAFETY: only clears the per-thread SIGTRAP block bit inherited from
313        // the spawning signal handler; disposition is already installed.
314        unsafe {
315            let mut set: libc::sigset_t = std::mem::zeroed();
316            libc::sigemptyset(std::ptr::addr_of_mut!(set));
317            libc::sigaddset(std::ptr::addr_of_mut!(set), libc::SIGTRAP);
318            libc::pthread_sigmask(
319                libc::SIG_UNBLOCK,
320                std::ptr::addr_of!(set),
321                std::ptr::null_mut(),
322            );
323        }
324    }
325
326    #[allow(clippy::as_conversions, function_casts_as_integer)]
327    fn host_thread_exit_pc() -> u64 {
328        u64::try_from(host_thread_exit as usize).unwrap_or(0)
329    }
330
331    /// Landing pad on the **host** stack after `bsdthread_terminate`.
332    unsafe extern "C" fn host_thread_exit() -> ! {
333        finish_worker_on_host();
334    }
335
336    /// Publish guest join state, drop live-worker count, end this host thread.
337    ///
338    /// Must run with SP on the host stack (never on the guest worker stack that
339    /// `pthread_join` may munmap once `done` is visible).
340    fn finish_worker_on_host() -> ! {
341        clear_host_exit_frame();
342        // Host TLS before any further host libc (join publish, munmap, exit).
343        crate::tls::clear_guest_tls_on_exit();
344        publish_guest_join_done();
345        host_slot::with_worker_mut(|w| {
346            w.guest_pthread = 0;
347        });
348        super::drop_host_alt_stack();
349        host_slot::clear_current();
350        worker_finished();
351        // End **this** OS thread only. Do **not** call `pthread_exit`:
352        // glibc runs `_Unwind_ForcedUnwind` for cleanup handlers; after a
353        // hypercall/guest jump the FP chain still points into guest memory
354        // and the DWARF walker SEGV's in `libgcc_s` (seen as pc=libgcc+0xe320
355        // under 7zz `-mmt>1`). Raw `SYS_exit` skips forced unwind.
356        // SAFETY: Linux `exit` (not `exit_group`) terminates only the caller.
357        unsafe {
358            libc::syscall(libc::SYS_exit, 0);
359        }
360        // syscall exit is noreturn; keep the type checker happy.
361        loop {
362            core::hint::spin_loop();
363        }
364    }
365
366    /// Set freestanding `KhThread.done` and futex-wake joiners.
367    ///
368    /// Guest `kh_pthread_start` stores `result` before `bsdthread_terminate` but
369    /// **must not** set `done` early: that allowed `pthread_join` to munmap the
370    /// guest stack while hypercall terminate / Rust still ran on it (intermittent
371    /// SEGV under 7zz `-mmt>1` with freestanding hypercall).
372    fn publish_guest_join_done() {
373        let pthread = host_slot::worker_get().guest_pthread;
374        if pthread == 0 {
375            return;
376        }
377        let base = usize::try_from(pthread).unwrap_or(0);
378        if base == 0 {
379            return;
380        }
381        // SAFETY: identity-mapped guest heap block from freestanding pthread_create.
382        let magic_ptr = std::ptr::with_exposed_provenance::<u64>(base);
383        let magic = unsafe { core::ptr::read_volatile(magic_ptr) };
384        if magic != KH_THREAD_MAGIC {
385            tracing::warn!(
386                pthread = format_args!("{pthread:#x}"),
387                magic = format_args!("{magic:#x}"),
388                "guest pthread magic mismatch; skip join publish"
389            );
390            return;
391        }
392        let done_addr = base.saturating_add(KH_THREAD_DONE_OFF);
393        let done_ptr = std::ptr::with_exposed_provenance_mut::<AtomicU32>(done_addr);
394        // SAFETY: aligned AtomicU32 at known freestanding layout offset.
395        unsafe {
396            (*done_ptr).store(1, Ordering::Release);
397        }
398        futex_wake(done_ptr.cast::<u32>(), i32::MAX);
399    }
400
401    fn futex_wake(addr: *mut u32, n: i32) {
402        // SAFETY: same identity-mapped word guests park on via KH_HELPER_PARK.
403        // FUTEX_WAKE_PRIVATE (129) must match `helpers::wake_u32` (park uses PRIVATE).
404        let _ = unsafe {
405            libc::syscall(
406                libc::SYS_futex,
407                addr,
408                129_i32, // FUTEX_WAKE_PRIVATE
409                n,
410                core::ptr::null::<libc::timespec>(),
411                core::ptr::null_mut::<u32>(),
412                0_i32,
413            )
414        };
415    }
416
417    fn worker_finished() {
418        let _ = LIVE_WORKERS.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| {
419            Some(n.saturating_sub(1))
420        });
421    }
422
423    /// Hypercall / trap path: leave the guest stack, then end this host worker.
424    ///
425    /// Switching SP (and clearing FP) before teardown is mandatory: freestanding
426    /// hypercall may have left `x29` pointing at a guest frame that joiners
427    /// munmap once `done` is published; forced unwind must never see that chain.
428    pub(crate) fn exit_worker_now() -> ! {
429        let (pc, sp, has) = host_slot::with_worker_mut(|w| {
430            let has = w.has_exit;
431            let pc = w.exit_pc;
432            let sp = w.exit_sp;
433            w.has_exit = false;
434            (pc, sp, has)
435        });
436        if has {
437            // SAFETY: frame was recorded on this host thread before guest entry;
438            // `pc` is `host_thread_exit`, `sp` is 16-byte aligned host stack.
439            // Zero `x29` so any accidental libc unwind cannot follow guest FP.
440            unsafe {
441                std::arch::asm!(
442                    "mov sp, {sp}",
443                    "mov x29, xzr",
444                    "br {pc}",
445                    sp = in(reg) sp,
446                    pc = in(reg) pc,
447                    options(noreturn),
448                );
449            }
450        }
451        // No frame (should not happen for workers): best-effort host exit.
452        finish_worker_on_host();
453    }
454
455    fn set_host_exit_frame(pc: u64, sp: u64) {
456        host_slot::with_worker_mut(|w| {
457            w.exit_pc = pc;
458            w.exit_sp = sp;
459            w.has_exit = true;
460        });
461    }
462
463    fn clear_host_exit_frame() {
464        host_slot::with_worker_mut(|w| {
465            w.has_exit = false;
466        });
467    }
468
469    /// Rewrite `mcontext` so sigreturn jumps to the host worker-exit trampoline.
470    pub(crate) fn redirect_ucontext_to_host_exit(m: &mut libc::mcontext_t) -> bool {
471        host_slot::with_worker_mut(|w| {
472            if !w.has_exit {
473                return false;
474            }
475            // Consume so a second terminate does not reuse a stale frame.
476            w.has_exit = false;
477            m.pc = w.exit_pc;
478            m.sp = w.exit_sp;
479            // Drop guest FP chain (same rationale as `exit_worker_now`).
480            m.regs[29] = 0;
481            true
482        })
483    }
484}
485
486#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
487use linux::linux_spawn;
488
489/// Rewrite `mcontext` so sigreturn jumps to the host worker-exit trampoline.
490///
491/// Returns `false` when no frame is installed (main thread / non-worker).
492#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
493pub(crate) use linux::redirect_ucontext_to_host_exit;
494
495/// End the current guest worker from hypercall/trap (not signal ucontext).
496///
497/// Switches to the saved host stack, publishes join `done`, then ends the
498/// host OS thread (`SYS_exit` — see `finish_worker_on_host`).
499#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
500pub(crate) fn exit_current_guest_worker() -> ! {
501    linux::exit_worker_now();
502}
503
504#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
505#[allow(dead_code)] // live only via Linux aarch64 trap/hypercall path
506pub(crate) fn redirect_ucontext_to_host_exit() -> bool {
507    false
508}
509
510#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
511#[allow(dead_code)] // live only via Linux aarch64 trap/hypercall path
512pub(crate) fn exit_current_guest_worker() -> ! {
513    loop {
514        core::hint::spin_loop();
515    }
516}
517
518#[cfg(test)]
519mod tests {
520    use super::*;
521
522    #[test]
523    fn selfid_stable_on_thread() {
524        let a = thread_selfid();
525        let b = thread_selfid();
526        assert_eq!(a, b);
527        assert!(a > 0);
528    }
529
530    #[test]
531    fn alloc_tid_monotonic() {
532        let a = alloc_tid();
533        let b = alloc_tid();
534        assert!(b > a);
535    }
536}