Skip to main content

kh_runtime/
thread.rs

1//! Guest thread support: host pthread spawn + `bsdthread_*` helpers.
2//!
3//! Model (micro / clean-room):
4//! - `bsdthread_register` stores the libpthread start trampoline VA.
5//! - `bsdthread_create` spawns a **host** thread that jumps into that trampoline
6//!   with Darwin `_pthread_start` register convention (`x0`–`x3`).
7//! - `bsdthread_terminate` switches back to a saved **host** stack then ends
8//!   this host thread only (never exit on the guest stack).
9//! - Guest `pthread_join` completion (`done` + futex wake) is published from
10//!   the host stack so joiners never `munmap` a stack still in use.
11//! - Worker teardown uses raw `SYS_exit` (not `pthread_exit`): glibc
12//!   `pthread_exit` runs `_Unwind_ForcedUnwind`, which walks `x29` into guest /
13//!   hypercall frames and SEGV's in `libgcc_s` under 7zz MT.
14//! - Live spawn requires **Linux aarch64** (same as the trap backend).
15
16#![allow(unsafe_code)]
17
18use std::sync::atomic::{AtomicU64, Ordering};
19
20use crate::host_slot;
21
22/// Monotonic guest-visible thread id counter (`thread_selfid`).
23static NEXT_TID: AtomicU64 = AtomicU64::new(1);
24
25/// Size of the per-thread host hypercall stack (grows down from the top).
26#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
27const HOST_ALT_STACK_SIZE: usize = 512 * 1024;
28
29/// Returns a 16-byte-aligned host SP for freestanding hypercall dispatch.
30///
31/// Prefer calling [`ensure_host_alt_stack`] while host `TPIDR_EL0` is live
32/// (worker prepare / main TLS install). The hypercall entry also restores host
33/// TLS before invoking this so a cold first map is safe.
34///
35/// Uses [`host_slot`] — never Rust `thread_local!`.
36///
37/// # Safety
38///
39/// Caller must restore guest SP after the call returns. The buffer is
40/// valid until [`drop_host_alt_stack`] / process exit.
41#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
42#[unsafe(no_mangle)]
43pub unsafe extern "C" fn kh_host_alt_sp() -> u64 {
44    ensure_host_alt_stack()
45}
46
47/// Map this OS thread's hypercall alt stack if missing; return its top SP.
48///
49/// Safe to call repeatedly. Must run with **host** `TPIDR_EL0` for the first
50/// map (glibc `mmap` / map mutex).
51#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
52#[must_use]
53pub fn ensure_host_alt_stack() -> u64 {
54    host_slot::with_alt_mut(|cell| {
55        if let Some(s) = *cell {
56            return s.top;
57        }
58        match map_host_alt_stack() {
59            Some(s) => {
60                let top = s.top;
61                *cell = Some(s);
62                top
63            }
64            None => 0,
65        }
66    })
67}
68
69#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
70#[must_use]
71pub fn ensure_host_alt_stack() -> u64 {
72    0
73}
74
75#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
76fn map_host_alt_stack() -> Option<host_slot::AltStackSlot> {
77    let len = HOST_ALT_STACK_SIZE;
78    let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
79    let prot = libc::PROT_READ | libc::PROT_WRITE;
80    // Prefer raw syscall so a mistaken guest-TPIDR call does not enter glibc
81    // arena code that depends on host TLS.
82    let ptr = mmap_anon_raw(len, prot, flags)?;
83    let base_u = crate::host::ptr_addr_u64(ptr);
84    let top = base_u.saturating_add(u64::try_from(len).unwrap_or(0)) & !0xF;
85    if top <= base_u {
86        let _ = crate::host::munmap(ptr, len);
87        return None;
88    }
89    Some(host_slot::AltStackSlot {
90        base: ptr,
91        len,
92        top,
93    })
94}
95
96/// Anonymous `mmap` via raw syscall (no glibc TLS dependency).
97#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
98fn mmap_anon_raw(len: usize, prot: libc::c_int, flags: libc::c_int) -> Option<*mut u8> {
99    // SAFETY: anonymous map; fd/offset unused with MAP_ANONYMOUS.
100    let raw = unsafe {
101        libc::syscall(
102            libc::SYS_mmap,
103            core::ptr::null_mut::<libc::c_void>(),
104            len,
105            prot,
106            flags,
107            -1_i32,
108            0_i64,
109        )
110    };
111    if raw < 0 {
112        return None;
113    }
114    let addr = usize::try_from(raw).ok()?;
115    Some(core::ptr::with_exposed_provenance_mut(addr))
116}
117
118/// Unmaps this thread's hypercall alt stack (worker exit / tests).
119#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
120pub(crate) fn drop_host_alt_stack() {
121    host_slot::with_alt_mut(|cell| {
122        if let Some(s) = cell.take() {
123            let _ = crate::host::munmap(s.base, s.len);
124        }
125    });
126}
127
128#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
129#[allow(dead_code)] // Linux aarch64 only; stub keeps call sites compiling.
130pub(crate) fn drop_host_alt_stack() {}
131
132/// Live guest worker threads (not including the main `LC_MAIN` thread).
133static LIVE_WORKERS: AtomicU64 = AtomicU64::new(0);
134
135/// Darwin `EAGAIN`.
136#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
137const EAGAIN: i64 = 35;
138
139/// Resets thread counters for a new guest run.
140pub fn reset_thread_runtime() {
141    NEXT_TID.store(1, Ordering::SeqCst);
142    LIVE_WORKERS.store(0, Ordering::SeqCst);
143}
144
145/// Allocates a new guest thread id (unique within the process run).
146#[must_use]
147pub fn alloc_tid() -> u64 {
148    NEXT_TID.fetch_add(1, Ordering::SeqCst)
149}
150
151/// Current host thread's guest tid (stable for the host thread lifetime).
152///
153/// Uses [`host_slot`] so it remains safe under guest `TPIDR_EL0`.
154#[must_use]
155pub fn thread_selfid() -> u64 {
156    host_slot::with_worker_mut(|w| {
157        if w.guest_tid == 0 {
158            w.guest_tid = alloc_tid();
159        }
160        w.guest_tid
161    })
162}
163
164/// Number of live worker threads spawned via `bsdthread_create`.
165#[must_use]
166pub fn live_workers() -> u64 {
167    LIVE_WORKERS.load(Ordering::SeqCst)
168}
169
170/// Arguments for a newly created guest worker.
171#[derive(Debug, Clone, Copy)]
172pub struct GuestThreadStart {
173    /// Registered `threadstart` trampoline VA.
174    pub entry: u64,
175    /// Guest stack pointer (16-byte aligned top of stack).
176    pub sp: u64,
177    /// `x0` — pthread structure VA.
178    pub pthread: u64,
179    /// `x1` — Mach thread port (stubbed to 0).
180    pub port: u64,
181    /// `x2` — user start routine VA.
182    pub func: u64,
183    /// `x3` — user argument.
184    pub func_arg: u64,
185}
186
187/// Spawns a host thread that enters guest code at `start.entry`.
188pub fn spawn_guest_thread(start: GuestThreadStart) -> Result<(), i64> {
189    #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
190    {
191        let _ = start;
192        Err(crate::syscall::ENOSYS)
193    }
194
195    #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
196    {
197        linux_spawn(start)
198    }
199}
200
201#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
202mod linux {
203    use super::{EAGAIN, GuestThreadStart, LIVE_WORKERS, thread_selfid};
204    use crate::host_slot;
205    use std::sync::atomic::{AtomicU32, Ordering};
206
207    /// Must match freestanding `kh-libsystem` `KhThread` / `MAGIC`.
208    ///
209    /// Layout (`repr(C, align(16))` on aarch64):
210    /// ```text
211    /// offset 0:  magic: u64
212    /// offset 8:  done: AtomicU32   ← join wait / host publish
213    /// offset 12: detached: AtomicU32
214    /// offset 16: result: AtomicUsize
215    /// … stack fields follow; host only touches magic + done.
216    /// offset 56: tsd
217    /// ```
218    const KH_THREAD_MAGIC: u64 = 0x4B48_5054_4852_4401;
219    const KH_THREAD_DONE_OFF: usize = 8;
220
221    pub(super) fn linux_spawn(start: GuestThreadStart) -> Result<(), i64> {
222        if start.entry == 0 || start.sp == 0 || !start.sp.is_multiple_of(16) {
223            return Err(crate::syscall::EINVAL);
224        }
225
226        let builder = std::thread::Builder::new().name("kh-guest".into());
227        match builder.spawn(move || guest_worker_main(start)) {
228            Ok(_handle) => {
229                LIVE_WORKERS.fetch_add(1, Ordering::SeqCst);
230                Ok(())
231            }
232            Err(_) => Err(EAGAIN),
233        }
234    }
235
236    fn guest_worker_main(start: GuestThreadStart) {
237        let _ = thread_selfid();
238        host_slot::with_worker_mut(|w| {
239            w.guest_pthread = start.pthread;
240        });
241
242        // Capture host TPIDR *before* guest code may msr guest TSD into TPIDR_EL0.
243        crate::tls::prepare_host_meta();
244        // Map hypercall alt stack while host TLS is still live (workers hit
245        // freestanding hypercall immediately under 7zz `-mmt>1`).
246        let _ = super::ensure_host_alt_stack();
247
248        // `bsdthread_create` runs inside the SIGTRAP handler, where SIGTRAP is
249        // blocked. `std::thread::spawn` copies that mask, so the worker would
250        // die on the first guest `brk` unless we unblock here.
251        unblock_sigtrap();
252
253        let host_sp = crate::cpu::read_sp();
254        // Leave room for host exit / join-publish frames (not just 512 B).
255        let frame_sp = host_sp.saturating_sub(4096) & !0xF;
256        set_host_exit_frame(host_thread_exit_pc(), frame_sp);
257
258        // Prefer freestanding KhThread.tsd (offset 56) when present; else guest
259        // trampoline will install TLS itself and boundary will mrs it.
260        if let Some(tsd) = guest_tsd_from_pthread(start.pthread) {
261            crate::tls::enter_guest_tls(tsd);
262        }
263
264        // SAFETY: trap handlers installed by `run_micro` before create; entry/stack
265        // are guest `bsdthread_create` arguments (mapped stack region).
266        let result = unsafe {
267            crate::entry::jump_to_guest_args(
268                start.entry,
269                start.sp,
270                start.pthread,
271                start.port,
272                start.func,
273                start.func_arg,
274            )
275        };
276        if let Err(err) = result {
277            tracing::error!(?err, "guest worker failed to enter");
278        }
279        // jump_to_guest is noreturn on success; if we return, publish join + exit.
280        finish_worker_on_host();
281    }
282
283    /// Freestanding `KhThread` layout: `tsd: *mut GuestTls` at offset 56.
284    const KH_THREAD_TSD_OFF: usize = 56;
285
286    fn guest_tsd_from_pthread(pthread: u64) -> Option<u64> {
287        let base = usize::try_from(pthread).ok()?;
288        if base == 0 {
289            return None;
290        }
291        // SAFETY: identity-mapped freestanding KhThread from pthread_create.
292        let magic_ptr = std::ptr::with_exposed_provenance::<u64>(base);
293        let magic = unsafe { core::ptr::read_volatile(magic_ptr) };
294        if magic != KH_THREAD_MAGIC {
295            return None;
296        }
297        let tsd_ptr = std::ptr::with_exposed_provenance::<u64>(base.saturating_add(KH_THREAD_TSD_OFF));
298        let tsd = unsafe { core::ptr::read_volatile(tsd_ptr) };
299        if tsd == 0 {
300            return None;
301        }
302        Some(tsd)
303    }
304
305    fn unblock_sigtrap() {
306        // SAFETY: only clears the per-thread SIGTRAP block bit inherited from
307        // the spawning signal handler; disposition is already installed.
308        unsafe {
309            let mut set: libc::sigset_t = std::mem::zeroed();
310            libc::sigemptyset(std::ptr::addr_of_mut!(set));
311            libc::sigaddset(std::ptr::addr_of_mut!(set), libc::SIGTRAP);
312            libc::pthread_sigmask(
313                libc::SIG_UNBLOCK,
314                std::ptr::addr_of!(set),
315                std::ptr::null_mut(),
316            );
317        }
318    }
319
320    #[allow(clippy::as_conversions, function_casts_as_integer)]
321    fn host_thread_exit_pc() -> u64 {
322        u64::try_from(host_thread_exit as usize).unwrap_or(0)
323    }
324
325    /// Landing pad on the **host** stack after `bsdthread_terminate`.
326    unsafe extern "C" fn host_thread_exit() -> ! {
327        finish_worker_on_host();
328    }
329
330    /// Publish guest join state, drop live-worker count, end this host thread.
331    ///
332    /// Must run with SP on the host stack (never on the guest worker stack that
333    /// `pthread_join` may munmap once `done` is visible).
334    fn finish_worker_on_host() -> ! {
335        clear_host_exit_frame();
336        // Host TLS before any further host libc (join publish, munmap, exit).
337        crate::tls::clear_guest_tls_on_exit();
338        publish_guest_join_done();
339        host_slot::with_worker_mut(|w| {
340            w.guest_pthread = 0;
341        });
342        super::drop_host_alt_stack();
343        host_slot::clear_current();
344        worker_finished();
345        // End **this** OS thread only. Do **not** call `pthread_exit`:
346        // glibc runs `_Unwind_ForcedUnwind` for cleanup handlers; after a
347        // hypercall/guest jump the FP chain still points into guest memory
348        // and the DWARF walker SEGV's in `libgcc_s` (seen as pc=libgcc+0xe320
349        // under 7zz `-mmt>1`). Raw `SYS_exit` skips forced unwind.
350        // SAFETY: Linux `exit` (not `exit_group`) terminates only the caller.
351        unsafe {
352            libc::syscall(libc::SYS_exit, 0);
353        }
354        // syscall exit is noreturn; keep the type checker happy.
355        loop {
356            core::hint::spin_loop();
357        }
358    }
359
360    /// Set freestanding `KhThread.done` and futex-wake joiners.
361    ///
362    /// Guest `kh_pthread_start` stores `result` before `bsdthread_terminate` but
363    /// **must not** set `done` early: that allowed `pthread_join` to munmap the
364    /// guest stack while hypercall terminate / Rust still ran on it (intermittent
365    /// SEGV under 7zz `-mmt>1` with freestanding hypercall).
366    fn publish_guest_join_done() {
367        let pthread = host_slot::worker_get().guest_pthread;
368        if pthread == 0 {
369            return;
370        }
371        let base = usize::try_from(pthread).unwrap_or(0);
372        if base == 0 {
373            return;
374        }
375        // SAFETY: identity-mapped guest heap block from freestanding pthread_create.
376        let magic_ptr = std::ptr::with_exposed_provenance::<u64>(base);
377        let magic = unsafe { core::ptr::read_volatile(magic_ptr) };
378        if magic != KH_THREAD_MAGIC {
379            tracing::warn!(
380                pthread = format_args!("{pthread:#x}"),
381                magic = format_args!("{magic:#x}"),
382                "guest pthread magic mismatch; skip join publish"
383            );
384            return;
385        }
386        let done_addr = base.saturating_add(KH_THREAD_DONE_OFF);
387        let done_ptr = std::ptr::with_exposed_provenance_mut::<AtomicU32>(done_addr);
388        // SAFETY: aligned AtomicU32 at known freestanding layout offset.
389        unsafe {
390            (*done_ptr).store(1, Ordering::Release);
391        }
392        futex_wake(done_ptr.cast::<u32>(), i32::MAX);
393    }
394
395    fn futex_wake(addr: *mut u32, n: i32) {
396        // SAFETY: same identity-mapped word guests park on via KH_HELPER_PARK.
397        let _ = unsafe {
398            libc::syscall(
399                libc::SYS_futex,
400                addr,
401                1_i32, // FUTEX_WAKE
402                n,
403                core::ptr::null::<libc::timespec>(),
404                core::ptr::null_mut::<u32>(),
405                0_i32,
406            )
407        };
408    }
409
410    fn worker_finished() {
411        let _ = LIVE_WORKERS.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| {
412            Some(n.saturating_sub(1))
413        });
414    }
415
416    /// Hypercall / trap path: leave the guest stack, then end this host worker.
417    ///
418    /// Switching SP (and clearing FP) before teardown is mandatory: freestanding
419    /// hypercall may have left `x29` pointing at a guest frame that joiners
420    /// munmap once `done` is published; forced unwind must never see that chain.
421    pub(crate) fn exit_worker_now() -> ! {
422        let (pc, sp, has) = host_slot::with_worker_mut(|w| {
423            let has = w.has_exit;
424            let pc = w.exit_pc;
425            let sp = w.exit_sp;
426            w.has_exit = false;
427            (pc, sp, has)
428        });
429        if has {
430            // SAFETY: frame was recorded on this host thread before guest entry;
431            // `pc` is `host_thread_exit`, `sp` is 16-byte aligned host stack.
432            // Zero `x29` so any accidental libc unwind cannot follow guest FP.
433            unsafe {
434                std::arch::asm!(
435                    "mov sp, {sp}",
436                    "mov x29, xzr",
437                    "br {pc}",
438                    sp = in(reg) sp,
439                    pc = in(reg) pc,
440                    options(noreturn),
441                );
442            }
443        }
444        // No frame (should not happen for workers): best-effort host exit.
445        finish_worker_on_host();
446    }
447
448    fn set_host_exit_frame(pc: u64, sp: u64) {
449        host_slot::with_worker_mut(|w| {
450            w.exit_pc = pc;
451            w.exit_sp = sp;
452            w.has_exit = true;
453        });
454    }
455
456    fn clear_host_exit_frame() {
457        host_slot::with_worker_mut(|w| {
458            w.has_exit = false;
459        });
460    }
461
462    /// Rewrite `mcontext` so sigreturn jumps to the host worker-exit trampoline.
463    pub(crate) fn redirect_ucontext_to_host_exit(m: &mut libc::mcontext_t) -> bool {
464        host_slot::with_worker_mut(|w| {
465            if !w.has_exit {
466                return false;
467            }
468            // Consume so a second terminate does not reuse a stale frame.
469            w.has_exit = false;
470            m.pc = w.exit_pc;
471            m.sp = w.exit_sp;
472            // Drop guest FP chain (same rationale as `exit_worker_now`).
473            m.regs[29] = 0;
474            true
475        })
476    }
477}
478
479#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
480use linux::linux_spawn;
481
482/// Rewrite `mcontext` so sigreturn jumps to the host worker-exit trampoline.
483///
484/// Returns `false` when no frame is installed (main thread / non-worker).
485#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
486pub(crate) use linux::redirect_ucontext_to_host_exit;
487
488/// End the current guest worker from hypercall/trap (not signal ucontext).
489///
490/// Switches to the saved host stack, publishes join `done`, then ends the
491/// host OS thread (`SYS_exit` — see `finish_worker_on_host`).
492#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
493pub(crate) fn exit_current_guest_worker() -> ! {
494    linux::exit_worker_now();
495}
496
497#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
498#[allow(dead_code)] // live only via Linux aarch64 trap/hypercall path
499pub(crate) fn redirect_ucontext_to_host_exit() -> bool {
500    false
501}
502
503#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
504#[allow(dead_code)] // live only via Linux aarch64 trap/hypercall path
505pub(crate) fn exit_current_guest_worker() -> ! {
506    loop {
507        core::hint::spin_loop();
508    }
509}
510
511#[cfg(test)]
512mod tests {
513    use super::*;
514
515    #[test]
516    fn selfid_stable_on_thread() {
517        let a = thread_selfid();
518        let b = thread_selfid();
519        assert_eq!(a, b);
520        assert!(a > 0);
521    }
522
523    #[test]
524    fn alloc_tid_monotonic() {
525        let a = alloc_tid();
526        let b = alloc_tid();
527        assert!(b > a);
528    }
529}