Skip to main content

kh_runtime/thread/
guest.rs

1//! Guest threads: host pthread spawn + `bsdthread_*` helpers.
2//!
3//! - `bsdthread_register` stores the libpthread start trampoline VA.
4//! - `bsdthread_create` spawns a host thread and jumps with Darwin
5//!   `_pthread_start` regs (`x0`–`x3`).
6//! - `bsdthread_terminate` returns to a saved host stack, then ends **this**
7//!   host thread only (never exit on the guest stack).
8//! - Join completion is published from the host stack so joiners never unmap
9//!   a live stack. Worker teardown uses raw `SYS_exit` (glibc `pthread_exit`
10//!   unwinds through guest frames). Live spawn: Linux aarch64 only.
11
12#![allow(unsafe_code)]
13
14use std::sync::atomic::{AtomicU64, Ordering};
15
16use crate::host_slot;
17
18/// Monotonic guest-visible thread id counter (`thread_selfid`).
19static NEXT_TID: AtomicU64 = AtomicU64::new(1);
20
21/// Size of the per-thread host hypercall stack (grows down from the top).
22#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
23const HOST_ALT_STACK_SIZE: usize = 512 * 1024;
24
25/// Returns a 16-byte-aligned host SP for freestanding hypercall dispatch.
26///
27/// Prefer calling [`ensure_host_alt_stack`] while host `TPIDR_EL0` is live
28/// (worker prepare / main TLS install). The hypercall entry also restores host
29/// TLS before invoking this so a cold first map is safe.
30///
31/// Uses [`host_slot`] — never Rust `thread_local!`.
32///
33/// # Safety
34///
35/// Caller must restore guest SP after the call returns. The buffer is
36/// valid until [`drop_host_alt_stack`] / process exit.
37#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
38#[unsafe(no_mangle)]
39pub unsafe extern "C" fn kh_host_alt_sp() -> u64 {
40    ensure_host_alt_stack()
41}
42
43/// Map this OS thread's hypercall alt stack if missing; return its top SP.
44///
45/// Safe to call repeatedly. Must run with **host** `TPIDR_EL0` for the first
46/// map (glibc `mmap` / map mutex).
47#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
48#[must_use]
49pub fn ensure_host_alt_stack() -> u64 {
50    let top = host_slot::with_alt_mut(|cell| {
51        if let Some(s) = *cell {
52            return s.top;
53        }
54        match map_host_alt_stack() {
55            Some(s) => {
56                let t = s.top;
57                *cell = Some(s);
58                t
59            }
60            None => 0,
61        }
62    });
63    // A1: keep guest-TLS alt_top mirror current for gettid-free enter.
64    if top != 0 {
65        crate::tls::publish_alt_top_to_current_guest(top);
66    }
67    top
68}
69
70#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
71#[must_use]
72pub fn ensure_host_alt_stack() -> u64 {
73    0
74}
75
76#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
77fn map_host_alt_stack() -> Option<host_slot::AltStackSlot> {
78    let len = HOST_ALT_STACK_SIZE;
79    let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
80    let prot = libc::PROT_READ | libc::PROT_WRITE;
81    // Prefer raw syscall so a mistaken guest-TPIDR call does not enter glibc
82    // arena code that depends on host TLS.
83    let ptr = mmap_anon_raw(len, prot, flags)?;
84    let base_u = crate::host::ptr_addr_u64(ptr);
85    let top = base_u.saturating_add(u64::try_from(len).unwrap_or(0)) & !0xF;
86    if top <= base_u {
87        let _ = crate::host::munmap(ptr, len);
88        return None;
89    }
90    Some(host_slot::AltStackSlot {
91        base: ptr,
92        len,
93        top,
94    })
95}
96
97/// Anonymous `mmap` via raw syscall (no glibc TLS dependency).
98#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
99fn mmap_anon_raw(len: usize, prot: libc::c_int, flags: libc::c_int) -> Option<*mut u8> {
100    // SAFETY: anonymous map; fd/offset unused with MAP_ANONYMOUS.
101    let raw = unsafe {
102        libc::syscall(
103            libc::SYS_mmap,
104            core::ptr::null_mut::<libc::c_void>(),
105            len,
106            prot,
107            flags,
108            -1_i32,
109            0_i64,
110        )
111    };
112    if raw < 0 {
113        return None;
114    }
115    let addr = usize::try_from(raw).ok()?;
116    Some(core::ptr::with_exposed_provenance_mut(addr))
117}
118
119/// Unmaps this thread's hypercall alt stack (worker exit / tests).
120#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
121pub(crate) fn drop_host_alt_stack() {
122    host_slot::with_alt_mut(|cell| {
123        if let Some(s) = cell.take() {
124            let _ = crate::host::munmap(s.base, s.len);
125        }
126    });
127}
128
129#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
130#[allow(dead_code)] // Linux aarch64 only; stub keeps call sites compiling.
131pub(crate) fn drop_host_alt_stack() {}
132
133/// Live guest worker threads (not including the main `LC_MAIN` thread).
134static LIVE_WORKERS: AtomicU64 = AtomicU64::new(0);
135
136/// Darwin `EAGAIN`.
137#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
138const EAGAIN: i64 = 35;
139
140/// Resets thread counters for a new guest run.
141pub fn reset_thread_runtime() {
142    NEXT_TID.store(1, Ordering::SeqCst);
143    LIVE_WORKERS.store(0, Ordering::SeqCst);
144}
145
146/// Allocates a new guest thread id (unique within the process run).
147#[must_use]
148pub fn alloc_tid() -> u64 {
149    NEXT_TID.fetch_add(1, Ordering::SeqCst)
150}
151
152/// Current host thread's guest tid (stable for the host thread lifetime).
153///
154/// Uses [`host_slot`] so it remains safe under guest `TPIDR_EL0`.
155#[must_use]
156pub fn thread_selfid() -> u64 {
157    host_slot::with_worker_mut(|w| {
158        if w.guest_tid == 0 {
159            w.guest_tid = alloc_tid();
160        }
161        w.guest_tid
162    })
163}
164
165/// Number of live worker threads spawned via `bsdthread_create`.
166#[must_use]
167pub fn live_workers() -> u64 {
168    LIVE_WORKERS.load(Ordering::SeqCst)
169}
170
171/// Arguments for a newly created guest worker.
172#[derive(Debug, Clone, Copy)]
173pub struct GuestThreadStart {
174    /// Registered `threadstart` trampoline VA.
175    pub entry: u64,
176    /// Guest stack pointer (16-byte aligned top of stack).
177    pub sp: u64,
178    /// `x0` — pthread structure VA.
179    pub pthread: u64,
180    /// `x1` — Mach thread port (stubbed to 0).
181    pub port: u64,
182    /// `x2` — user start routine VA.
183    pub func: u64,
184    /// `x3` — user argument.
185    pub func_arg: u64,
186}
187
188/// Spawns a host thread that enters guest code at `start.entry`.
189pub fn spawn_guest_thread(start: GuestThreadStart) -> Result<(), i64> {
190    #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
191    {
192        let _ = start;
193        Err(crate::syscall::ENOSYS)
194    }
195
196    #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
197    {
198        linux_spawn(start)
199    }
200}
201
202#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
203mod linux {
204    use super::{EAGAIN, GuestThreadStart, LIVE_WORKERS, thread_selfid};
205    use crate::host_slot;
206    use std::sync::atomic::{AtomicU32, Ordering};
207
208    /// Must match freestanding `kh-libsystem` `KhThread` / `MAGIC`.
209    ///
210    /// Layout (`repr(C, align(16))` on aarch64):
211    /// ```text
212    /// offset 0:  magic: u64
213    /// offset 8:  done: AtomicU32   ← join wait / host publish
214    /// offset 12: detached: AtomicU32
215    /// offset 16: result: AtomicUsize
216    /// … stack fields follow; host only touches magic + done.
217    /// offset 56: tsd
218    /// ```
219    const KH_THREAD_MAGIC: u64 = 0x4B48_5054_4852_4401;
220    const KH_THREAD_DONE_OFF: usize = 8;
221
222    pub(super) fn linux_spawn(start: GuestThreadStart) -> Result<(), i64> {
223        if start.entry == 0 || start.sp == 0 || !start.sp.is_multiple_of(16) {
224            return Err(crate::syscall::EINVAL);
225        }
226
227        let builder = std::thread::Builder::new().name("kh-guest".into());
228        match builder.spawn(move || guest_worker_main(start)) {
229            Ok(_handle) => {
230                LIVE_WORKERS.fetch_add(1, Ordering::SeqCst);
231                Ok(())
232            }
233            Err(_) => Err(EAGAIN),
234        }
235    }
236
237    fn guest_worker_main(start: GuestThreadStart) {
238        let _ = thread_selfid();
239        host_slot::with_worker_mut(|w| {
240            w.guest_pthread = start.pthread;
241        });
242
243        // Capture host TPIDR *before* guest code may msr guest TSD into TPIDR_EL0.
244        crate::tls::prepare_host_meta();
245        // Map hypercall alt stack while host TLS is still live (workers hit
246        // freestanding hypercall immediately under 7zz `-mmt>1`).
247        let _ = super::ensure_host_alt_stack();
248
249        // `bsdthread_create` runs inside the SIGTRAP handler, where SIGTRAP is
250        // blocked. `std::thread::spawn` copies that mask, so the worker would
251        // die on the first guest `brk` unless we unblock here.
252        unblock_sigtrap();
253
254        let host_sp = crate::cpu::read_sp();
255        // Leave room for host exit / join-publish frames (not just 512 B).
256        let frame_sp = host_sp.saturating_sub(4096) & !0xF;
257        set_host_exit_frame(host_thread_exit_pc(), frame_sp);
258
259        // Prefer freestanding KhThread.tsd (offset 56) when present; else guest
260        // trampoline will install TLS itself and boundary will mrs it.
261        if let Some(tsd) = guest_tsd_from_pthread(start.pthread) {
262            crate::tls::enter_guest_tls(tsd);
263        }
264
265        // SAFETY: trap handlers installed by `run_micro` before create; entry/stack
266        // are guest `bsdthread_create` arguments (mapped stack region).
267        let result = unsafe {
268            crate::entry::jump_to_guest_args(
269                start.entry,
270                start.sp,
271                start.pthread,
272                start.port,
273                start.func,
274                start.func_arg,
275            )
276        };
277        if let Err(err) = result {
278            tracing::error!(?err, "guest worker failed to enter");
279        }
280        // jump_to_guest is noreturn on success; if we return, publish join + exit.
281        finish_worker_on_host();
282    }
283
284    /// Freestanding `KhThread` layout: `tsd: *mut GuestTls` at offset 56.
285    const KH_THREAD_TSD_OFF: usize = 56;
286
287    fn guest_tsd_from_pthread(pthread: u64) -> Option<u64> {
288        let base = usize::try_from(pthread).ok()?;
289        if base == 0 {
290            return None;
291        }
292        // SAFETY: identity-mapped freestanding KhThread from pthread_create.
293        let magic_ptr = std::ptr::with_exposed_provenance::<u64>(base);
294        let magic = unsafe { core::ptr::read_volatile(magic_ptr) };
295        if magic != KH_THREAD_MAGIC {
296            return None;
297        }
298        let tsd_ptr =
299            std::ptr::with_exposed_provenance::<u64>(base.saturating_add(KH_THREAD_TSD_OFF));
300        let tsd = unsafe { core::ptr::read_volatile(tsd_ptr) };
301        if tsd == 0 {
302            return None;
303        }
304        Some(tsd)
305    }
306
307    fn unblock_sigtrap() {
308        // SAFETY: only clears the per-thread SIGTRAP block bit inherited from
309        // the spawning signal handler; disposition is already installed.
310        unsafe {
311            let mut set: libc::sigset_t = std::mem::zeroed();
312            libc::sigemptyset(std::ptr::addr_of_mut!(set));
313            libc::sigaddset(std::ptr::addr_of_mut!(set), libc::SIGTRAP);
314            libc::pthread_sigmask(
315                libc::SIG_UNBLOCK,
316                std::ptr::addr_of!(set),
317                std::ptr::null_mut(),
318            );
319        }
320    }
321
322    #[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
323    fn host_thread_exit_pc() -> u64 {
324        u64::try_from(host_thread_exit as usize).unwrap_or(0)
325    }
326
327    /// Landing pad on the **host** stack after `bsdthread_terminate`.
328    unsafe extern "C" fn host_thread_exit() -> ! {
329        finish_worker_on_host();
330    }
331
332    /// Publish guest join state, drop live-worker count, end this host thread.
333    ///
334    /// Must run with SP on the host stack (never on the guest worker stack that
335    /// `pthread_join` may munmap once `done` is visible).
336    fn finish_worker_on_host() -> ! {
337        clear_host_exit_frame();
338        // Host TLS before any further host libc (join publish, munmap, exit).
339        crate::tls::clear_guest_tls_on_exit();
340        publish_guest_join_done();
341        host_slot::with_worker_mut(|w| {
342            w.guest_pthread = 0;
343        });
344        super::drop_host_alt_stack();
345        host_slot::clear_current();
346        worker_finished();
347        // End **this** OS thread only. Do **not** call `pthread_exit`:
348        // glibc runs `_Unwind_ForcedUnwind` for cleanup handlers; after a
349        // hypercall/guest jump the FP chain still points into guest memory
350        // and the DWARF walker SEGV's in `libgcc_s` (seen as pc=libgcc+0xe320
351        // under 7zz `-mmt>1`). Raw `SYS_exit` skips forced unwind.
352        // SAFETY: Linux `exit` (not `exit_group`) terminates only the caller.
353        unsafe {
354            libc::syscall(libc::SYS_exit, 0);
355        }
356        // syscall exit is noreturn; keep the type checker happy.
357        loop {
358            core::hint::spin_loop();
359        }
360    }
361
362    /// Set freestanding `KhThread.done` and futex-wake joiners.
363    ///
364    /// Guest `kh_pthread_start` stores `result` before `bsdthread_terminate` but
365    /// **must not** set `done` early: that allowed `pthread_join` to munmap the
366    /// guest stack while hypercall terminate / Rust still ran on it (intermittent
367    /// SEGV under 7zz `-mmt>1` with freestanding hypercall).
368    fn publish_guest_join_done() {
369        let pthread = host_slot::worker_get().guest_pthread;
370        if pthread == 0 {
371            return;
372        }
373        let base = usize::try_from(pthread).unwrap_or(0);
374        if base == 0 {
375            return;
376        }
377        // SAFETY: identity-mapped guest heap block from freestanding pthread_create.
378        let magic_ptr = std::ptr::with_exposed_provenance::<u64>(base);
379        let magic = unsafe { core::ptr::read_volatile(magic_ptr) };
380        if magic != KH_THREAD_MAGIC {
381            tracing::warn!(
382                pthread = format_args!("{pthread:#x}"),
383                magic = format_args!("{magic:#x}"),
384                "guest pthread magic mismatch; skip join publish"
385            );
386            return;
387        }
388        let done_addr = base.saturating_add(KH_THREAD_DONE_OFF);
389        let done_ptr = std::ptr::with_exposed_provenance_mut::<AtomicU32>(done_addr);
390        // SAFETY: aligned AtomicU32 at known freestanding layout offset.
391        unsafe {
392            (*done_ptr).store(1, Ordering::Release);
393        }
394        futex_wake(done_ptr.cast::<u32>(), i32::MAX);
395    }
396
397    fn futex_wake(addr: *mut u32, n: i32) {
398        // SAFETY: same identity-mapped word guests park on via KH_HELPER_PARK.
399        // FUTEX_WAKE_PRIVATE (129) must match `helpers::wake_u32` (park uses PRIVATE).
400        let _ = unsafe {
401            libc::syscall(
402                libc::SYS_futex,
403                addr,
404                129_i32, // FUTEX_WAKE_PRIVATE
405                n,
406                core::ptr::null::<libc::timespec>(),
407                core::ptr::null_mut::<u32>(),
408                0_i32,
409            )
410        };
411    }
412
413    fn worker_finished() {
414        let _ = LIVE_WORKERS.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| {
415            Some(n.saturating_sub(1))
416        });
417    }
418
419    /// Hypercall / trap path: leave the guest stack, then end this host worker.
420    ///
421    /// Switching SP (and clearing FP) before teardown is mandatory: freestanding
422    /// hypercall may have left `x29` pointing at a guest frame that joiners
423    /// munmap once `done` is published; forced unwind must never see that chain.
424    pub(crate) fn exit_worker_now() -> ! {
425        let (pc, sp, has) = host_slot::with_worker_mut(|w| {
426            let has = w.has_exit;
427            let pc = w.exit_pc;
428            let sp = w.exit_sp;
429            w.has_exit = false;
430            (pc, sp, has)
431        });
432        if has {
433            // SAFETY: frame was recorded on this host thread before guest entry;
434            // `pc` is `host_thread_exit`, `sp` is 16-byte aligned host stack.
435            // Zero `x29` so any accidental libc unwind cannot follow guest FP.
436            unsafe {
437                std::arch::asm!(
438                    "mov sp, {sp}",
439                    "mov x29, xzr",
440                    "br {pc}",
441                    sp = in(reg) sp,
442                    pc = in(reg) pc,
443                    options(noreturn),
444                );
445            }
446        }
447        // No frame (should not happen for workers): best-effort host exit.
448        finish_worker_on_host();
449    }
450
451    fn set_host_exit_frame(pc: u64, sp: u64) {
452        host_slot::with_worker_mut(|w| {
453            w.exit_pc = pc;
454            w.exit_sp = sp;
455            w.has_exit = true;
456        });
457    }
458
459    fn clear_host_exit_frame() {
460        host_slot::with_worker_mut(|w| {
461            w.has_exit = false;
462        });
463    }
464
465    /// Rewrite `mcontext` so sigreturn jumps to the host worker-exit trampoline.
466    pub(crate) fn redirect_ucontext_to_host_exit(m: &mut libc::mcontext_t) -> bool {
467        host_slot::with_worker_mut(|w| {
468            if !w.has_exit {
469                return false;
470            }
471            // Consume so a second terminate does not reuse a stale frame.
472            w.has_exit = false;
473            m.pc = w.exit_pc;
474            m.sp = w.exit_sp;
475            // Drop guest FP chain (same rationale as `exit_worker_now`).
476            m.regs[29] = 0;
477            true
478        })
479    }
480}
481
482#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
483use linux::linux_spawn;
484
485/// Rewrite `mcontext` so sigreturn jumps to the host worker-exit trampoline.
486///
487/// Returns `false` when no frame is installed (main thread / non-worker).
488#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
489pub(crate) use linux::redirect_ucontext_to_host_exit;
490
491/// End the current guest worker from hypercall/trap (not signal ucontext).
492///
493/// Switches to the saved host stack, publishes join `done`, then ends the
494/// host OS thread (`SYS_exit` — see `finish_worker_on_host`).
495#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
496pub(crate) fn exit_current_guest_worker() -> ! {
497    linux::exit_worker_now();
498}
499
500#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
501#[allow(dead_code)] // live only via Linux aarch64 trap/hypercall path
502pub(crate) fn redirect_ucontext_to_host_exit() -> bool {
503    false
504}
505
506#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
507#[allow(dead_code)] // live only via Linux aarch64 trap/hypercall path
508pub(crate) fn exit_current_guest_worker() -> ! {
509    loop {
510        core::hint::spin_loop();
511    }
512}
513
514#[cfg(test)]
515mod tests {
516    use super::*;
517
518    #[test]
519    fn selfid_stable_on_thread() {
520        let a = thread_selfid();
521        let b = thread_selfid();
522        assert_eq!(a, b);
523        assert!(a > 0);
524    }
525
526    #[test]
527    fn alloc_tid_monotonic() {
528        let a = alloc_tid();
529        let b = alloc_tid();
530        assert!(b > a);
531    }
532}