Skip to main content

kh_runtime/
thread.rs

1//! Guest thread support: host pthread spawn + `bsdthread_*` helpers.
2//!
3//! Model (micro / clean-room):
4//! - `bsdthread_register` stores the libpthread start trampoline VA.
5//! - `bsdthread_create` spawns a **host** thread that jumps into that trampoline
6//!   with Darwin `_pthread_start` register convention (`x0`–`x3`).
7//! - `bsdthread_terminate` switches back to a saved **host** stack then
8//!   `pthread_exit`s this host thread only (never exit on the guest stack).
9//! - Guest `pthread_join` completion (`done` + futex wake) is published from
10//!   the host stack so joiners never `munmap` a stack still in use.
11//! - Live spawn requires **Linux aarch64** (same as the trap backend).
12
13#![allow(unsafe_code)]
14
15use std::cell::Cell;
16use std::sync::atomic::{AtomicU64, Ordering};
17
18thread_local! {
19    static TID: Cell<u64> = const { Cell::new(0) };
20}
21
22// Per-thread host stack for freestanding hypercall dispatch (see `kh_host_alt_sp`).
23// Guest SP is unsafe for host Rust under 7zz `-mmt>1`.
24#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
25thread_local! {
26    static HOST_ALT: Cell<Option<HostAltStack>> = const { Cell::new(None) };
27}
28
29/// Monotonic guest-visible thread id counter (`thread_selfid`).
30static NEXT_TID: AtomicU64 = AtomicU64::new(1);
31
32/// Size of the per-thread host hypercall stack (grows down from the top).
33#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
34const HOST_ALT_STACK_SIZE: usize = 512 * 1024;
35
36#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
37#[derive(Clone, Copy)]
38struct HostAltStack {
39    base: *mut u8,
40    len: usize,
41    /// 16-byte-aligned top address used as initial SP.
42    top: u64,
43}
44
45/// Returns a 16-byte-aligned host SP for freestanding hypercall dispatch.
46///
47/// Called from the hypercall asm entry **before** Rust dispatch. Lazy-maps a
48/// private stack the first time each host thread hypercalls.
49///
50/// # Safety
51///
52/// Caller must restore guest SP after the call returns. The buffer is
53/// thread-local and valid until [`drop_host_alt_stack`] / process exit.
54#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
55#[unsafe(no_mangle)]
56pub unsafe extern "C" fn kh_host_alt_sp() -> u64 {
57    HOST_ALT.with(|cell| {
58        if let Some(s) = cell.get() {
59            return s.top;
60        }
61        match map_host_alt_stack() {
62            Some(s) => {
63                cell.set(Some(s));
64                s.top
65            }
66            None => 0,
67        }
68    })
69}
70
71#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
72fn map_host_alt_stack() -> Option<HostAltStack> {
73    let len = HOST_ALT_STACK_SIZE;
74    let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
75    // Linux: MAP_STACK is advisory; omit for portability of the flag set.
76    let prot = libc::PROT_READ | libc::PROT_WRITE;
77    let ptr = crate::host::mmap(None, len, prot, flags, -1, 0)?;
78    let base_u = crate::host::ptr_addr_u64(ptr);
79    let top = base_u.saturating_add(u64::try_from(len).unwrap_or(0)) & !0xF;
80    if top <= base_u {
81        let _ = crate::host::munmap(ptr, len);
82        return None;
83    }
84    Some(HostAltStack {
85        base: ptr,
86        len,
87        top,
88    })
89}
90
91/// Unmaps this thread's hypercall alt stack (worker exit / tests).
92#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
93pub(crate) fn drop_host_alt_stack() {
94    HOST_ALT.with(|cell| {
95        if let Some(s) = cell.take() {
96            let _ = crate::host::munmap(s.base, s.len);
97        }
98    });
99}
100
101#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
102#[allow(dead_code)] // Linux aarch64 only; stub keeps call sites compiling.
103pub(crate) fn drop_host_alt_stack() {}
104
105/// Live guest worker threads (not including the main `LC_MAIN` thread).
106static LIVE_WORKERS: AtomicU64 = AtomicU64::new(0);
107
108/// Darwin `EAGAIN`.
109#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
110const EAGAIN: i64 = 35;
111
112/// Resets thread counters for a new guest run.
113pub fn reset_thread_runtime() {
114    NEXT_TID.store(1, Ordering::SeqCst);
115    LIVE_WORKERS.store(0, Ordering::SeqCst);
116}
117
118/// Allocates a new guest thread id (unique within the process run).
119#[must_use]
120pub fn alloc_tid() -> u64 {
121    NEXT_TID.fetch_add(1, Ordering::SeqCst)
122}
123
124/// Current host thread's guest tid (stable for the host thread lifetime).
125#[must_use]
126pub fn thread_selfid() -> u64 {
127    TID.with(|c| {
128        let mut v = c.get();
129        if v == 0 {
130            v = alloc_tid();
131            c.set(v);
132        }
133        v
134    })
135}
136
137/// Number of live worker threads spawned via `bsdthread_create`.
138#[must_use]
139pub fn live_workers() -> u64 {
140    LIVE_WORKERS.load(Ordering::SeqCst)
141}
142
143/// Arguments for a newly created guest worker.
144#[derive(Debug, Clone, Copy)]
145pub struct GuestThreadStart {
146    /// Registered `threadstart` trampoline VA.
147    pub entry: u64,
148    /// Guest stack pointer (16-byte aligned top of stack).
149    pub sp: u64,
150    /// `x0` — pthread structure VA.
151    pub pthread: u64,
152    /// `x1` — Mach thread port (stubbed to 0).
153    pub port: u64,
154    /// `x2` — user start routine VA.
155    pub func: u64,
156    /// `x3` — user argument.
157    pub func_arg: u64,
158}
159
160/// Spawns a host thread that enters guest code at `start.entry`.
161pub fn spawn_guest_thread(start: GuestThreadStart) -> Result<(), i64> {
162    #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
163    {
164        let _ = start;
165        Err(crate::syscall::ENOSYS)
166    }
167
168    #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
169    {
170        linux_spawn(start)
171    }
172}
173
174#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
175mod linux {
176    use super::{EAGAIN, GuestThreadStart, LIVE_WORKERS, thread_selfid};
177    use std::cell::Cell;
178    use std::sync::atomic::{AtomicU32, Ordering};
179
180    /// Must match freestanding `kh-libsystem` `KhThread` / `MAGIC`.
181    ///
182    /// Layout (`repr(C, align(16))` on aarch64):
183    /// ```text
184    /// offset 0:  magic: u64
185    /// offset 8:  done: AtomicU32   ← join wait / host publish
186    /// offset 12: detached: AtomicU32
187    /// offset 16: result: AtomicUsize
188    /// … stack fields follow; host only touches magic + done.
189    /// ```
190    const KH_THREAD_MAGIC: u64 = 0x4B48_5054_4852_4401;
191    const KH_THREAD_DONE_OFF: usize = 8;
192
193    /// Host `(pc, sp)` used when a guest worker hits `bsdthread_terminate`.
194    #[derive(Clone, Copy)]
195    struct HostExitFrame {
196        pc: u64,
197        sp: u64,
198    }
199
200    thread_local! {
201        static HOST_EXIT: Cell<Option<HostExitFrame>> = const { Cell::new(None) };
202        /// Guest `pthread_t` VA for this worker (identity-mapped `KhThread`).
203        static GUEST_PTHREAD: Cell<u64> = const { Cell::new(0) };
204    }
205
206    pub(super) fn linux_spawn(start: GuestThreadStart) -> Result<(), i64> {
207        if start.entry == 0 || start.sp == 0 || !start.sp.is_multiple_of(16) {
208            return Err(crate::syscall::EINVAL);
209        }
210
211        let builder = std::thread::Builder::new().name("kh-guest".into());
212        match builder.spawn(move || guest_worker_main(start)) {
213            Ok(_handle) => {
214                LIVE_WORKERS.fetch_add(1, Ordering::SeqCst);
215                Ok(())
216            }
217            Err(_) => Err(EAGAIN),
218        }
219    }
220
221    fn guest_worker_main(start: GuestThreadStart) {
222        let _ = thread_selfid();
223        GUEST_PTHREAD.set(start.pthread);
224
225        // `bsdthread_create` runs inside the SIGTRAP handler, where SIGTRAP is
226        // blocked. `std::thread::spawn` copies that mask, so the worker would
227        // die on the first guest `brk` unless we unblock here.
228        unblock_sigtrap();
229
230        let host_sp = read_sp();
231        // Leave room for host exit / join-publish frames (not just 512 B).
232        let frame_sp = host_sp.saturating_sub(4096) & !0xF;
233        set_host_exit_frame(host_thread_exit_pc(), frame_sp);
234
235        // SAFETY: trap handlers installed by `run_micro` before create; entry/stack
236        // are guest `bsdthread_create` arguments (mapped stack region).
237        let result = unsafe {
238            crate::entry::jump_to_guest_args(
239                start.entry,
240                start.sp,
241                start.pthread,
242                start.port,
243                start.func,
244                start.func_arg,
245            )
246        };
247        if let Err(err) = result {
248            tracing::error!(?err, "guest worker failed to enter");
249        }
250        // jump_to_guest is noreturn on success; if we return, publish join + exit.
251        finish_worker_on_host();
252    }
253
254    fn unblock_sigtrap() {
255        // SAFETY: only clears the per-thread SIGTRAP block bit inherited from
256        // the spawning signal handler; disposition is already installed.
257        unsafe {
258            let mut set: libc::sigset_t = std::mem::zeroed();
259            libc::sigemptyset(std::ptr::addr_of_mut!(set));
260            libc::sigaddset(std::ptr::addr_of_mut!(set), libc::SIGTRAP);
261            libc::pthread_sigmask(
262                libc::SIG_UNBLOCK,
263                std::ptr::addr_of!(set),
264                std::ptr::null_mut(),
265            );
266        }
267    }
268
269    fn read_sp() -> u64 {
270        let sp: u64;
271        // SAFETY: reading SP is a pure register move.
272        unsafe {
273            std::arch::asm!("mov {}, sp", out(reg) sp, options(nomem, nostack, preserves_flags));
274        }
275        sp
276    }
277
278    #[allow(clippy::as_conversions, function_casts_as_integer)]
279    fn host_thread_exit_pc() -> u64 {
280        u64::try_from(host_thread_exit as usize).unwrap_or(0)
281    }
282
283    /// Landing pad on the **host** stack after `bsdthread_terminate`.
284    unsafe extern "C" fn host_thread_exit() -> ! {
285        finish_worker_on_host();
286    }
287
288    /// Publish guest join state, drop live-worker count, end this host thread.
289    ///
290    /// Must run with SP on the host stack (never on the guest worker stack that
291    /// `pthread_join` may munmap once `done` is visible).
292    fn finish_worker_on_host() -> ! {
293        clear_host_exit_frame();
294        publish_guest_join_done();
295        GUEST_PTHREAD.set(0);
296        super::drop_host_alt_stack();
297        worker_finished();
298        // SAFETY: intentional end of this host thread only (not the process).
299        unsafe {
300            libc::pthread_exit(std::ptr::null_mut());
301        }
302    }
303
304    /// Set freestanding `KhThread.done` and futex-wake joiners.
305    ///
306    /// Guest `kh_pthread_start` stores `result` before `bsdthread_terminate` but
307    /// **must not** set `done` early: that allowed `pthread_join` to munmap the
308    /// guest stack while hypercall terminate / Rust still ran on it (intermittent
309    /// SEGV under 7zz `-mmt>1` with freestanding hypercall).
310    fn publish_guest_join_done() {
311        let pthread = GUEST_PTHREAD.get();
312        if pthread == 0 {
313            return;
314        }
315        let base = usize::try_from(pthread).unwrap_or(0);
316        if base == 0 {
317            return;
318        }
319        // SAFETY: identity-mapped guest heap block from freestanding pthread_create.
320        let magic_ptr = std::ptr::with_exposed_provenance::<u64>(base);
321        let magic = unsafe { core::ptr::read_volatile(magic_ptr) };
322        if magic != KH_THREAD_MAGIC {
323            tracing::warn!(
324                pthread = format_args!("{pthread:#x}"),
325                magic = format_args!("{magic:#x}"),
326                "guest pthread magic mismatch; skip join publish"
327            );
328            return;
329        }
330        let done_addr = base.saturating_add(KH_THREAD_DONE_OFF);
331        let done_ptr = std::ptr::with_exposed_provenance_mut::<AtomicU32>(done_addr);
332        // SAFETY: aligned AtomicU32 at known freestanding layout offset.
333        unsafe {
334            (*done_ptr).store(1, Ordering::Release);
335        }
336        futex_wake(done_ptr.cast::<u32>(), i32::MAX);
337    }
338
339    fn futex_wake(addr: *mut u32, n: i32) {
340        // SAFETY: same identity-mapped word guests park on via KH_HELPER_PARK.
341        let _ = unsafe {
342            libc::syscall(
343                libc::SYS_futex,
344                addr,
345                1_i32, // FUTEX_WAKE
346                n,
347                core::ptr::null::<libc::timespec>(),
348                core::ptr::null_mut::<u32>(),
349                0_i32,
350            )
351        };
352    }
353
354    fn worker_finished() {
355        let _ = LIVE_WORKERS.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| {
356            Some(n.saturating_sub(1))
357        });
358    }
359
360    /// Hypercall / trap path: leave the guest stack, then end this host worker.
361    ///
362    /// Switching SP before `pthread_exit` is mandatory: freestanding hypercall
363    /// runs host Rust on the guest stack, and joiners reclaim that stack once
364    /// the worker is joinable.
365    pub(crate) fn exit_worker_now() -> ! {
366        let frame = HOST_EXIT.take();
367        if let Some(frame) = frame {
368            // SAFETY: frame was recorded on this host thread before guest entry;
369            // `pc` is `host_thread_exit`, `sp` is 16-byte aligned host stack.
370            unsafe {
371                std::arch::asm!(
372                    "mov sp, {sp}",
373                    "br {pc}",
374                    sp = in(reg) frame.sp,
375                    pc = in(reg) frame.pc,
376                    options(noreturn),
377                );
378            }
379        }
380        // No frame (should not happen for workers): best-effort host exit.
381        finish_worker_on_host();
382    }
383
384    fn set_host_exit_frame(pc: u64, sp: u64) {
385        HOST_EXIT.set(Some(HostExitFrame { pc, sp }));
386    }
387
388    fn clear_host_exit_frame() {
389        HOST_EXIT.set(None);
390    }
391
392    /// Rewrite `mcontext` so sigreturn jumps to the host `pthread_exit` trampoline.
393    pub(crate) fn redirect_ucontext_to_host_exit(m: &mut libc::mcontext_t) -> bool {
394        HOST_EXIT.with(|cell| {
395            let Some(frame) = cell.get() else {
396                return false;
397            };
398            // Consume so a second terminate does not reuse a stale frame.
399            cell.set(None);
400            m.pc = frame.pc;
401            m.sp = frame.sp;
402            true
403        })
404    }
405}
406
407#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
408use linux::linux_spawn;
409
410/// Rewrite `mcontext` so sigreturn jumps to the host `pthread_exit` trampoline.
411///
412/// Returns `false` when no frame is installed (main thread / non-worker).
413#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
414pub(crate) use linux::redirect_ucontext_to_host_exit;
415
416/// End the current guest worker from hypercall/trap (not signal ucontext).
417///
418/// Switches to the saved host stack, publishes join `done`, then `pthread_exit`s.
419#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
420pub(crate) fn exit_current_guest_worker() -> ! {
421    linux::exit_worker_now();
422}
423
424#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
425#[allow(dead_code)] // live only via Linux aarch64 trap/hypercall path
426pub(crate) fn redirect_ucontext_to_host_exit() -> bool {
427    false
428}
429
430#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
431#[allow(dead_code)] // live only via Linux aarch64 trap/hypercall path
432pub(crate) fn exit_current_guest_worker() -> ! {
433    loop {
434        core::hint::spin_loop();
435    }
436}
437
438#[cfg(test)]
439mod tests {
440    use super::*;
441
442    #[test]
443    fn selfid_stable_on_thread() {
444        let a = thread_selfid();
445        let b = thread_selfid();
446        assert_eq!(a, b);
447        assert!(a > 0);
448    }
449
450    #[test]
451    fn alloc_tid_monotonic() {
452        let a = alloc_tid();
453        let b = alloc_tid();
454        assert!(b > a);
455    }
456}