Skip to main content

kh_runtime/
tls.rs

1//! Guest↔host TLS boundary: `TPIDR_EL0` + per-thread host snapshot.
2//!
3//! On Linux aarch64, host glibc and Darwin guest both use `TPIDR_EL0`. While
4//! guest code runs the register points at a freestanding [`GUEST_TLS_MAGIC`]
5//! block; every host entry (hypercall / SIGTRAP) restores the host value saved
6//! at thread prepare time.
7//!
8//! **Host snapshots live in [`crate::host_slot`]** (gettid map) **and** are
9//! mirrored into the guest TLS block (A1) so hypercall enter can restore host
10//! TPIDR + alt SP **without** `gettid` / map `Mutex` / host `thread_local!`.
11//!
12//! Layout of the guest block (must match `kh-libsystem`):
13//! ```text
14//! offset 0:  magic: u64   == GUEST_TLS_MAGIC
15//! offset 8:  errno: i32
16//! offset 12: pad
17//! offset 16: pthread_self: u64  (guest pthread_t VA, optional)
18//! offset 24: host_tpidr: u64    (host-owned mirror; A1)
19//! offset 32: alt_top: u64       (host-owned mirror; A1)
20//! offset 40: tsd_vals: u64      (guest-owned; per-thread pthread TSD array)
21//! ```
22//! Host only publishes offsets 24/32; freestanding owns `tsd_vals`.
23#![allow(unsafe_code)]
24
25use std::sync::atomic::{AtomicU64, Ordering};
26
27use crate::cpu;
28use crate::host_slot;
29
30/// Freestanding guest TLS magic (`"KHTLS\x01"`).
31pub const GUEST_TLS_MAGIC: u64 = 0x4B48_544C_5301;
32/// Size of the guest TLS control block (one page is fine; we use 64 B).
33pub const GUEST_TLS_SIZE: usize = 64;
34/// Offset of the `errno` cell inside the guest TLS block.
35pub const GUEST_TLS_ERRNO_OFF: usize = 8;
36/// Offset of `pthread_self` pointer storage.
37pub const GUEST_TLS_PTHREAD_OFF: usize = 16;
38/// Host `TPIDR_EL0` mirror (written only by host while host TPIDR is live).
39pub const GUEST_TLS_HOST_TPIDR_OFF: usize = 24;
40/// Hypercall alt-stack top mirror (host-owned).
41pub const GUEST_TLS_ALT_TOP_OFF: usize = 32;
42/// Guest-owned per-thread `pthread` TSD array pointer (freestanding).
43pub const GUEST_TLS_TSD_VALS_OFF: usize = 40;
44
45/// Process-wide main guest TLS VA (for diagnostics / tests).
46static MAIN_GUEST_TLS: AtomicU64 = AtomicU64::new(0);
47
48/// Hypercall enter return: alt SP + guest TLS VA for leave (AAPCS64 `x0`/`x1`).
49#[repr(C)]
50#[derive(Clone, Copy, Debug, Default)]
51pub struct HyperEnterRet {
52    /// Host alt stack top, or 0 if unavailable.
53    pub alt_top: u64,
54    /// Guest `TPIDR_EL0` to restore on leave (0 if unknown).
55    pub guest_tpidr: u64,
56}
57
58/// Captures the current (host) `TPIDR_EL0` into this thread's host slot.
59///
60/// Must run on every host thread **before** guest code may `msr tpidr_el0`.
61pub fn prepare_host_meta() {
62    let host = cpu::read_tpidr_el0();
63    // May allocate map entry — must run with host TPIDR live.
64    host_slot::with_tls_init(|m| {
65        m.host_tpidr = host;
66        m.active = true;
67    });
68}
69
70/// Allocates the main-thread guest TLS block and records it in [`host_slot`].
71///
72/// **Does not** leave `TPIDR_EL0` on the guest value. Host Rust / glibc / tracing
73/// must keep host TPIDR until [`enter_guest_tls`] at the actual guest entry
74/// (`call_guest` / `jump_to_guest`). Leaving guest TPIDR active caused immediate
75/// `SIGSEGV` in host `libc` (`si_addr=0xa0`) on some Ubuntu/glibc builds when
76/// any host code touched TLS (e.g. `tracing` thread-local dispatch).
77///
78/// Returns the guest TLS VA, or `0` on allocation failure / non-Linux aarch64.
79#[must_use]
80pub fn install_main_guest_tls() -> u64 {
81    prepare_host_meta();
82    // Hypercall alt stack for the main thread while host TPIDR is live.
83    let alt = crate::thread::ensure_host_alt_stack();
84    #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
85    {
86        let Some(base) = map_guest_tls_page() else {
87            return 0;
88        };
89        init_guest_tls_block(base, 0);
90        let va = crate::host::ptr_addr_u64(base);
91        MAIN_GUEST_TLS.store(va, Ordering::Release);
92        let host = cpu::read_tpidr_el0();
93        // Record guest TSD for the boundary; keep host TPIDR for host code.
94        host_slot::with_tls_init(|m| {
95            m.guest_tpidr = va;
96            m.host_tpidr = host;
97            m.active = true;
98        });
99        publish_boundary_to_guest_tls(va, host, alt);
100        va
101    }
102    #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
103    {
104        let _ = alt;
105        0
106    }
107}
108
109/// Switches `TPIDR_EL0` to `guest_tpidr` and records it in the host slot.
110///
111/// `guest_tpidr` must point at a live guest TLS block (or 0 to skip). Host
112/// meta must already be prepared so host TPIDR can be restored later.
113///
114/// Publishes host/alt mirrors into the guest block (A1 fast enter).
115pub fn enter_guest_tls(guest_tpidr: u64) {
116    if guest_tpidr == 0 {
117        return;
118    }
119    let host = cpu::read_tpidr_el0();
120    let alt = host_slot::with_alt_mut(|cell| cell.map_or(0, |s| s.top));
121    // Still on host TPIDR here — ensure slot exists, then msr guest.
122    host_slot::with_tls_init(|m| {
123        if !m.active {
124            m.host_tpidr = host;
125            m.active = true;
126        } else if m.host_tpidr == 0 {
127            m.host_tpidr = host;
128        }
129        m.guest_tpidr = guest_tpidr;
130    });
131    let host_pub = host_slot::tls_get().host_tpidr;
132    publish_boundary_to_guest_tls(guest_tpidr, host_pub, alt);
133    // SAFETY: caller / freestanding layout owns this block for the thread life.
134    unsafe {
135        cpu::write_tpidr_el0(guest_tpidr);
136    }
137}
138
139/// Write host boundary fields into a freestanding guest TLS block.
140///
141/// **MUST** run only while host `TPIDR_EL0` is live (or with raw stores that do
142/// not depend on host TLS). Safe no-op if `guest_va` is 0 / unmapped magic.
143pub fn publish_boundary_to_guest_tls(guest_va: u64, host_tpidr: u64, alt_top: u64) {
144    if guest_va == 0 || host_tpidr == 0 {
145        return;
146    }
147    let Ok(addr) = usize::try_from(guest_va) else {
148        return;
149    };
150    // SAFETY: identity-mapped guest TLS when magic matches; otherwise skip.
151    let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
152    if magic != GUEST_TLS_MAGIC {
153        return;
154    }
155    unsafe {
156        core::ptr::with_exposed_provenance_mut::<u64>(
157            addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF),
158        )
159        .write(host_tpidr);
160        core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
161            .write(alt_top);
162    }
163}
164
165/// Update alt_top mirror for the current thread's guest TLS (host TPIDR live).
166pub fn publish_alt_top_to_current_guest(alt_top: u64) {
167    let guest = host_slot::tls_get().guest_tpidr;
168    let host = host_slot::tls_get().host_tpidr;
169    if guest != 0 && host != 0 {
170        publish_boundary_to_guest_tls(guest, host, alt_top);
171    }
172}
173
174/// Hypercall entry: restore host glibc TLS; return alt top + guest VA (`x0`/`x1`).
175///
176/// A1: prefer guest-TLS mirror (no gettid). A2 fallback: one gettid for map.
177///
178/// # Safety
179///
180/// Must be paired with [`kh_tls_leave_host`] before returning to guest.
181#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
182#[unsafe(no_mangle)]
183pub unsafe extern "C" fn kh_tls_enter_host() -> HyperEnterRet {
184    enter_host_tls_inner()
185}
186
187/// Hypercall leave: restore guest `TPIDR_EL0`.
188///
189/// `guest_tpidr` is the value parked by enter/asm (A1). Pass `0` to fall back
190/// to the gettid-keyed host slot (SIGTRAP / failed enter).
191///
192/// # Safety
193///
194/// Guest TLS block must still be mapped when `guest_tpidr != 0`.
195#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
196#[unsafe(no_mangle)]
197pub unsafe extern "C" fn kh_tls_leave_host(guest_tpidr: u64) {
198    leave_host_tls_inner(guest_tpidr);
199}
200
201/// # Safety
202///
203/// Stub on non-Linux aarch64.
204#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
205#[allow(dead_code)]
206pub unsafe extern "C" fn kh_tls_enter_host() -> HyperEnterRet {
207    HyperEnterRet::default()
208}
209
210/// # Safety
211///
212/// Stub on non-Linux aarch64.
213#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
214#[allow(dead_code)]
215pub unsafe extern "C" fn kh_tls_leave_host(_guest_tpidr: u64) {}
216
217/// Safe wrapper used from Rust trap handlers.
218#[inline]
219pub fn enter_host_tls() {
220    let _ = enter_host_tls_inner();
221}
222
223/// Safe wrapper used from Rust trap handlers (map fallback).
224#[inline]
225pub fn leave_host_tls() {
226    leave_host_tls_inner(0);
227}
228
229/// Try A1 fast path: guest TLS block holds host_tpidr + alt_top.
230///
231/// **Only** valid when `cur` is known-or-likely guest TPIDR. Rejects host glibc
232/// TLS because magic will not match [`GUEST_TLS_MAGIC`].
233fn try_enter_from_guest_tls(cur: u64) -> Option<(u64 /*host*/, u64 /*alt*/)> {
234    if cur == 0 {
235        return None;
236    }
237    let Ok(addr) = usize::try_from(cur) else {
238        return None;
239    };
240    // SAFETY: identity-map probe; magic gate before other fields.
241    let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
242    if magic != GUEST_TLS_MAGIC {
243        return None;
244    }
245    let host = unsafe {
246        core::ptr::with_exposed_provenance::<u64>(addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF))
247            .read_volatile()
248    };
249    if host == 0 || host == cur {
250        // Uninitialized mirror, or nonsense (host==guest).
251        return None;
252    }
253    let alt = unsafe {
254        core::ptr::with_exposed_provenance::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
255            .read_volatile()
256    };
257    Some((host, alt))
258}
259
260fn enter_host_tls_inner() -> HyperEnterRet {
261    let cur = cpu::read_tpidr_el0();
262
263    // --- A1 fast path: no gettid, no map Mutex ---
264    if let Some((host, alt)) = try_enter_from_guest_tls(cur) {
265        if cur != host {
266            // SAFETY: restoring host glibc TLS from guest-TLS mirror.
267            unsafe {
268                cpu::write_tpidr_el0(host);
269            }
270        }
271        let alt_top = if alt != 0 {
272            alt
273        } else {
274            // Cold alt map after host TPIDR; refresh mirror for next enter.
275            // (gettid only on this rare cold path via ensure_host_alt_stack.)
276            let top = crate::thread::ensure_host_alt_stack();
277            publish_boundary_to_guest_tls(cur, host, top);
278            top
279        };
280        // No map/gettid on the hot path — leave uses guest_tpidr from this ret.
281        return HyperEnterRet {
282            alt_top,
283            guest_tpidr: cur,
284        };
285    }
286
287    // --- A2 slow path: gettid + map ---
288    let Some(prep) = host_slot::prepare_enter_under_guest(cur) else {
289        return HyperEnterRet {
290            alt_top: 0,
291            guest_tpidr: cur,
292        };
293    };
294    if cur != prep.host_tpidr {
295        // SAFETY: restoring host glibc TLS pointer captured at prepare time.
296        unsafe {
297            cpu::write_tpidr_el0(prep.host_tpidr);
298        }
299    }
300    let alt_top = if prep.alt_top != 0 {
301        prep.alt_top
302    } else {
303        crate::thread::ensure_host_alt_stack()
304    };
305    // If we were on guest TLS without mirrors, publish for next hypercall.
306    if is_guest_tls(cur) {
307        publish_boundary_to_guest_tls(cur, prep.host_tpidr, alt_top);
308    }
309    HyperEnterRet {
310        alt_top,
311        guest_tpidr: if is_guest_tls(cur) {
312            cur
313        } else {
314            host_slot::tls_get().guest_tpidr
315        },
316    }
317}
318
319fn leave_host_tls_inner(guest_hint: u64) {
320    let guest = if guest_hint != 0 {
321        guest_hint
322    } else {
323        host_slot::with_tls_existing_mut(|m| if m.active { m.guest_tpidr } else { 0 }).unwrap_or(0)
324    };
325    if guest == 0 {
326        return;
327    }
328    // SAFETY: guest TLS block still valid for this live guest thread.
329    unsafe {
330        cpu::write_tpidr_el0(guest);
331    }
332}
333
334/// Restores host TLS and clears the active guest snapshot (worker exit).
335pub fn clear_guest_tls_on_exit() {
336    let ret = enter_host_tls_inner();
337    let _ = host_slot::with_tls_existing_mut(|m| {
338        m.guest_tpidr = 0;
339    });
340    if ret.guest_tpidr != 0 {
341        // Clear host mirrors so a reused block cannot fast-enter.
342        publish_boundary_to_guest_tls(ret.guest_tpidr, 0, 0);
343        // publish skips host==0 — force clear:
344        clear_guest_tls_mirrors(ret.guest_tpidr);
345    }
346}
347
348fn clear_guest_tls_mirrors(guest_va: u64) {
349    let Ok(addr) = usize::try_from(guest_va) else {
350        return;
351    };
352    let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
353    if magic != GUEST_TLS_MAGIC {
354        return;
355    }
356    unsafe {
357        core::ptr::with_exposed_provenance_mut::<u64>(
358            addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF),
359        )
360        .write(0);
361        core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
362            .write(0);
363    }
364}
365
366/// Current host TPIDR snapshot (0 if unprepared).
367#[must_use]
368pub fn host_tpidr() -> u64 {
369    host_slot::tls_get().host_tpidr
370}
371
372/// Current guest TPIDR snapshot (0 if none).
373#[must_use]
374pub fn guest_tpidr() -> u64 {
375    host_slot::tls_get().guest_tpidr
376}
377
378/// Main thread guest TLS VA installed by [`install_main_guest_tls`].
379#[must_use]
380pub fn main_guest_tls() -> u64 {
381    MAIN_GUEST_TLS.load(Ordering::Acquire)
382}
383
384#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
385fn map_guest_tls_page() -> Option<*mut u8> {
386    let len = GUEST_TLS_SIZE.max(64);
387    let page = crate::host::page_size().unwrap_or(4096);
388    let map_len = len.saturating_add(page.saturating_sub(1)) & !(page.saturating_sub(1));
389    let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
390    let prot = libc::PROT_READ | libc::PROT_WRITE;
391    crate::host::mmap(None, map_len, prot, flags, -1, 0)
392}
393
394/// Writes magic + errno=0 + optional pthread into a guest TLS block.
395///
396/// `base` must be 16-byte aligned and at least [`GUEST_TLS_SIZE`] bytes.
397/// Host boundary fields (`host_tpidr` / `alt_top`) start as 0 until
398/// [`publish_boundary_to_guest_tls`].
399pub fn init_guest_tls_block(base: *mut u8, pthread_va: u64) {
400    if base.is_null() {
401        return;
402    }
403    // SAFETY: caller-owned RW mapping ≥ GUEST_TLS_SIZE; mmap returns page-aligned.
404    unsafe {
405        let addr = base.addr();
406        core::ptr::with_exposed_provenance_mut::<u64>(addr).write(GUEST_TLS_MAGIC);
407        core::ptr::with_exposed_provenance_mut::<i32>(addr.saturating_add(GUEST_TLS_ERRNO_OFF))
408            .write(0);
409        core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_PTHREAD_OFF))
410            .write(pthread_va);
411        core::ptr::with_exposed_provenance_mut::<u64>(
412            addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF),
413        )
414        .write(0);
415        core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
416            .write(0);
417        // Freestanding lazily allocates the TSD array; start null.
418        core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_TSD_VALS_OFF))
419            .write(0);
420    }
421}
422
423/// True when `tpidr` looks like a freestanding guest TLS block.
424#[must_use]
425pub fn is_guest_tls(tpidr: u64) -> bool {
426    if tpidr == 0 {
427        return false;
428    }
429    let Ok(addr) = usize::try_from(tpidr) else {
430        return false;
431    };
432    // SAFETY: best-effort identity-map probe; only used diagnostically.
433    let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
434    magic == GUEST_TLS_MAGIC
435}
436
437#[cfg(test)]
438mod tests {
439    use super::*;
440
441    #[test]
442    fn prepare_captures_host() {
443        prepare_host_meta();
444        let h = host_tpidr();
445        let _ = h;
446        assert!(host_slot::tls_get().active);
447    }
448
449    #[test]
450    fn guest_tls_fast_enter_without_gettid_map() {
451        #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
452        {
453            prepare_host_meta();
454            let host = host_tpidr();
455            let Some(base) = map_guest_tls_page() else {
456                return;
457            };
458            init_guest_tls_block(base, 0);
459            let guest = crate::host::ptr_addr_u64(base);
460            let fake_alt = 0x_aaa0_u64;
461            publish_boundary_to_guest_tls(guest, host, fake_alt);
462
463            let ret = unsafe {
464                cpu::write_tpidr_el0(guest);
465                enter_host_tls_inner()
466            };
467            assert_eq!(cpu::read_tpidr_el0(), host);
468            assert_eq!(ret.alt_top, fake_alt);
469            assert_eq!(ret.guest_tpidr, guest);
470
471            leave_host_tls_inner(ret.guest_tpidr);
472            assert_eq!(cpu::read_tpidr_el0(), guest);
473
474            clear_guest_tls_on_exit();
475            assert_eq!(cpu::read_tpidr_el0(), host);
476            let _ = crate::host::munmap(base, crate::host::page_size().unwrap_or(4096));
477        }
478    }
479
480    #[test]
481    fn enter_leave_restores_host() {
482        #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
483        {
484            prepare_host_meta();
485            let host = host_tpidr();
486            let Some(base) = map_guest_tls_page() else {
487                return;
488            };
489            init_guest_tls_block(base, 0);
490            let fake_guest = crate::host::ptr_addr_u64(base);
491            host_slot::with_tls_init(|m| {
492                m.guest_tpidr = fake_guest;
493            });
494            publish_boundary_to_guest_tls(fake_guest, host, 0);
495            // SAFETY: TPIDR briefly non-host; A1 mirror or gettid recovers host.
496            let after_enter = unsafe {
497                cpu::write_tpidr_el0(fake_guest);
498                enter_host_tls();
499                cpu::read_tpidr_el0()
500            };
501            assert_eq!(after_enter, host);
502
503            let after_leave = {
504                leave_host_tls();
505                let v = cpu::read_tpidr_el0();
506                enter_host_tls();
507                v
508            };
509            assert_eq!(after_leave, fake_guest);
510
511            clear_guest_tls_on_exit();
512            assert_eq!(cpu::read_tpidr_el0(), host);
513            let _ = crate::host::munmap(base, crate::host::page_size().unwrap_or(4096));
514        }
515    }
516}