Skip to main content

kh_runtime/
tls.rs

1//! Guest↔host TLS boundary: `TPIDR_EL0` + per-thread host snapshot.
2//!
3//! On Linux aarch64, host glibc and Darwin guest both use `TPIDR_EL0`. While
4//! guest code runs the register points at a freestanding [`GUEST_TLS_MAGIC`]
5//! block; every host entry (hypercall / SIGTRAP) restores the host value saved
6//! at thread prepare time.
7//!
8//! **Host snapshots live in [`crate::host_slot`]** (gettid map) **and** are
9//! mirrored into the guest TLS block (A1) so hypercall enter can restore host
10//! TPIDR + alt SP **without** `gettid` / map `Mutex` / host `thread_local!`.
11//!
12//! Layout of the guest block (must match `kh-libsystem`):
13//! ```text
14//! offset 0:  magic: u64   == GUEST_TLS_MAGIC
15//! offset 8:  errno: i32
16//! offset 12: pad
17//! offset 16: pthread_self: u64  (guest pthread_t VA, optional)
18//! offset 24: host_tpidr: u64    (host-owned mirror; A1)
19//! offset 32: alt_top: u64       (host-owned mirror; A1)
20//! ```
21#![allow(unsafe_code)]
22
23use std::sync::atomic::{AtomicU64, Ordering};
24
25use crate::cpu;
26use crate::host_slot;
27
28/// Freestanding guest TLS magic (`"KHTLS\x01"`).
29pub const GUEST_TLS_MAGIC: u64 = 0x4B48_544C_5301;
30/// Size of the guest TLS control block (one page is fine; we use 64 B).
31pub const GUEST_TLS_SIZE: usize = 64;
32/// Offset of the `errno` cell inside the guest TLS block.
33pub const GUEST_TLS_ERRNO_OFF: usize = 8;
34/// Offset of `pthread_self` pointer storage.
35pub const GUEST_TLS_PTHREAD_OFF: usize = 16;
36/// Host `TPIDR_EL0` mirror (written only by host while host TPIDR is live).
37pub const GUEST_TLS_HOST_TPIDR_OFF: usize = 24;
38/// Hypercall alt-stack top mirror (host-owned).
39pub const GUEST_TLS_ALT_TOP_OFF: usize = 32;
40
41/// Process-wide main guest TLS VA (for diagnostics / tests).
42static MAIN_GUEST_TLS: AtomicU64 = AtomicU64::new(0);
43
44/// Hypercall enter return: alt SP + guest TLS VA for leave (AAPCS64 `x0`/`x1`).
45#[repr(C)]
46#[derive(Clone, Copy, Debug, Default)]
47pub struct HyperEnterRet {
48    /// Host alt stack top, or 0 if unavailable.
49    pub alt_top: u64,
50    /// Guest `TPIDR_EL0` to restore on leave (0 if unknown).
51    pub guest_tpidr: u64,
52}
53
54/// Captures the current (host) `TPIDR_EL0` into this thread's host slot.
55///
56/// Must run on every host thread **before** guest code may `msr tpidr_el0`.
57pub fn prepare_host_meta() {
58    let host = cpu::read_tpidr_el0();
59    // May allocate map entry — must run with host TPIDR live.
60    host_slot::with_tls_init(|m| {
61        m.host_tpidr = host;
62        m.active = true;
63    });
64}
65
66/// Allocates the main-thread guest TLS block and records it in [`host_slot`].
67///
68/// **Does not** leave `TPIDR_EL0` on the guest value. Host Rust / glibc / tracing
69/// must keep host TPIDR until [`enter_guest_tls`] at the actual guest entry
70/// (`call_guest` / `jump_to_guest`). Leaving guest TPIDR active caused immediate
71/// `SIGSEGV` in host `libc` (`si_addr=0xa0`) on some Ubuntu/glibc builds when
72/// any host code touched TLS (e.g. `tracing` thread-local dispatch).
73///
74/// Returns the guest TLS VA, or `0` on allocation failure / non-Linux aarch64.
75#[must_use]
76pub fn install_main_guest_tls() -> u64 {
77    prepare_host_meta();
78    // Hypercall alt stack for the main thread while host TPIDR is live.
79    let alt = crate::thread::ensure_host_alt_stack();
80    #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
81    {
82        let Some(base) = map_guest_tls_page() else {
83            return 0;
84        };
85        init_guest_tls_block(base, 0);
86        let va = crate::host::ptr_addr_u64(base);
87        MAIN_GUEST_TLS.store(va, Ordering::Release);
88        let host = cpu::read_tpidr_el0();
89        // Record guest TSD for the boundary; keep host TPIDR for host code.
90        host_slot::with_tls_init(|m| {
91            m.guest_tpidr = va;
92            m.host_tpidr = host;
93            m.active = true;
94        });
95        publish_boundary_to_guest_tls(va, host, alt);
96        va
97    }
98    #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
99    {
100        let _ = alt;
101        0
102    }
103}
104
105/// Switches `TPIDR_EL0` to `guest_tpidr` and records it in the host slot.
106///
107/// `guest_tpidr` must point at a live guest TLS block (or 0 to skip). Host
108/// meta must already be prepared so host TPIDR can be restored later.
109///
110/// Publishes host/alt mirrors into the guest block (A1 fast enter).
111pub fn enter_guest_tls(guest_tpidr: u64) {
112    if guest_tpidr == 0 {
113        return;
114    }
115    let host = cpu::read_tpidr_el0();
116    let alt = host_slot::with_alt_mut(|cell| cell.map_or(0, |s| s.top));
117    // Still on host TPIDR here — ensure slot exists, then msr guest.
118    host_slot::with_tls_init(|m| {
119        if !m.active {
120            m.host_tpidr = host;
121            m.active = true;
122        } else if m.host_tpidr == 0 {
123            m.host_tpidr = host;
124        }
125        m.guest_tpidr = guest_tpidr;
126    });
127    let host_pub = host_slot::tls_get().host_tpidr;
128    publish_boundary_to_guest_tls(guest_tpidr, host_pub, alt);
129    // SAFETY: caller / freestanding layout owns this block for the thread life.
130    unsafe {
131        cpu::write_tpidr_el0(guest_tpidr);
132    }
133}
134
135/// Write host boundary fields into a freestanding guest TLS block.
136///
137/// **MUST** run only while host `TPIDR_EL0` is live (or with raw stores that do
138/// not depend on host TLS). Safe no-op if `guest_va` is 0 / unmapped magic.
139pub fn publish_boundary_to_guest_tls(guest_va: u64, host_tpidr: u64, alt_top: u64) {
140    if guest_va == 0 || host_tpidr == 0 {
141        return;
142    }
143    let Ok(addr) = usize::try_from(guest_va) else {
144        return;
145    };
146    // SAFETY: identity-mapped guest TLS when magic matches; otherwise skip.
147    let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
148    if magic != GUEST_TLS_MAGIC {
149        return;
150    }
151    unsafe {
152        core::ptr::with_exposed_provenance_mut::<u64>(
153            addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF),
154        )
155        .write(host_tpidr);
156        core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
157            .write(alt_top);
158    }
159}
160
161/// Update alt_top mirror for the current thread's guest TLS (host TPIDR live).
162pub fn publish_alt_top_to_current_guest(alt_top: u64) {
163    let guest = host_slot::tls_get().guest_tpidr;
164    let host = host_slot::tls_get().host_tpidr;
165    if guest != 0 && host != 0 {
166        publish_boundary_to_guest_tls(guest, host, alt_top);
167    }
168}
169
170/// Hypercall entry: restore host glibc TLS; return alt top + guest VA (`x0`/`x1`).
171///
172/// A1: prefer guest-TLS mirror (no gettid). A2 fallback: one gettid for map.
173///
174/// # Safety
175///
176/// Must be paired with [`kh_tls_leave_host`] before returning to guest.
177#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
178#[unsafe(no_mangle)]
179pub unsafe extern "C" fn kh_tls_enter_host() -> HyperEnterRet {
180    enter_host_tls_inner()
181}
182
183/// Hypercall leave: restore guest `TPIDR_EL0`.
184///
185/// `guest_tpidr` is the value parked by enter/asm (A1). Pass `0` to fall back
186/// to the gettid-keyed host slot (SIGTRAP / failed enter).
187///
188/// # Safety
189///
190/// Guest TLS block must still be mapped when `guest_tpidr != 0`.
191#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
192#[unsafe(no_mangle)]
193pub unsafe extern "C" fn kh_tls_leave_host(guest_tpidr: u64) {
194    leave_host_tls_inner(guest_tpidr);
195}
196
197/// # Safety
198///
199/// Stub on non-Linux aarch64.
200#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
201#[allow(dead_code)]
202pub unsafe extern "C" fn kh_tls_enter_host() -> HyperEnterRet {
203    HyperEnterRet::default()
204}
205
206/// # Safety
207///
208/// Stub on non-Linux aarch64.
209#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
210#[allow(dead_code)]
211pub unsafe extern "C" fn kh_tls_leave_host(_guest_tpidr: u64) {}
212
213/// Safe wrapper used from Rust trap handlers.
214#[inline]
215pub fn enter_host_tls() {
216    let _ = enter_host_tls_inner();
217}
218
219/// Safe wrapper used from Rust trap handlers (map fallback).
220#[inline]
221pub fn leave_host_tls() {
222    leave_host_tls_inner(0);
223}
224
225/// Try A1 fast path: guest TLS block holds host_tpidr + alt_top.
226///
227/// **Only** valid when `cur` is known-or-likely guest TPIDR. Rejects host glibc
228/// TLS because magic will not match [`GUEST_TLS_MAGIC`].
229fn try_enter_from_guest_tls(cur: u64) -> Option<(u64 /*host*/, u64 /*alt*/)> {
230    if cur == 0 {
231        return None;
232    }
233    let Ok(addr) = usize::try_from(cur) else {
234        return None;
235    };
236    // SAFETY: identity-map probe; magic gate before other fields.
237    let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
238    if magic != GUEST_TLS_MAGIC {
239        return None;
240    }
241    let host = unsafe {
242        core::ptr::with_exposed_provenance::<u64>(addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF))
243            .read_volatile()
244    };
245    if host == 0 || host == cur {
246        // Uninitialized mirror, or nonsense (host==guest).
247        return None;
248    }
249    let alt = unsafe {
250        core::ptr::with_exposed_provenance::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
251            .read_volatile()
252    };
253    Some((host, alt))
254}
255
256fn enter_host_tls_inner() -> HyperEnterRet {
257    let cur = cpu::read_tpidr_el0();
258
259    // --- A1 fast path: no gettid, no map Mutex ---
260    if let Some((host, alt)) = try_enter_from_guest_tls(cur) {
261        if cur != host {
262            // SAFETY: restoring host glibc TLS from guest-TLS mirror.
263            unsafe {
264                cpu::write_tpidr_el0(host);
265            }
266        }
267        let alt_top = if alt != 0 {
268            alt
269        } else {
270            // Cold alt map after host TPIDR; refresh mirror for next enter.
271            // (gettid only on this rare cold path via ensure_host_alt_stack.)
272            let top = crate::thread::ensure_host_alt_stack();
273            publish_boundary_to_guest_tls(cur, host, top);
274            top
275        };
276        // No map/gettid on the hot path — leave uses guest_tpidr from this ret.
277        return HyperEnterRet {
278            alt_top,
279            guest_tpidr: cur,
280        };
281    }
282
283    // --- A2 slow path: gettid + map ---
284    let Some(prep) = host_slot::prepare_enter_under_guest(cur) else {
285        return HyperEnterRet {
286            alt_top: 0,
287            guest_tpidr: cur,
288        };
289    };
290    if cur != prep.host_tpidr {
291        // SAFETY: restoring host glibc TLS pointer captured at prepare time.
292        unsafe {
293            cpu::write_tpidr_el0(prep.host_tpidr);
294        }
295    }
296    let alt_top = if prep.alt_top != 0 {
297        prep.alt_top
298    } else {
299        crate::thread::ensure_host_alt_stack()
300    };
301    // If we were on guest TLS without mirrors, publish for next hypercall.
302    if is_guest_tls(cur) {
303        publish_boundary_to_guest_tls(cur, prep.host_tpidr, alt_top);
304    }
305    HyperEnterRet {
306        alt_top,
307        guest_tpidr: if is_guest_tls(cur) {
308            cur
309        } else {
310            host_slot::tls_get().guest_tpidr
311        },
312    }
313}
314
315fn leave_host_tls_inner(guest_hint: u64) {
316    let guest = if guest_hint != 0 {
317        guest_hint
318    } else {
319        host_slot::with_tls_existing_mut(|m| if m.active { m.guest_tpidr } else { 0 }).unwrap_or(0)
320    };
321    if guest == 0 {
322        return;
323    }
324    // SAFETY: guest TLS block still valid for this live guest thread.
325    unsafe {
326        cpu::write_tpidr_el0(guest);
327    }
328}
329
330/// Restores host TLS and clears the active guest snapshot (worker exit).
331pub fn clear_guest_tls_on_exit() {
332    let ret = enter_host_tls_inner();
333    let _ = host_slot::with_tls_existing_mut(|m| {
334        m.guest_tpidr = 0;
335    });
336    if ret.guest_tpidr != 0 {
337        // Clear host mirrors so a reused block cannot fast-enter.
338        publish_boundary_to_guest_tls(ret.guest_tpidr, 0, 0);
339        // publish skips host==0 — force clear:
340        clear_guest_tls_mirrors(ret.guest_tpidr);
341    }
342}
343
344fn clear_guest_tls_mirrors(guest_va: u64) {
345    let Ok(addr) = usize::try_from(guest_va) else {
346        return;
347    };
348    let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
349    if magic != GUEST_TLS_MAGIC {
350        return;
351    }
352    unsafe {
353        core::ptr::with_exposed_provenance_mut::<u64>(
354            addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF),
355        )
356        .write(0);
357        core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
358            .write(0);
359    }
360}
361
362/// Current host TPIDR snapshot (0 if unprepared).
363#[must_use]
364pub fn host_tpidr() -> u64 {
365    host_slot::tls_get().host_tpidr
366}
367
368/// Current guest TPIDR snapshot (0 if none).
369#[must_use]
370pub fn guest_tpidr() -> u64 {
371    host_slot::tls_get().guest_tpidr
372}
373
374/// Main thread guest TLS VA installed by [`install_main_guest_tls`].
375#[must_use]
376pub fn main_guest_tls() -> u64 {
377    MAIN_GUEST_TLS.load(Ordering::Acquire)
378}
379
380#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
381fn map_guest_tls_page() -> Option<*mut u8> {
382    let len = GUEST_TLS_SIZE.max(64);
383    let page = crate::host::page_size().unwrap_or(4096);
384    let map_len = len.saturating_add(page.saturating_sub(1)) & !(page.saturating_sub(1));
385    let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
386    let prot = libc::PROT_READ | libc::PROT_WRITE;
387    crate::host::mmap(None, map_len, prot, flags, -1, 0)
388}
389
390/// Writes magic + errno=0 + optional pthread into a guest TLS block.
391///
392/// `base` must be 16-byte aligned and at least [`GUEST_TLS_SIZE`] bytes.
393/// Host boundary fields (`host_tpidr` / `alt_top`) start as 0 until
394/// [`publish_boundary_to_guest_tls`].
395pub fn init_guest_tls_block(base: *mut u8, pthread_va: u64) {
396    if base.is_null() {
397        return;
398    }
399    // SAFETY: caller-owned RW mapping ≥ GUEST_TLS_SIZE; mmap returns page-aligned.
400    unsafe {
401        let addr = base.addr();
402        core::ptr::with_exposed_provenance_mut::<u64>(addr).write(GUEST_TLS_MAGIC);
403        core::ptr::with_exposed_provenance_mut::<i32>(addr.saturating_add(GUEST_TLS_ERRNO_OFF))
404            .write(0);
405        core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_PTHREAD_OFF))
406            .write(pthread_va);
407        core::ptr::with_exposed_provenance_mut::<u64>(
408            addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF),
409        )
410        .write(0);
411        core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
412            .write(0);
413    }
414}
415
416/// True when `tpidr` looks like a freestanding guest TLS block.
417#[must_use]
418pub fn is_guest_tls(tpidr: u64) -> bool {
419    if tpidr == 0 {
420        return false;
421    }
422    let Ok(addr) = usize::try_from(tpidr) else {
423        return false;
424    };
425    // SAFETY: best-effort identity-map probe; only used diagnostically.
426    let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
427    magic == GUEST_TLS_MAGIC
428}
429
430#[cfg(test)]
431mod tests {
432    use super::*;
433
434    #[test]
435    fn prepare_captures_host() {
436        prepare_host_meta();
437        let h = host_tpidr();
438        let _ = h;
439        assert!(host_slot::tls_get().active);
440    }
441
442    #[test]
443    fn guest_tls_fast_enter_without_gettid_map() {
444        #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
445        {
446            prepare_host_meta();
447            let host = host_tpidr();
448            let Some(base) = map_guest_tls_page() else {
449                return;
450            };
451            init_guest_tls_block(base, 0);
452            let guest = crate::host::ptr_addr_u64(base);
453            let fake_alt = 0x_aaa0_u64;
454            publish_boundary_to_guest_tls(guest, host, fake_alt);
455
456            let ret = unsafe {
457                cpu::write_tpidr_el0(guest);
458                enter_host_tls_inner()
459            };
460            assert_eq!(cpu::read_tpidr_el0(), host);
461            assert_eq!(ret.alt_top, fake_alt);
462            assert_eq!(ret.guest_tpidr, guest);
463
464            leave_host_tls_inner(ret.guest_tpidr);
465            assert_eq!(cpu::read_tpidr_el0(), guest);
466
467            clear_guest_tls_on_exit();
468            assert_eq!(cpu::read_tpidr_el0(), host);
469            let _ = crate::host::munmap(base, crate::host::page_size().unwrap_or(4096));
470        }
471    }
472
473    #[test]
474    fn enter_leave_restores_host() {
475        #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
476        {
477            prepare_host_meta();
478            let host = host_tpidr();
479            let Some(base) = map_guest_tls_page() else {
480                return;
481            };
482            init_guest_tls_block(base, 0);
483            let fake_guest = crate::host::ptr_addr_u64(base);
484            host_slot::with_tls_init(|m| {
485                m.guest_tpidr = fake_guest;
486            });
487            publish_boundary_to_guest_tls(fake_guest, host, 0);
488            // SAFETY: TPIDR briefly non-host; A1 mirror or gettid recovers host.
489            let after_enter = unsafe {
490                cpu::write_tpidr_el0(fake_guest);
491                enter_host_tls();
492                cpu::read_tpidr_el0()
493            };
494            assert_eq!(after_enter, host);
495
496            let after_leave = {
497                leave_host_tls();
498                let v = cpu::read_tpidr_el0();
499                enter_host_tls();
500                v
501            };
502            assert_eq!(after_leave, fake_guest);
503
504            clear_guest_tls_on_exit();
505            assert_eq!(cpu::read_tpidr_el0(), host);
506            let _ = crate::host::munmap(base, crate::host::page_size().unwrap_or(4096));
507        }
508    }
509}