Skip to main content

kh_runtime/thread/
tls.rs

1//! Guest↔host TLS: `TPIDR_EL0` + per-thread host snapshot.
2//!
3//! Guest code runs with TPIDR → freestanding [`GUEST_TLS_MAGIC`] block; every
4//! host entry restores the host TPIDR saved at prepare. Snapshots live in
5//! [`crate::host_slot`] and are mirrored into the guest block (A1) so enter can
6//! restore host TPIDR + alt SP without `gettid` / map locks.
7//!
8//! Guest block layout (must match `kh-libsystem`):
9//! ```text
10//! 0 magic  8 errno  16 pthread_self  24 host_tpidr  32 alt_top  40 tsd_vals
11//! ```
12//! Host owns 24/32; freestanding owns `tsd_vals`.
13#![allow(unsafe_code)]
14
15use std::sync::atomic::{AtomicU64, Ordering};
16
17use crate::cpu;
18use crate::host_slot;
19
20/// Freestanding guest TLS magic (`"KHTLS\x01"`).
21pub const GUEST_TLS_MAGIC: u64 = 0x4B48_544C_5301;
22/// Size of the guest TLS control block (one page is fine; we use 64 B).
23pub const GUEST_TLS_SIZE: usize = 64;
24/// Offset of the `errno` cell inside the guest TLS block.
25pub const GUEST_TLS_ERRNO_OFF: usize = 8;
26/// Offset of `pthread_self` pointer storage.
27pub const GUEST_TLS_PTHREAD_OFF: usize = 16;
28/// Host `TPIDR_EL0` mirror (written only by host while host TPIDR is live).
29pub const GUEST_TLS_HOST_TPIDR_OFF: usize = 24;
30/// Hypercall alt-stack top mirror (host-owned).
31pub const GUEST_TLS_ALT_TOP_OFF: usize = 32;
32/// Guest-owned per-thread `pthread` TSD array pointer (freestanding).
33pub const GUEST_TLS_TSD_VALS_OFF: usize = 40;
34
35/// Process-wide main guest TLS VA (for diagnostics / tests).
36static MAIN_GUEST_TLS: AtomicU64 = AtomicU64::new(0);
37
38/// Hypercall enter return: alt SP + guest TLS VA for leave (AAPCS64 `x0`/`x1`).
39#[repr(C)]
40#[derive(Clone, Copy, Debug, Default)]
41pub struct HyperEnterRet {
42    /// Host alt stack top, or 0 if unavailable.
43    pub alt_top: u64,
44    /// Guest `TPIDR_EL0` to restore on leave (0 if unknown).
45    pub guest_tpidr: u64,
46}
47
48/// Captures the current (host) `TPIDR_EL0` into this thread's host slot.
49///
50/// Must run on every host thread **before** guest code may `msr tpidr_el0`.
51pub fn prepare_host_meta() {
52    let host = cpu::read_tpidr_el0();
53    // May allocate map entry — must run with host TPIDR live.
54    host_slot::with_tls_init(|m| {
55        m.host_tpidr = host;
56        m.active = true;
57    });
58}
59
60/// Allocates the main-thread guest TLS block and records it in [`host_slot`].
61///
62/// **Does not** leave `TPIDR_EL0` on the guest value. Host Rust / glibc / tracing
63/// must keep host TPIDR until [`enter_guest_tls`] at the actual guest entry
64/// (`call_guest` / `jump_to_guest`). Leaving guest TPIDR active caused immediate
65/// `SIGSEGV` in host `libc` (`si_addr=0xa0`) on some Ubuntu/glibc builds when
66/// any host code touched TLS (e.g. `tracing` thread-local dispatch).
67///
68/// Returns the guest TLS VA, or `0` on allocation failure / non-Linux aarch64.
69#[must_use]
70pub fn install_main_guest_tls() -> u64 {
71    prepare_host_meta();
72    // Hypercall alt stack for the main thread while host TPIDR is live.
73    let alt = crate::thread::ensure_host_alt_stack();
74    #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
75    {
76        let Some(base) = map_guest_tls_page() else {
77            return 0;
78        };
79        init_guest_tls_block(base, 0);
80        let va = crate::host::ptr_addr_u64(base);
81        MAIN_GUEST_TLS.store(va, Ordering::Release);
82        let host = cpu::read_tpidr_el0();
83        // Record guest TSD for the boundary; keep host TPIDR for host code.
84        host_slot::with_tls_init(|m| {
85            m.guest_tpidr = va;
86            m.host_tpidr = host;
87            m.active = true;
88        });
89        publish_boundary_to_guest_tls(va, host, alt);
90        va
91    }
92    #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
93    {
94        let _ = alt;
95        0
96    }
97}
98
99/// Switches `TPIDR_EL0` to `guest_tpidr` and records it in the host slot.
100///
101/// `guest_tpidr` must point at a live guest TLS block (or 0 to skip). Host
102/// meta must already be prepared so host TPIDR can be restored later.
103///
104/// Publishes host/alt mirrors into the guest block (A1 fast enter).
105pub fn enter_guest_tls(guest_tpidr: u64) {
106    if guest_tpidr == 0 {
107        return;
108    }
109    let host = cpu::read_tpidr_el0();
110    let alt = host_slot::with_alt_mut(|cell| cell.map_or(0, |s| s.top));
111    // Still on host TPIDR here — ensure slot exists, then msr guest.
112    host_slot::with_tls_init(|m| {
113        if !m.active {
114            m.host_tpidr = host;
115            m.active = true;
116        } else if m.host_tpidr == 0 {
117            m.host_tpidr = host;
118        }
119        m.guest_tpidr = guest_tpidr;
120    });
121    let host_pub = host_slot::tls_get().host_tpidr;
122    publish_boundary_to_guest_tls(guest_tpidr, host_pub, alt);
123    // SAFETY: caller / freestanding layout owns this block for the thread life.
124    unsafe {
125        cpu::write_tpidr_el0(guest_tpidr);
126    }
127}
128
129/// Write host boundary fields into a freestanding guest TLS block.
130///
131/// **MUST** run only while host `TPIDR_EL0` is live (or with raw stores that do
132/// not depend on host TLS). Safe no-op if `guest_va` is 0 / unmapped magic.
133pub fn publish_boundary_to_guest_tls(guest_va: u64, host_tpidr: u64, alt_top: u64) {
134    if guest_va == 0 || host_tpidr == 0 {
135        return;
136    }
137    let Ok(addr) = usize::try_from(guest_va) else {
138        return;
139    };
140    // SAFETY: identity-mapped guest TLS when magic matches; otherwise skip.
141    let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
142    if magic != GUEST_TLS_MAGIC {
143        return;
144    }
145    unsafe {
146        core::ptr::with_exposed_provenance_mut::<u64>(
147            addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF),
148        )
149        .write(host_tpidr);
150        core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
151            .write(alt_top);
152    }
153}
154
155/// Update alt_top mirror for the current thread's guest TLS (host TPIDR live).
156pub fn publish_alt_top_to_current_guest(alt_top: u64) {
157    let guest = host_slot::tls_get().guest_tpidr;
158    let host = host_slot::tls_get().host_tpidr;
159    if guest != 0 && host != 0 {
160        publish_boundary_to_guest_tls(guest, host, alt_top);
161    }
162}
163
164/// Hypercall entry: restore host glibc TLS; return alt top + guest VA (`x0`/`x1`).
165///
166/// A1: prefer guest-TLS mirror (no gettid). A2 fallback: one gettid for map.
167///
168/// # Safety
169///
170/// Must be paired with [`kh_tls_leave_host`] before returning to guest.
171#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
172#[unsafe(no_mangle)]
173pub unsafe extern "C" fn kh_tls_enter_host() -> HyperEnterRet {
174    enter_host_tls_inner()
175}
176
177/// Hypercall leave: restore guest `TPIDR_EL0`.
178///
179/// `guest_tpidr` is the value parked by enter/asm (A1). Pass `0` to fall back
180/// to the gettid-keyed host slot (SIGTRAP / failed enter).
181///
182/// # Safety
183///
184/// Guest TLS block must still be mapped when `guest_tpidr != 0`.
185#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
186#[unsafe(no_mangle)]
187pub unsafe extern "C" fn kh_tls_leave_host(guest_tpidr: u64) {
188    leave_host_tls_inner(guest_tpidr);
189}
190
191/// # Safety
192///
193/// Stub on non-Linux aarch64.
194#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
195#[allow(dead_code)]
196pub unsafe extern "C" fn kh_tls_enter_host() -> HyperEnterRet {
197    HyperEnterRet::default()
198}
199
200/// # Safety
201///
202/// Stub on non-Linux aarch64.
203#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
204#[allow(dead_code)]
205pub unsafe extern "C" fn kh_tls_leave_host(_guest_tpidr: u64) {}
206
207/// Safe wrapper used from Rust trap handlers.
208#[inline]
209pub fn enter_host_tls() {
210    let _ = enter_host_tls_inner();
211}
212
213/// Safe wrapper used from Rust trap handlers (map fallback).
214#[inline]
215pub fn leave_host_tls() {
216    leave_host_tls_inner(0);
217}
218
219/// Try A1 fast path: guest TLS block holds host_tpidr + alt_top.
220///
221/// **Only** valid when `cur` is known-or-likely guest TPIDR. Rejects host glibc
222/// TLS because magic will not match [`GUEST_TLS_MAGIC`].
223fn try_enter_from_guest_tls(cur: u64) -> Option<(u64 /*host*/, u64 /*alt*/)> {
224    if cur == 0 {
225        return None;
226    }
227    let Ok(addr) = usize::try_from(cur) else {
228        return None;
229    };
230    // SAFETY: identity-map probe; magic gate before other fields.
231    let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
232    if magic != GUEST_TLS_MAGIC {
233        return None;
234    }
235    let host = unsafe {
236        core::ptr::with_exposed_provenance::<u64>(addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF))
237            .read_volatile()
238    };
239    if host == 0 || host == cur {
240        // Uninitialized mirror, or nonsense (host==guest).
241        return None;
242    }
243    let alt = unsafe {
244        core::ptr::with_exposed_provenance::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
245            .read_volatile()
246    };
247    Some((host, alt))
248}
249
250fn enter_host_tls_inner() -> HyperEnterRet {
251    let cur = cpu::read_tpidr_el0();
252
253    // --- A1 fast path: no gettid, no map Mutex ---
254    if let Some((host, alt)) = try_enter_from_guest_tls(cur) {
255        if cur != host {
256            // SAFETY: restoring host glibc TLS from guest-TLS mirror.
257            unsafe {
258                cpu::write_tpidr_el0(host);
259            }
260        }
261        let alt_top = if alt != 0 {
262            alt
263        } else {
264            // Cold alt map after host TPIDR; refresh mirror for next enter.
265            // (gettid only on this rare cold path via ensure_host_alt_stack.)
266            let top = crate::thread::ensure_host_alt_stack();
267            publish_boundary_to_guest_tls(cur, host, top);
268            top
269        };
270        // No map/gettid on the hot path — leave uses guest_tpidr from this ret.
271        return HyperEnterRet {
272            alt_top,
273            guest_tpidr: cur,
274        };
275    }
276
277    // --- A2 slow path: gettid + map ---
278    let Some(prep) = host_slot::prepare_enter_under_guest(cur) else {
279        return HyperEnterRet {
280            alt_top: 0,
281            guest_tpidr: cur,
282        };
283    };
284    if cur != prep.host_tpidr {
285        // SAFETY: restoring host glibc TLS pointer captured at prepare time.
286        unsafe {
287            cpu::write_tpidr_el0(prep.host_tpidr);
288        }
289    }
290    let alt_top = if prep.alt_top != 0 {
291        prep.alt_top
292    } else {
293        crate::thread::ensure_host_alt_stack()
294    };
295    // If we were on guest TLS without mirrors, publish for next hypercall.
296    if is_guest_tls(cur) {
297        publish_boundary_to_guest_tls(cur, prep.host_tpidr, alt_top);
298    }
299    HyperEnterRet {
300        alt_top,
301        guest_tpidr: if is_guest_tls(cur) {
302            cur
303        } else {
304            host_slot::tls_get().guest_tpidr
305        },
306    }
307}
308
309fn leave_host_tls_inner(guest_hint: u64) {
310    let guest = if guest_hint != 0 {
311        guest_hint
312    } else {
313        host_slot::with_tls_existing_mut(|m| if m.active { m.guest_tpidr } else { 0 }).unwrap_or(0)
314    };
315    if guest == 0 {
316        return;
317    }
318    // SAFETY: guest TLS block still valid for this live guest thread.
319    unsafe {
320        cpu::write_tpidr_el0(guest);
321    }
322}
323
324/// Restores host TLS and clears the active guest snapshot (worker exit).
325pub fn clear_guest_tls_on_exit() {
326    let ret = enter_host_tls_inner();
327    let _ = host_slot::with_tls_existing_mut(|m| {
328        m.guest_tpidr = 0;
329    });
330    if ret.guest_tpidr != 0 {
331        // Clear host mirrors so a reused block cannot fast-enter.
332        publish_boundary_to_guest_tls(ret.guest_tpidr, 0, 0);
333        // publish skips host==0 — force clear:
334        clear_guest_tls_mirrors(ret.guest_tpidr);
335    }
336}
337
338fn clear_guest_tls_mirrors(guest_va: u64) {
339    let Ok(addr) = usize::try_from(guest_va) else {
340        return;
341    };
342    let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
343    if magic != GUEST_TLS_MAGIC {
344        return;
345    }
346    unsafe {
347        core::ptr::with_exposed_provenance_mut::<u64>(
348            addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF),
349        )
350        .write(0);
351        core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
352            .write(0);
353    }
354}
355
356/// Current host TPIDR snapshot (0 if unprepared).
357#[must_use]
358pub fn host_tpidr() -> u64 {
359    host_slot::tls_get().host_tpidr
360}
361
362/// Current guest TPIDR snapshot (0 if none).
363#[must_use]
364pub fn guest_tpidr() -> u64 {
365    host_slot::tls_get().guest_tpidr
366}
367
368/// Main thread guest TLS VA installed by [`install_main_guest_tls`].
369#[must_use]
370pub fn main_guest_tls() -> u64 {
371    MAIN_GUEST_TLS.load(Ordering::Acquire)
372}
373
374#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
375fn map_guest_tls_page() -> Option<*mut u8> {
376    let len = GUEST_TLS_SIZE.max(64);
377    let page = crate::host::page_size().unwrap_or(4096);
378    let map_len = len.saturating_add(page.saturating_sub(1)) & !(page.saturating_sub(1));
379    let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
380    let prot = libc::PROT_READ | libc::PROT_WRITE;
381    crate::host::mmap(None, map_len, prot, flags, -1, 0)
382}
383
384/// Writes magic + errno=0 + optional pthread into a guest TLS block.
385///
386/// `base` must be 16-byte aligned and at least [`GUEST_TLS_SIZE`] bytes.
387/// Host boundary fields (`host_tpidr` / `alt_top`) start as 0 until
388/// [`publish_boundary_to_guest_tls`].
389pub fn init_guest_tls_block(base: *mut u8, pthread_va: u64) {
390    if base.is_null() {
391        return;
392    }
393    // SAFETY: caller-owned RW mapping ≥ GUEST_TLS_SIZE; mmap returns page-aligned.
394    unsafe {
395        let addr = base.addr();
396        core::ptr::with_exposed_provenance_mut::<u64>(addr).write(GUEST_TLS_MAGIC);
397        core::ptr::with_exposed_provenance_mut::<i32>(addr.saturating_add(GUEST_TLS_ERRNO_OFF))
398            .write(0);
399        core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_PTHREAD_OFF))
400            .write(pthread_va);
401        core::ptr::with_exposed_provenance_mut::<u64>(
402            addr.saturating_add(GUEST_TLS_HOST_TPIDR_OFF),
403        )
404        .write(0);
405        core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_ALT_TOP_OFF))
406            .write(0);
407        // Freestanding lazily allocates the TSD array; start null.
408        core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_TSD_VALS_OFF))
409            .write(0);
410    }
411}
412
413/// True when `tpidr` looks like a freestanding guest TLS block.
414#[must_use]
415pub fn is_guest_tls(tpidr: u64) -> bool {
416    if tpidr == 0 {
417        return false;
418    }
419    let Ok(addr) = usize::try_from(tpidr) else {
420        return false;
421    };
422    // SAFETY: best-effort identity-map probe; only used diagnostically.
423    let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
424    magic == GUEST_TLS_MAGIC
425}
426
427#[cfg(test)]
428mod tests {
429    use super::*;
430
431    #[test]
432    fn prepare_captures_host() {
433        prepare_host_meta();
434        let h = host_tpidr();
435        let _ = h;
436        assert!(host_slot::tls_get().active);
437    }
438
439    #[test]
440    fn guest_tls_fast_enter_without_gettid_map() {
441        #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
442        {
443            prepare_host_meta();
444            let host = host_tpidr();
445            let Some(base) = map_guest_tls_page() else {
446                return;
447            };
448            init_guest_tls_block(base, 0);
449            let guest = crate::host::ptr_addr_u64(base);
450            let fake_alt = 0x_aaa0_u64;
451            publish_boundary_to_guest_tls(guest, host, fake_alt);
452
453            let ret = unsafe {
454                cpu::write_tpidr_el0(guest);
455                enter_host_tls_inner()
456            };
457            assert_eq!(cpu::read_tpidr_el0(), host);
458            assert_eq!(ret.alt_top, fake_alt);
459            assert_eq!(ret.guest_tpidr, guest);
460
461            leave_host_tls_inner(ret.guest_tpidr);
462            assert_eq!(cpu::read_tpidr_el0(), guest);
463
464            clear_guest_tls_on_exit();
465            assert_eq!(cpu::read_tpidr_el0(), host);
466            let _ = crate::host::munmap(base, crate::host::page_size().unwrap_or(4096));
467        }
468    }
469
470    #[test]
471    fn enter_leave_restores_host() {
472        #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
473        {
474            prepare_host_meta();
475            let host = host_tpidr();
476            let Some(base) = map_guest_tls_page() else {
477                return;
478            };
479            init_guest_tls_block(base, 0);
480            let fake_guest = crate::host::ptr_addr_u64(base);
481            host_slot::with_tls_init(|m| {
482                m.guest_tpidr = fake_guest;
483            });
484            publish_boundary_to_guest_tls(fake_guest, host, 0);
485            // SAFETY: TPIDR briefly non-host; A1 mirror or gettid recovers host.
486            let after_enter = unsafe {
487                cpu::write_tpidr_el0(fake_guest);
488                enter_host_tls();
489                cpu::read_tpidr_el0()
490            };
491            assert_eq!(after_enter, host);
492
493            let after_leave = {
494                leave_host_tls();
495                let v = cpu::read_tpidr_el0();
496                enter_host_tls();
497                v
498            };
499            assert_eq!(after_leave, fake_guest);
500
501            clear_guest_tls_on_exit();
502            assert_eq!(cpu::read_tpidr_el0(), host);
503            let _ = crate::host::munmap(base, crate::host::page_size().unwrap_or(4096));
504        }
505    }
506}