#![allow(unsafe_code)]
use std::sync::atomic::{AtomicU64, Ordering};
use crate::cpu;
use crate::host_slot;
pub const GUEST_TLS_MAGIC: u64 = 0x4B48_544C_5301;
pub const GUEST_TLS_SIZE: usize = 64;
pub const GUEST_TLS_ERRNO_OFF: usize = 8;
pub const GUEST_TLS_PTHREAD_OFF: usize = 16;
static MAIN_GUEST_TLS: AtomicU64 = AtomicU64::new(0);
pub fn prepare_host_meta() {
let host = cpu::read_tpidr_el0();
host_slot::with_tls_init(|m| {
m.host_tpidr = host;
m.active = true;
});
}
#[must_use]
pub fn install_main_guest_tls() -> u64 {
prepare_host_meta();
let _ = crate::thread::ensure_host_alt_stack();
#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
{
let Some(base) = map_guest_tls_page() else {
return 0;
};
init_guest_tls_block(base, 0);
let va = crate::host::ptr_addr_u64(base);
MAIN_GUEST_TLS.store(va, Ordering::Release);
enter_guest_tls(va);
va
}
#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
{
0
}
}
pub fn enter_guest_tls(guest_tpidr: u64) {
if guest_tpidr == 0 {
return;
}
host_slot::with_tls_init(|m| {
if !m.active {
m.host_tpidr = cpu::read_tpidr_el0();
m.active = true;
}
m.guest_tpidr = guest_tpidr;
});
unsafe {
cpu::write_tpidr_el0(guest_tpidr);
}
}
#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
#[unsafe(no_mangle)]
pub unsafe extern "C" fn kh_tls_enter_host() {
enter_host_tls_inner();
}
#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
#[unsafe(no_mangle)]
pub unsafe extern "C" fn kh_tls_leave_host() {
leave_host_tls_inner();
}
#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
#[allow(dead_code)]
pub unsafe extern "C" fn kh_tls_enter_host() {}
#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
#[allow(dead_code)]
pub unsafe extern "C" fn kh_tls_leave_host() {}
#[inline]
pub fn enter_host_tls() {
enter_host_tls_inner();
}
#[inline]
pub fn leave_host_tls() {
leave_host_tls_inner();
}
fn enter_host_tls_inner() {
let cur = cpu::read_tpidr_el0();
let host = host_slot::with_tls_existing_mut(|m| {
if !m.active {
m.host_tpidr = cur;
m.active = true;
return m.host_tpidr;
}
if cur != m.host_tpidr {
m.guest_tpidr = cur;
}
m.host_tpidr
});
let Some(host) = host else {
return;
};
if cur != host {
unsafe {
cpu::write_tpidr_el0(host);
}
}
}
fn leave_host_tls_inner() {
let guest = host_slot::with_tls_existing_mut(|m| {
if m.active {
m.guest_tpidr
} else {
0
}
})
.unwrap_or(0);
if guest == 0 {
return;
}
unsafe {
cpu::write_tpidr_el0(guest);
}
}
pub fn clear_guest_tls_on_exit() {
enter_host_tls_inner();
let _ = host_slot::with_tls_existing_mut(|m| {
m.guest_tpidr = 0;
});
}
#[must_use]
pub fn host_tpidr() -> u64 {
host_slot::tls_get().host_tpidr
}
#[must_use]
pub fn guest_tpidr() -> u64 {
host_slot::tls_get().guest_tpidr
}
#[must_use]
pub fn main_guest_tls() -> u64 {
MAIN_GUEST_TLS.load(Ordering::Acquire)
}
#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
fn map_guest_tls_page() -> Option<*mut u8> {
let len = GUEST_TLS_SIZE.max(64);
let page = crate::host::page_size().unwrap_or(4096);
let map_len = len.saturating_add(page.saturating_sub(1)) & !(page.saturating_sub(1));
let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
let prot = libc::PROT_READ | libc::PROT_WRITE;
crate::host::mmap(None, map_len, prot, flags, -1, 0)
}
pub fn init_guest_tls_block(base: *mut u8, pthread_va: u64) {
if base.is_null() {
return;
}
unsafe {
let addr = base.addr();
core::ptr::with_exposed_provenance_mut::<u64>(addr).write(GUEST_TLS_MAGIC);
core::ptr::with_exposed_provenance_mut::<i32>(addr.saturating_add(GUEST_TLS_ERRNO_OFF))
.write(0);
core::ptr::with_exposed_provenance_mut::<u64>(addr.saturating_add(GUEST_TLS_PTHREAD_OFF))
.write(pthread_va);
}
}
#[must_use]
pub fn is_guest_tls(tpidr: u64) -> bool {
if tpidr == 0 {
return false;
}
let Ok(addr) = usize::try_from(tpidr) else {
return false;
};
let magic = unsafe { core::ptr::with_exposed_provenance::<u64>(addr).read_volatile() };
magic == GUEST_TLS_MAGIC
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn prepare_captures_host() {
prepare_host_meta();
let h = host_tpidr();
let _ = h;
assert!(host_slot::tls_get().active);
}
#[test]
fn enter_leave_restores_host() {
#[cfg(target_arch = "aarch64")]
{
prepare_host_meta();
let host = host_tpidr();
let fake_guest = 0x1000_u64;
host_slot::with_tls_init(|m| {
m.guest_tpidr = fake_guest;
});
let after_enter = unsafe {
cpu::write_tpidr_el0(fake_guest);
enter_host_tls();
cpu::read_tpidr_el0()
};
assert_eq!(after_enter, host);
let after_leave = {
leave_host_tls();
let v = cpu::read_tpidr_el0();
enter_host_tls();
v
};
assert_eq!(after_leave, fake_guest);
clear_guest_tls_on_exit();
assert_eq!(cpu::read_tpidr_el0(), host);
}
}
}