#![allow(unsafe_code)]
use std::cell::Cell;
use std::sync::atomic::{AtomicU64, Ordering};
thread_local! {
static TID: Cell<u64> = const { Cell::new(0) };
}
#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
thread_local! {
static HOST_ALT: Cell<Option<HostAltStack>> = const { Cell::new(None) };
}
static NEXT_TID: AtomicU64 = AtomicU64::new(1);
#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
const HOST_ALT_STACK_SIZE: usize = 512 * 1024;
#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
#[derive(Clone, Copy)]
struct HostAltStack {
base: *mut u8,
len: usize,
top: u64,
}
#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
#[unsafe(no_mangle)]
pub unsafe extern "C" fn kh_host_alt_sp() -> u64 {
HOST_ALT.with(|cell| {
if let Some(s) = cell.get() {
return s.top;
}
match map_host_alt_stack() {
Some(s) => {
cell.set(Some(s));
s.top
}
None => 0,
}
})
}
#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
fn map_host_alt_stack() -> Option<HostAltStack> {
let len = HOST_ALT_STACK_SIZE;
let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
let prot = libc::PROT_READ | libc::PROT_WRITE;
let ptr = crate::host::mmap(None, len, prot, flags, -1, 0)?;
let base_u = crate::host::ptr_addr_u64(ptr);
let top = base_u.saturating_add(u64::try_from(len).unwrap_or(0)) & !0xF;
if top <= base_u {
let _ = crate::host::munmap(ptr, len);
return None;
}
Some(HostAltStack {
base: ptr,
len,
top,
})
}
#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
pub(crate) fn drop_host_alt_stack() {
HOST_ALT.with(|cell| {
if let Some(s) = cell.take() {
let _ = crate::host::munmap(s.base, s.len);
}
});
}
#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
#[allow(dead_code)] pub(crate) fn drop_host_alt_stack() {}
static LIVE_WORKERS: AtomicU64 = AtomicU64::new(0);
#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
const EAGAIN: i64 = 35;
pub fn reset_thread_runtime() {
NEXT_TID.store(1, Ordering::SeqCst);
LIVE_WORKERS.store(0, Ordering::SeqCst);
}
#[must_use]
pub fn alloc_tid() -> u64 {
NEXT_TID.fetch_add(1, Ordering::SeqCst)
}
#[must_use]
pub fn thread_selfid() -> u64 {
TID.with(|c| {
let mut v = c.get();
if v == 0 {
v = alloc_tid();
c.set(v);
}
v
})
}
#[must_use]
pub fn live_workers() -> u64 {
LIVE_WORKERS.load(Ordering::SeqCst)
}
#[derive(Debug, Clone, Copy)]
pub struct GuestThreadStart {
pub entry: u64,
pub sp: u64,
pub pthread: u64,
pub port: u64,
pub func: u64,
pub func_arg: u64,
}
pub fn spawn_guest_thread(start: GuestThreadStart) -> Result<(), i64> {
#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
{
let _ = start;
Err(crate::syscall::ENOSYS)
}
#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
{
linux_spawn(start)
}
}
#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
mod linux {
use super::{EAGAIN, GuestThreadStart, LIVE_WORKERS, thread_selfid};
use std::cell::Cell;
use std::sync::atomic::{AtomicU32, Ordering};
const KH_THREAD_MAGIC: u64 = 0x4B48_5054_4852_4401;
const KH_THREAD_DONE_OFF: usize = 8;
#[derive(Clone, Copy)]
struct HostExitFrame {
pc: u64,
sp: u64,
}
thread_local! {
static HOST_EXIT: Cell<Option<HostExitFrame>> = const { Cell::new(None) };
static GUEST_PTHREAD: Cell<u64> = const { Cell::new(0) };
}
pub(super) fn linux_spawn(start: GuestThreadStart) -> Result<(), i64> {
if start.entry == 0 || start.sp == 0 || !start.sp.is_multiple_of(16) {
return Err(crate::syscall::EINVAL);
}
let builder = std::thread::Builder::new().name("kh-guest".into());
match builder.spawn(move || guest_worker_main(start)) {
Ok(_handle) => {
LIVE_WORKERS.fetch_add(1, Ordering::SeqCst);
Ok(())
}
Err(_) => Err(EAGAIN),
}
}
fn guest_worker_main(start: GuestThreadStart) {
let _ = thread_selfid();
GUEST_PTHREAD.set(start.pthread);
unblock_sigtrap();
let host_sp = read_sp();
let frame_sp = host_sp.saturating_sub(4096) & !0xF;
set_host_exit_frame(host_thread_exit_pc(), frame_sp);
let result = unsafe {
crate::entry::jump_to_guest_args(
start.entry,
start.sp,
start.pthread,
start.port,
start.func,
start.func_arg,
)
};
if let Err(err) = result {
tracing::error!(?err, "guest worker failed to enter");
}
finish_worker_on_host();
}
fn unblock_sigtrap() {
unsafe {
let mut set: libc::sigset_t = std::mem::zeroed();
libc::sigemptyset(std::ptr::addr_of_mut!(set));
libc::sigaddset(std::ptr::addr_of_mut!(set), libc::SIGTRAP);
libc::pthread_sigmask(
libc::SIG_UNBLOCK,
std::ptr::addr_of!(set),
std::ptr::null_mut(),
);
}
}
fn read_sp() -> u64 {
let sp: u64;
unsafe {
std::arch::asm!("mov {}, sp", out(reg) sp, options(nomem, nostack, preserves_flags));
}
sp
}
#[allow(clippy::as_conversions, function_casts_as_integer)]
fn host_thread_exit_pc() -> u64 {
u64::try_from(host_thread_exit as usize).unwrap_or(0)
}
unsafe extern "C" fn host_thread_exit() -> ! {
finish_worker_on_host();
}
fn finish_worker_on_host() -> ! {
clear_host_exit_frame();
publish_guest_join_done();
GUEST_PTHREAD.set(0);
super::drop_host_alt_stack();
worker_finished();
unsafe {
libc::pthread_exit(std::ptr::null_mut());
}
}
fn publish_guest_join_done() {
let pthread = GUEST_PTHREAD.get();
if pthread == 0 {
return;
}
let base = usize::try_from(pthread).unwrap_or(0);
if base == 0 {
return;
}
let magic_ptr = std::ptr::with_exposed_provenance::<u64>(base);
let magic = unsafe { core::ptr::read_volatile(magic_ptr) };
if magic != KH_THREAD_MAGIC {
tracing::warn!(
pthread = format_args!("{pthread:#x}"),
magic = format_args!("{magic:#x}"),
"guest pthread magic mismatch; skip join publish"
);
return;
}
let done_addr = base.saturating_add(KH_THREAD_DONE_OFF);
let done_ptr = std::ptr::with_exposed_provenance_mut::<AtomicU32>(done_addr);
unsafe {
(*done_ptr).store(1, Ordering::Release);
}
futex_wake(done_ptr.cast::<u32>(), i32::MAX);
}
fn futex_wake(addr: *mut u32, n: i32) {
let _ = unsafe {
libc::syscall(
libc::SYS_futex,
addr,
1_i32, n,
core::ptr::null::<libc::timespec>(),
core::ptr::null_mut::<u32>(),
0_i32,
)
};
}
fn worker_finished() {
let _ = LIVE_WORKERS.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| {
Some(n.saturating_sub(1))
});
}
pub(crate) fn exit_worker_now() -> ! {
let frame = HOST_EXIT.take();
if let Some(frame) = frame {
unsafe {
std::arch::asm!(
"mov sp, {sp}",
"br {pc}",
sp = in(reg) frame.sp,
pc = in(reg) frame.pc,
options(noreturn),
);
}
}
finish_worker_on_host();
}
fn set_host_exit_frame(pc: u64, sp: u64) {
HOST_EXIT.set(Some(HostExitFrame { pc, sp }));
}
fn clear_host_exit_frame() {
HOST_EXIT.set(None);
}
pub(crate) fn redirect_ucontext_to_host_exit(m: &mut libc::mcontext_t) -> bool {
HOST_EXIT.with(|cell| {
let Some(frame) = cell.get() else {
return false;
};
cell.set(None);
m.pc = frame.pc;
m.sp = frame.sp;
true
})
}
}
#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
use linux::linux_spawn;
#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
pub(crate) use linux::redirect_ucontext_to_host_exit;
#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
pub(crate) fn exit_current_guest_worker() -> ! {
linux::exit_worker_now();
}
#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
#[allow(dead_code)] pub(crate) fn redirect_ucontext_to_host_exit() -> bool {
false
}
#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
#[allow(dead_code)] pub(crate) fn exit_current_guest_worker() -> ! {
loop {
core::hint::spin_loop();
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn selfid_stable_on_thread() {
let a = thread_selfid();
let b = thread_selfid();
assert_eq!(a, b);
assert!(a > 0);
}
#[test]
fn alloc_tid_monotonic() {
let a = alloc_tid();
let b = alloc_tid();
assert!(b > a);
}
}