1#![allow(unsafe_code)]
13
14use std::sync::atomic::{AtomicU64, Ordering};
15
16use crate::host_slot;
17
18static NEXT_TID: AtomicU64 = AtomicU64::new(1);
20
21#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
23const HOST_ALT_STACK_SIZE: usize = 512 * 1024;
24
25#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
38#[unsafe(no_mangle)]
39pub unsafe extern "C" fn kh_host_alt_sp() -> u64 {
40 ensure_host_alt_stack()
41}
42
43#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
48#[must_use]
49pub fn ensure_host_alt_stack() -> u64 {
50 let top = host_slot::with_alt_mut(|cell| {
51 if let Some(s) = *cell {
52 return s.top;
53 }
54 match map_host_alt_stack() {
55 Some(s) => {
56 let t = s.top;
57 *cell = Some(s);
58 t
59 }
60 None => 0,
61 }
62 });
63 if top != 0 {
65 crate::tls::publish_alt_top_to_current_guest(top);
66 }
67 top
68}
69
70#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
71#[must_use]
72pub fn ensure_host_alt_stack() -> u64 {
73 0
74}
75
76#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
77fn map_host_alt_stack() -> Option<host_slot::AltStackSlot> {
78 let len = HOST_ALT_STACK_SIZE;
79 let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
80 let prot = libc::PROT_READ | libc::PROT_WRITE;
81 let ptr = mmap_anon_raw(len, prot, flags)?;
84 let base_u = crate::host::ptr_addr_u64(ptr);
85 let top = base_u.saturating_add(u64::try_from(len).unwrap_or(0)) & !0xF;
86 if top <= base_u {
87 let _ = crate::host::munmap(ptr, len);
88 return None;
89 }
90 Some(host_slot::AltStackSlot {
91 base: ptr,
92 len,
93 top,
94 })
95}
96
97#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
99fn mmap_anon_raw(len: usize, prot: libc::c_int, flags: libc::c_int) -> Option<*mut u8> {
100 let raw = unsafe {
102 libc::syscall(
103 libc::SYS_mmap,
104 core::ptr::null_mut::<libc::c_void>(),
105 len,
106 prot,
107 flags,
108 -1_i32,
109 0_i64,
110 )
111 };
112 if raw < 0 {
113 return None;
114 }
115 let addr = usize::try_from(raw).ok()?;
116 Some(core::ptr::with_exposed_provenance_mut(addr))
117}
118
119#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
121pub(crate) fn drop_host_alt_stack() {
122 host_slot::with_alt_mut(|cell| {
123 if let Some(s) = cell.take() {
124 let _ = crate::host::munmap(s.base, s.len);
125 }
126 });
127}
128
129#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
130#[allow(dead_code)] pub(crate) fn drop_host_alt_stack() {}
132
133static LIVE_WORKERS: AtomicU64 = AtomicU64::new(0);
135
136#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
138const EAGAIN: i64 = 35;
139
140pub fn reset_thread_runtime() {
142 NEXT_TID.store(1, Ordering::SeqCst);
143 LIVE_WORKERS.store(0, Ordering::SeqCst);
144}
145
146#[must_use]
148pub fn alloc_tid() -> u64 {
149 NEXT_TID.fetch_add(1, Ordering::SeqCst)
150}
151
152#[must_use]
156pub fn thread_selfid() -> u64 {
157 host_slot::with_worker_mut(|w| {
158 if w.guest_tid == 0 {
159 w.guest_tid = alloc_tid();
160 }
161 w.guest_tid
162 })
163}
164
165#[must_use]
167pub fn live_workers() -> u64 {
168 LIVE_WORKERS.load(Ordering::SeqCst)
169}
170
171#[derive(Debug, Clone, Copy)]
173pub struct GuestThreadStart {
174 pub entry: u64,
176 pub sp: u64,
178 pub pthread: u64,
180 pub port: u64,
182 pub func: u64,
184 pub func_arg: u64,
186}
187
188pub fn spawn_guest_thread(start: GuestThreadStart) -> Result<(), i64> {
190 #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
191 {
192 let _ = start;
193 Err(crate::syscall::ENOSYS)
194 }
195
196 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
197 {
198 linux_spawn(start)
199 }
200}
201
202#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
203mod linux {
204 use super::{EAGAIN, GuestThreadStart, LIVE_WORKERS, thread_selfid};
205 use crate::host_slot;
206 use std::sync::atomic::{AtomicU32, Ordering};
207
208 const KH_THREAD_MAGIC: u64 = 0x4B48_5054_4852_4401;
220 const KH_THREAD_DONE_OFF: usize = 8;
221
222 pub(super) fn linux_spawn(start: GuestThreadStart) -> Result<(), i64> {
223 if start.entry == 0 || start.sp == 0 || !start.sp.is_multiple_of(16) {
224 return Err(crate::syscall::EINVAL);
225 }
226
227 let builder = std::thread::Builder::new().name("kh-guest".into());
228 match builder.spawn(move || guest_worker_main(start)) {
229 Ok(_handle) => {
230 LIVE_WORKERS.fetch_add(1, Ordering::SeqCst);
231 Ok(())
232 }
233 Err(_) => Err(EAGAIN),
234 }
235 }
236
237 fn guest_worker_main(start: GuestThreadStart) {
238 let _ = thread_selfid();
239 host_slot::with_worker_mut(|w| {
240 w.guest_pthread = start.pthread;
241 });
242
243 crate::tls::prepare_host_meta();
245 let _ = super::ensure_host_alt_stack();
248
249 unblock_sigtrap();
253
254 let host_sp = crate::cpu::read_sp();
255 let frame_sp = host_sp.saturating_sub(4096) & !0xF;
257 set_host_exit_frame(host_thread_exit_pc(), frame_sp);
258
259 if let Some(tsd) = guest_tsd_from_pthread(start.pthread) {
262 crate::tls::enter_guest_tls(tsd);
263 }
264
265 let result = unsafe {
268 crate::entry::jump_to_guest_args(
269 start.entry,
270 start.sp,
271 start.pthread,
272 start.port,
273 start.func,
274 start.func_arg,
275 )
276 };
277 if let Err(err) = result {
278 tracing::error!(?err, "guest worker failed to enter");
279 }
280 finish_worker_on_host();
282 }
283
284 const KH_THREAD_TSD_OFF: usize = 56;
286
287 fn guest_tsd_from_pthread(pthread: u64) -> Option<u64> {
288 let base = usize::try_from(pthread).ok()?;
289 if base == 0 {
290 return None;
291 }
292 let magic_ptr = std::ptr::with_exposed_provenance::<u64>(base);
294 let magic = unsafe { core::ptr::read_volatile(magic_ptr) };
295 if magic != KH_THREAD_MAGIC {
296 return None;
297 }
298 let tsd_ptr =
299 std::ptr::with_exposed_provenance::<u64>(base.saturating_add(KH_THREAD_TSD_OFF));
300 let tsd = unsafe { core::ptr::read_volatile(tsd_ptr) };
301 if tsd == 0 {
302 return None;
303 }
304 Some(tsd)
305 }
306
307 fn unblock_sigtrap() {
308 unsafe {
311 let mut set: libc::sigset_t = std::mem::zeroed();
312 libc::sigemptyset(std::ptr::addr_of_mut!(set));
313 libc::sigaddset(std::ptr::addr_of_mut!(set), libc::SIGTRAP);
314 libc::pthread_sigmask(
315 libc::SIG_UNBLOCK,
316 std::ptr::addr_of!(set),
317 std::ptr::null_mut(),
318 );
319 }
320 }
321
322 #[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
323 fn host_thread_exit_pc() -> u64 {
324 u64::try_from(host_thread_exit as usize).unwrap_or(0)
325 }
326
327 unsafe extern "C" fn host_thread_exit() -> ! {
329 finish_worker_on_host();
330 }
331
332 fn finish_worker_on_host() -> ! {
337 clear_host_exit_frame();
338 crate::tls::clear_guest_tls_on_exit();
340 publish_guest_join_done();
341 host_slot::with_worker_mut(|w| {
342 w.guest_pthread = 0;
343 });
344 super::drop_host_alt_stack();
345 host_slot::clear_current();
346 worker_finished();
347 unsafe {
354 libc::syscall(libc::SYS_exit, 0);
355 }
356 loop {
358 core::hint::spin_loop();
359 }
360 }
361
362 fn publish_guest_join_done() {
369 let pthread = host_slot::worker_get().guest_pthread;
370 if pthread == 0 {
371 return;
372 }
373 let base = usize::try_from(pthread).unwrap_or(0);
374 if base == 0 {
375 return;
376 }
377 let magic_ptr = std::ptr::with_exposed_provenance::<u64>(base);
379 let magic = unsafe { core::ptr::read_volatile(magic_ptr) };
380 if magic != KH_THREAD_MAGIC {
381 tracing::warn!(
382 pthread = format_args!("{pthread:#x}"),
383 magic = format_args!("{magic:#x}"),
384 "guest pthread magic mismatch; skip join publish"
385 );
386 return;
387 }
388 let done_addr = base.saturating_add(KH_THREAD_DONE_OFF);
389 let done_ptr = std::ptr::with_exposed_provenance_mut::<AtomicU32>(done_addr);
390 unsafe {
392 (*done_ptr).store(1, Ordering::Release);
393 }
394 futex_wake(done_ptr.cast::<u32>(), i32::MAX);
395 }
396
397 fn futex_wake(addr: *mut u32, n: i32) {
398 let _ = unsafe {
401 libc::syscall(
402 libc::SYS_futex,
403 addr,
404 129_i32, n,
406 core::ptr::null::<libc::timespec>(),
407 core::ptr::null_mut::<u32>(),
408 0_i32,
409 )
410 };
411 }
412
413 fn worker_finished() {
414 let _ = LIVE_WORKERS.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| {
415 Some(n.saturating_sub(1))
416 });
417 }
418
419 pub(crate) fn exit_worker_now() -> ! {
425 let (pc, sp, has) = host_slot::with_worker_mut(|w| {
426 let has = w.has_exit;
427 let pc = w.exit_pc;
428 let sp = w.exit_sp;
429 w.has_exit = false;
430 (pc, sp, has)
431 });
432 if has {
433 unsafe {
437 std::arch::asm!(
438 "mov sp, {sp}",
439 "mov x29, xzr",
440 "br {pc}",
441 sp = in(reg) sp,
442 pc = in(reg) pc,
443 options(noreturn),
444 );
445 }
446 }
447 finish_worker_on_host();
449 }
450
451 fn set_host_exit_frame(pc: u64, sp: u64) {
452 host_slot::with_worker_mut(|w| {
453 w.exit_pc = pc;
454 w.exit_sp = sp;
455 w.has_exit = true;
456 });
457 }
458
459 fn clear_host_exit_frame() {
460 host_slot::with_worker_mut(|w| {
461 w.has_exit = false;
462 });
463 }
464
465 pub(crate) fn redirect_ucontext_to_host_exit(m: &mut libc::mcontext_t) -> bool {
467 host_slot::with_worker_mut(|w| {
468 if !w.has_exit {
469 return false;
470 }
471 w.has_exit = false;
473 m.pc = w.exit_pc;
474 m.sp = w.exit_sp;
475 m.regs[29] = 0;
477 true
478 })
479 }
480}
481
482#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
483use linux::linux_spawn;
484
485#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
489pub(crate) use linux::redirect_ucontext_to_host_exit;
490
491#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
496pub(crate) fn exit_current_guest_worker() -> ! {
497 linux::exit_worker_now();
498}
499
500#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
501#[allow(dead_code)] pub(crate) fn redirect_ucontext_to_host_exit() -> bool {
503 false
504}
505
506#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
507#[allow(dead_code)] pub(crate) fn exit_current_guest_worker() -> ! {
509 loop {
510 core::hint::spin_loop();
511 }
512}
513
514#[cfg(test)]
515mod tests {
516 use super::*;
517
518 #[test]
519 fn selfid_stable_on_thread() {
520 let a = thread_selfid();
521 let b = thread_selfid();
522 assert_eq!(a, b);
523 assert!(a > 0);
524 }
525
526 #[test]
527 fn alloc_tid_monotonic() {
528 let a = alloc_tid();
529 let b = alloc_tid();
530 assert!(b > a);
531 }
532}