1#![allow(unsafe_code)]
17
18use std::sync::atomic::{AtomicU64, Ordering};
19
20use crate::host_slot;
21
22static NEXT_TID: AtomicU64 = AtomicU64::new(1);
24
25#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
27const HOST_ALT_STACK_SIZE: usize = 512 * 1024;
28
29#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
42#[unsafe(no_mangle)]
43pub unsafe extern "C" fn kh_host_alt_sp() -> u64 {
44 ensure_host_alt_stack()
45}
46
47#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
52#[must_use]
53pub fn ensure_host_alt_stack() -> u64 {
54 let top = host_slot::with_alt_mut(|cell| {
55 if let Some(s) = *cell {
56 return s.top;
57 }
58 match map_host_alt_stack() {
59 Some(s) => {
60 let t = s.top;
61 *cell = Some(s);
62 t
63 }
64 None => 0,
65 }
66 });
67 if top != 0 {
69 crate::tls::publish_alt_top_to_current_guest(top);
70 }
71 top
72}
73
74#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
75#[must_use]
76pub fn ensure_host_alt_stack() -> u64 {
77 0
78}
79
80#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
81fn map_host_alt_stack() -> Option<host_slot::AltStackSlot> {
82 let len = HOST_ALT_STACK_SIZE;
83 let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
84 let prot = libc::PROT_READ | libc::PROT_WRITE;
85 let ptr = mmap_anon_raw(len, prot, flags)?;
88 let base_u = crate::host::ptr_addr_u64(ptr);
89 let top = base_u.saturating_add(u64::try_from(len).unwrap_or(0)) & !0xF;
90 if top <= base_u {
91 let _ = crate::host::munmap(ptr, len);
92 return None;
93 }
94 Some(host_slot::AltStackSlot {
95 base: ptr,
96 len,
97 top,
98 })
99}
100
101#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
103fn mmap_anon_raw(len: usize, prot: libc::c_int, flags: libc::c_int) -> Option<*mut u8> {
104 let raw = unsafe {
106 libc::syscall(
107 libc::SYS_mmap,
108 core::ptr::null_mut::<libc::c_void>(),
109 len,
110 prot,
111 flags,
112 -1_i32,
113 0_i64,
114 )
115 };
116 if raw < 0 {
117 return None;
118 }
119 let addr = usize::try_from(raw).ok()?;
120 Some(core::ptr::with_exposed_provenance_mut(addr))
121}
122
123#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
125pub(crate) fn drop_host_alt_stack() {
126 host_slot::with_alt_mut(|cell| {
127 if let Some(s) = cell.take() {
128 let _ = crate::host::munmap(s.base, s.len);
129 }
130 });
131}
132
133#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
134#[allow(dead_code)] pub(crate) fn drop_host_alt_stack() {}
136
137static LIVE_WORKERS: AtomicU64 = AtomicU64::new(0);
139
140#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
142const EAGAIN: i64 = 35;
143
144pub fn reset_thread_runtime() {
146 NEXT_TID.store(1, Ordering::SeqCst);
147 LIVE_WORKERS.store(0, Ordering::SeqCst);
148}
149
150#[must_use]
152pub fn alloc_tid() -> u64 {
153 NEXT_TID.fetch_add(1, Ordering::SeqCst)
154}
155
156#[must_use]
160pub fn thread_selfid() -> u64 {
161 host_slot::with_worker_mut(|w| {
162 if w.guest_tid == 0 {
163 w.guest_tid = alloc_tid();
164 }
165 w.guest_tid
166 })
167}
168
169#[must_use]
171pub fn live_workers() -> u64 {
172 LIVE_WORKERS.load(Ordering::SeqCst)
173}
174
175#[derive(Debug, Clone, Copy)]
177pub struct GuestThreadStart {
178 pub entry: u64,
180 pub sp: u64,
182 pub pthread: u64,
184 pub port: u64,
186 pub func: u64,
188 pub func_arg: u64,
190}
191
192pub fn spawn_guest_thread(start: GuestThreadStart) -> Result<(), i64> {
194 #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
195 {
196 let _ = start;
197 Err(crate::syscall::ENOSYS)
198 }
199
200 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
201 {
202 linux_spawn(start)
203 }
204}
205
206#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
207mod linux {
208 use super::{EAGAIN, GuestThreadStart, LIVE_WORKERS, thread_selfid};
209 use crate::host_slot;
210 use std::sync::atomic::{AtomicU32, Ordering};
211
212 const KH_THREAD_MAGIC: u64 = 0x4B48_5054_4852_4401;
224 const KH_THREAD_DONE_OFF: usize = 8;
225
226 pub(super) fn linux_spawn(start: GuestThreadStart) -> Result<(), i64> {
227 if start.entry == 0 || start.sp == 0 || !start.sp.is_multiple_of(16) {
228 return Err(crate::syscall::EINVAL);
229 }
230
231 let builder = std::thread::Builder::new().name("kh-guest".into());
232 match builder.spawn(move || guest_worker_main(start)) {
233 Ok(_handle) => {
234 LIVE_WORKERS.fetch_add(1, Ordering::SeqCst);
235 Ok(())
236 }
237 Err(_) => Err(EAGAIN),
238 }
239 }
240
241 fn guest_worker_main(start: GuestThreadStart) {
242 let _ = thread_selfid();
243 host_slot::with_worker_mut(|w| {
244 w.guest_pthread = start.pthread;
245 });
246
247 crate::tls::prepare_host_meta();
249 let _ = super::ensure_host_alt_stack();
252
253 unblock_sigtrap();
257
258 let host_sp = crate::cpu::read_sp();
259 let frame_sp = host_sp.saturating_sub(4096) & !0xF;
261 set_host_exit_frame(host_thread_exit_pc(), frame_sp);
262
263 if let Some(tsd) = guest_tsd_from_pthread(start.pthread) {
266 crate::tls::enter_guest_tls(tsd);
267 }
268
269 let result = unsafe {
272 crate::entry::jump_to_guest_args(
273 start.entry,
274 start.sp,
275 start.pthread,
276 start.port,
277 start.func,
278 start.func_arg,
279 )
280 };
281 if let Err(err) = result {
282 tracing::error!(?err, "guest worker failed to enter");
283 }
284 finish_worker_on_host();
286 }
287
288 const KH_THREAD_TSD_OFF: usize = 56;
290
291 fn guest_tsd_from_pthread(pthread: u64) -> Option<u64> {
292 let base = usize::try_from(pthread).ok()?;
293 if base == 0 {
294 return None;
295 }
296 let magic_ptr = std::ptr::with_exposed_provenance::<u64>(base);
298 let magic = unsafe { core::ptr::read_volatile(magic_ptr) };
299 if magic != KH_THREAD_MAGIC {
300 return None;
301 }
302 let tsd_ptr =
303 std::ptr::with_exposed_provenance::<u64>(base.saturating_add(KH_THREAD_TSD_OFF));
304 let tsd = unsafe { core::ptr::read_volatile(tsd_ptr) };
305 if tsd == 0 {
306 return None;
307 }
308 Some(tsd)
309 }
310
311 fn unblock_sigtrap() {
312 unsafe {
315 let mut set: libc::sigset_t = std::mem::zeroed();
316 libc::sigemptyset(std::ptr::addr_of_mut!(set));
317 libc::sigaddset(std::ptr::addr_of_mut!(set), libc::SIGTRAP);
318 libc::pthread_sigmask(
319 libc::SIG_UNBLOCK,
320 std::ptr::addr_of!(set),
321 std::ptr::null_mut(),
322 );
323 }
324 }
325
326 #[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
327 fn host_thread_exit_pc() -> u64 {
328 u64::try_from(host_thread_exit as usize).unwrap_or(0)
329 }
330
331 unsafe extern "C" fn host_thread_exit() -> ! {
333 finish_worker_on_host();
334 }
335
336 fn finish_worker_on_host() -> ! {
341 clear_host_exit_frame();
342 crate::tls::clear_guest_tls_on_exit();
344 publish_guest_join_done();
345 host_slot::with_worker_mut(|w| {
346 w.guest_pthread = 0;
347 });
348 super::drop_host_alt_stack();
349 host_slot::clear_current();
350 worker_finished();
351 unsafe {
358 libc::syscall(libc::SYS_exit, 0);
359 }
360 loop {
362 core::hint::spin_loop();
363 }
364 }
365
366 fn publish_guest_join_done() {
373 let pthread = host_slot::worker_get().guest_pthread;
374 if pthread == 0 {
375 return;
376 }
377 let base = usize::try_from(pthread).unwrap_or(0);
378 if base == 0 {
379 return;
380 }
381 let magic_ptr = std::ptr::with_exposed_provenance::<u64>(base);
383 let magic = unsafe { core::ptr::read_volatile(magic_ptr) };
384 if magic != KH_THREAD_MAGIC {
385 tracing::warn!(
386 pthread = format_args!("{pthread:#x}"),
387 magic = format_args!("{magic:#x}"),
388 "guest pthread magic mismatch; skip join publish"
389 );
390 return;
391 }
392 let done_addr = base.saturating_add(KH_THREAD_DONE_OFF);
393 let done_ptr = std::ptr::with_exposed_provenance_mut::<AtomicU32>(done_addr);
394 unsafe {
396 (*done_ptr).store(1, Ordering::Release);
397 }
398 futex_wake(done_ptr.cast::<u32>(), i32::MAX);
399 }
400
401 fn futex_wake(addr: *mut u32, n: i32) {
402 let _ = unsafe {
405 libc::syscall(
406 libc::SYS_futex,
407 addr,
408 129_i32, n,
410 core::ptr::null::<libc::timespec>(),
411 core::ptr::null_mut::<u32>(),
412 0_i32,
413 )
414 };
415 }
416
417 fn worker_finished() {
418 let _ = LIVE_WORKERS.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| {
419 Some(n.saturating_sub(1))
420 });
421 }
422
423 pub(crate) fn exit_worker_now() -> ! {
429 let (pc, sp, has) = host_slot::with_worker_mut(|w| {
430 let has = w.has_exit;
431 let pc = w.exit_pc;
432 let sp = w.exit_sp;
433 w.has_exit = false;
434 (pc, sp, has)
435 });
436 if has {
437 unsafe {
441 std::arch::asm!(
442 "mov sp, {sp}",
443 "mov x29, xzr",
444 "br {pc}",
445 sp = in(reg) sp,
446 pc = in(reg) pc,
447 options(noreturn),
448 );
449 }
450 }
451 finish_worker_on_host();
453 }
454
455 fn set_host_exit_frame(pc: u64, sp: u64) {
456 host_slot::with_worker_mut(|w| {
457 w.exit_pc = pc;
458 w.exit_sp = sp;
459 w.has_exit = true;
460 });
461 }
462
463 fn clear_host_exit_frame() {
464 host_slot::with_worker_mut(|w| {
465 w.has_exit = false;
466 });
467 }
468
469 pub(crate) fn redirect_ucontext_to_host_exit(m: &mut libc::mcontext_t) -> bool {
471 host_slot::with_worker_mut(|w| {
472 if !w.has_exit {
473 return false;
474 }
475 w.has_exit = false;
477 m.pc = w.exit_pc;
478 m.sp = w.exit_sp;
479 m.regs[29] = 0;
481 true
482 })
483 }
484}
485
486#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
487use linux::linux_spawn;
488
489#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
493pub(crate) use linux::redirect_ucontext_to_host_exit;
494
495#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
500pub(crate) fn exit_current_guest_worker() -> ! {
501 linux::exit_worker_now();
502}
503
504#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
505#[allow(dead_code)] pub(crate) fn redirect_ucontext_to_host_exit() -> bool {
507 false
508}
509
510#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
511#[allow(dead_code)] pub(crate) fn exit_current_guest_worker() -> ! {
513 loop {
514 core::hint::spin_loop();
515 }
516}
517
518#[cfg(test)]
519mod tests {
520 use super::*;
521
522 #[test]
523 fn selfid_stable_on_thread() {
524 let a = thread_selfid();
525 let b = thread_selfid();
526 assert_eq!(a, b);
527 assert!(a > 0);
528 }
529
530 #[test]
531 fn alloc_tid_monotonic() {
532 let a = alloc_tid();
533 let b = alloc_tid();
534 assert!(b > a);
535 }
536}