1#![allow(unsafe_code)]
17
18use std::sync::atomic::{AtomicU64, Ordering};
19
20use crate::host_slot;
21
22static NEXT_TID: AtomicU64 = AtomicU64::new(1);
24
25#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
27const HOST_ALT_STACK_SIZE: usize = 512 * 1024;
28
29#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
42#[unsafe(no_mangle)]
43pub unsafe extern "C" fn kh_host_alt_sp() -> u64 {
44 ensure_host_alt_stack()
45}
46
47#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
52#[must_use]
53pub fn ensure_host_alt_stack() -> u64 {
54 host_slot::with_alt_mut(|cell| {
55 if let Some(s) = *cell {
56 return s.top;
57 }
58 match map_host_alt_stack() {
59 Some(s) => {
60 let top = s.top;
61 *cell = Some(s);
62 top
63 }
64 None => 0,
65 }
66 })
67}
68
69#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
70#[must_use]
71pub fn ensure_host_alt_stack() -> u64 {
72 0
73}
74
75#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
76fn map_host_alt_stack() -> Option<host_slot::AltStackSlot> {
77 let len = HOST_ALT_STACK_SIZE;
78 let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
79 let prot = libc::PROT_READ | libc::PROT_WRITE;
80 let ptr = mmap_anon_raw(len, prot, flags)?;
83 let base_u = crate::host::ptr_addr_u64(ptr);
84 let top = base_u.saturating_add(u64::try_from(len).unwrap_or(0)) & !0xF;
85 if top <= base_u {
86 let _ = crate::host::munmap(ptr, len);
87 return None;
88 }
89 Some(host_slot::AltStackSlot {
90 base: ptr,
91 len,
92 top,
93 })
94}
95
96#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
98fn mmap_anon_raw(len: usize, prot: libc::c_int, flags: libc::c_int) -> Option<*mut u8> {
99 let raw = unsafe {
101 libc::syscall(
102 libc::SYS_mmap,
103 core::ptr::null_mut::<libc::c_void>(),
104 len,
105 prot,
106 flags,
107 -1_i32,
108 0_i64,
109 )
110 };
111 if raw < 0 {
112 return None;
113 }
114 let addr = usize::try_from(raw).ok()?;
115 Some(core::ptr::with_exposed_provenance_mut(addr))
116}
117
118#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
120pub(crate) fn drop_host_alt_stack() {
121 host_slot::with_alt_mut(|cell| {
122 if let Some(s) = cell.take() {
123 let _ = crate::host::munmap(s.base, s.len);
124 }
125 });
126}
127
128#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
129#[allow(dead_code)] pub(crate) fn drop_host_alt_stack() {}
131
132static LIVE_WORKERS: AtomicU64 = AtomicU64::new(0);
134
135#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
137const EAGAIN: i64 = 35;
138
139pub fn reset_thread_runtime() {
141 NEXT_TID.store(1, Ordering::SeqCst);
142 LIVE_WORKERS.store(0, Ordering::SeqCst);
143}
144
145#[must_use]
147pub fn alloc_tid() -> u64 {
148 NEXT_TID.fetch_add(1, Ordering::SeqCst)
149}
150
151#[must_use]
155pub fn thread_selfid() -> u64 {
156 host_slot::with_worker_mut(|w| {
157 if w.guest_tid == 0 {
158 w.guest_tid = alloc_tid();
159 }
160 w.guest_tid
161 })
162}
163
164#[must_use]
166pub fn live_workers() -> u64 {
167 LIVE_WORKERS.load(Ordering::SeqCst)
168}
169
170#[derive(Debug, Clone, Copy)]
172pub struct GuestThreadStart {
173 pub entry: u64,
175 pub sp: u64,
177 pub pthread: u64,
179 pub port: u64,
181 pub func: u64,
183 pub func_arg: u64,
185}
186
187pub fn spawn_guest_thread(start: GuestThreadStart) -> Result<(), i64> {
189 #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
190 {
191 let _ = start;
192 Err(crate::syscall::ENOSYS)
193 }
194
195 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
196 {
197 linux_spawn(start)
198 }
199}
200
201#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
202mod linux {
203 use super::{EAGAIN, GuestThreadStart, LIVE_WORKERS, thread_selfid};
204 use crate::host_slot;
205 use std::sync::atomic::{AtomicU32, Ordering};
206
207 const KH_THREAD_MAGIC: u64 = 0x4B48_5054_4852_4401;
219 const KH_THREAD_DONE_OFF: usize = 8;
220
221 pub(super) fn linux_spawn(start: GuestThreadStart) -> Result<(), i64> {
222 if start.entry == 0 || start.sp == 0 || !start.sp.is_multiple_of(16) {
223 return Err(crate::syscall::EINVAL);
224 }
225
226 let builder = std::thread::Builder::new().name("kh-guest".into());
227 match builder.spawn(move || guest_worker_main(start)) {
228 Ok(_handle) => {
229 LIVE_WORKERS.fetch_add(1, Ordering::SeqCst);
230 Ok(())
231 }
232 Err(_) => Err(EAGAIN),
233 }
234 }
235
236 fn guest_worker_main(start: GuestThreadStart) {
237 let _ = thread_selfid();
238 host_slot::with_worker_mut(|w| {
239 w.guest_pthread = start.pthread;
240 });
241
242 crate::tls::prepare_host_meta();
244 let _ = super::ensure_host_alt_stack();
247
248 unblock_sigtrap();
252
253 let host_sp = crate::cpu::read_sp();
254 let frame_sp = host_sp.saturating_sub(4096) & !0xF;
256 set_host_exit_frame(host_thread_exit_pc(), frame_sp);
257
258 if let Some(tsd) = guest_tsd_from_pthread(start.pthread) {
261 crate::tls::enter_guest_tls(tsd);
262 }
263
264 let result = unsafe {
267 crate::entry::jump_to_guest_args(
268 start.entry,
269 start.sp,
270 start.pthread,
271 start.port,
272 start.func,
273 start.func_arg,
274 )
275 };
276 if let Err(err) = result {
277 tracing::error!(?err, "guest worker failed to enter");
278 }
279 finish_worker_on_host();
281 }
282
283 const KH_THREAD_TSD_OFF: usize = 56;
285
286 fn guest_tsd_from_pthread(pthread: u64) -> Option<u64> {
287 let base = usize::try_from(pthread).ok()?;
288 if base == 0 {
289 return None;
290 }
291 let magic_ptr = std::ptr::with_exposed_provenance::<u64>(base);
293 let magic = unsafe { core::ptr::read_volatile(magic_ptr) };
294 if magic != KH_THREAD_MAGIC {
295 return None;
296 }
297 let tsd_ptr = std::ptr::with_exposed_provenance::<u64>(base.saturating_add(KH_THREAD_TSD_OFF));
298 let tsd = unsafe { core::ptr::read_volatile(tsd_ptr) };
299 if tsd == 0 {
300 return None;
301 }
302 Some(tsd)
303 }
304
305 fn unblock_sigtrap() {
306 unsafe {
309 let mut set: libc::sigset_t = std::mem::zeroed();
310 libc::sigemptyset(std::ptr::addr_of_mut!(set));
311 libc::sigaddset(std::ptr::addr_of_mut!(set), libc::SIGTRAP);
312 libc::pthread_sigmask(
313 libc::SIG_UNBLOCK,
314 std::ptr::addr_of!(set),
315 std::ptr::null_mut(),
316 );
317 }
318 }
319
320 #[allow(clippy::as_conversions, function_casts_as_integer)]
321 fn host_thread_exit_pc() -> u64 {
322 u64::try_from(host_thread_exit as usize).unwrap_or(0)
323 }
324
325 unsafe extern "C" fn host_thread_exit() -> ! {
327 finish_worker_on_host();
328 }
329
330 fn finish_worker_on_host() -> ! {
335 clear_host_exit_frame();
336 crate::tls::clear_guest_tls_on_exit();
338 publish_guest_join_done();
339 host_slot::with_worker_mut(|w| {
340 w.guest_pthread = 0;
341 });
342 super::drop_host_alt_stack();
343 host_slot::clear_current();
344 worker_finished();
345 unsafe {
352 libc::syscall(libc::SYS_exit, 0);
353 }
354 loop {
356 core::hint::spin_loop();
357 }
358 }
359
360 fn publish_guest_join_done() {
367 let pthread = host_slot::worker_get().guest_pthread;
368 if pthread == 0 {
369 return;
370 }
371 let base = usize::try_from(pthread).unwrap_or(0);
372 if base == 0 {
373 return;
374 }
375 let magic_ptr = std::ptr::with_exposed_provenance::<u64>(base);
377 let magic = unsafe { core::ptr::read_volatile(magic_ptr) };
378 if magic != KH_THREAD_MAGIC {
379 tracing::warn!(
380 pthread = format_args!("{pthread:#x}"),
381 magic = format_args!("{magic:#x}"),
382 "guest pthread magic mismatch; skip join publish"
383 );
384 return;
385 }
386 let done_addr = base.saturating_add(KH_THREAD_DONE_OFF);
387 let done_ptr = std::ptr::with_exposed_provenance_mut::<AtomicU32>(done_addr);
388 unsafe {
390 (*done_ptr).store(1, Ordering::Release);
391 }
392 futex_wake(done_ptr.cast::<u32>(), i32::MAX);
393 }
394
395 fn futex_wake(addr: *mut u32, n: i32) {
396 let _ = unsafe {
398 libc::syscall(
399 libc::SYS_futex,
400 addr,
401 1_i32, n,
403 core::ptr::null::<libc::timespec>(),
404 core::ptr::null_mut::<u32>(),
405 0_i32,
406 )
407 };
408 }
409
410 fn worker_finished() {
411 let _ = LIVE_WORKERS.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| {
412 Some(n.saturating_sub(1))
413 });
414 }
415
416 pub(crate) fn exit_worker_now() -> ! {
422 let (pc, sp, has) = host_slot::with_worker_mut(|w| {
423 let has = w.has_exit;
424 let pc = w.exit_pc;
425 let sp = w.exit_sp;
426 w.has_exit = false;
427 (pc, sp, has)
428 });
429 if has {
430 unsafe {
434 std::arch::asm!(
435 "mov sp, {sp}",
436 "mov x29, xzr",
437 "br {pc}",
438 sp = in(reg) sp,
439 pc = in(reg) pc,
440 options(noreturn),
441 );
442 }
443 }
444 finish_worker_on_host();
446 }
447
448 fn set_host_exit_frame(pc: u64, sp: u64) {
449 host_slot::with_worker_mut(|w| {
450 w.exit_pc = pc;
451 w.exit_sp = sp;
452 w.has_exit = true;
453 });
454 }
455
456 fn clear_host_exit_frame() {
457 host_slot::with_worker_mut(|w| {
458 w.has_exit = false;
459 });
460 }
461
462 pub(crate) fn redirect_ucontext_to_host_exit(m: &mut libc::mcontext_t) -> bool {
464 host_slot::with_worker_mut(|w| {
465 if !w.has_exit {
466 return false;
467 }
468 w.has_exit = false;
470 m.pc = w.exit_pc;
471 m.sp = w.exit_sp;
472 m.regs[29] = 0;
474 true
475 })
476 }
477}
478
479#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
480use linux::linux_spawn;
481
482#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
486pub(crate) use linux::redirect_ucontext_to_host_exit;
487
488#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
493pub(crate) fn exit_current_guest_worker() -> ! {
494 linux::exit_worker_now();
495}
496
497#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
498#[allow(dead_code)] pub(crate) fn redirect_ucontext_to_host_exit() -> bool {
500 false
501}
502
503#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
504#[allow(dead_code)] pub(crate) fn exit_current_guest_worker() -> ! {
506 loop {
507 core::hint::spin_loop();
508 }
509}
510
511#[cfg(test)]
512mod tests {
513 use super::*;
514
515 #[test]
516 fn selfid_stable_on_thread() {
517 let a = thread_selfid();
518 let b = thread_selfid();
519 assert_eq!(a, b);
520 assert!(a > 0);
521 }
522
523 #[test]
524 fn alloc_tid_monotonic() {
525 let a = alloc_tid();
526 let b = alloc_tid();
527 assert!(b > a);
528 }
529}