1#![allow(unsafe_code)]
14
15use std::cell::Cell;
16use std::sync::atomic::{AtomicU64, Ordering};
17
18thread_local! {
19 static TID: Cell<u64> = const { Cell::new(0) };
20}
21
22#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
25thread_local! {
26 static HOST_ALT: Cell<Option<HostAltStack>> = const { Cell::new(None) };
27}
28
29static NEXT_TID: AtomicU64 = AtomicU64::new(1);
31
32#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
34const HOST_ALT_STACK_SIZE: usize = 512 * 1024;
35
36#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
37#[derive(Clone, Copy)]
38struct HostAltStack {
39 base: *mut u8,
40 len: usize,
41 top: u64,
43}
44
45#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
55#[unsafe(no_mangle)]
56pub unsafe extern "C" fn kh_host_alt_sp() -> u64 {
57 HOST_ALT.with(|cell| {
58 if let Some(s) = cell.get() {
59 return s.top;
60 }
61 match map_host_alt_stack() {
62 Some(s) => {
63 cell.set(Some(s));
64 s.top
65 }
66 None => 0,
67 }
68 })
69}
70
71#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
72fn map_host_alt_stack() -> Option<HostAltStack> {
73 let len = HOST_ALT_STACK_SIZE;
74 let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
75 let prot = libc::PROT_READ | libc::PROT_WRITE;
77 let ptr = crate::host::mmap(None, len, prot, flags, -1, 0)?;
78 let base_u = crate::host::ptr_addr_u64(ptr);
79 let top = base_u.saturating_add(u64::try_from(len).unwrap_or(0)) & !0xF;
80 if top <= base_u {
81 let _ = crate::host::munmap(ptr, len);
82 return None;
83 }
84 Some(HostAltStack {
85 base: ptr,
86 len,
87 top,
88 })
89}
90
91#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
93pub(crate) fn drop_host_alt_stack() {
94 HOST_ALT.with(|cell| {
95 if let Some(s) = cell.take() {
96 let _ = crate::host::munmap(s.base, s.len);
97 }
98 });
99}
100
101#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
102#[allow(dead_code)] pub(crate) fn drop_host_alt_stack() {}
104
105static LIVE_WORKERS: AtomicU64 = AtomicU64::new(0);
107
108#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
110const EAGAIN: i64 = 35;
111
112pub fn reset_thread_runtime() {
114 NEXT_TID.store(1, Ordering::SeqCst);
115 LIVE_WORKERS.store(0, Ordering::SeqCst);
116}
117
118#[must_use]
120pub fn alloc_tid() -> u64 {
121 NEXT_TID.fetch_add(1, Ordering::SeqCst)
122}
123
124#[must_use]
126pub fn thread_selfid() -> u64 {
127 TID.with(|c| {
128 let mut v = c.get();
129 if v == 0 {
130 v = alloc_tid();
131 c.set(v);
132 }
133 v
134 })
135}
136
137#[must_use]
139pub fn live_workers() -> u64 {
140 LIVE_WORKERS.load(Ordering::SeqCst)
141}
142
143#[derive(Debug, Clone, Copy)]
145pub struct GuestThreadStart {
146 pub entry: u64,
148 pub sp: u64,
150 pub pthread: u64,
152 pub port: u64,
154 pub func: u64,
156 pub func_arg: u64,
158}
159
160pub fn spawn_guest_thread(start: GuestThreadStart) -> Result<(), i64> {
162 #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
163 {
164 let _ = start;
165 Err(crate::syscall::ENOSYS)
166 }
167
168 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
169 {
170 linux_spawn(start)
171 }
172}
173
174#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
175mod linux {
176 use super::{EAGAIN, GuestThreadStart, LIVE_WORKERS, thread_selfid};
177 use std::cell::Cell;
178 use std::sync::atomic::{AtomicU32, Ordering};
179
180 const KH_THREAD_MAGIC: u64 = 0x4B48_5054_4852_4401;
191 const KH_THREAD_DONE_OFF: usize = 8;
192
193 #[derive(Clone, Copy)]
195 struct HostExitFrame {
196 pc: u64,
197 sp: u64,
198 }
199
200 thread_local! {
201 static HOST_EXIT: Cell<Option<HostExitFrame>> = const { Cell::new(None) };
202 static GUEST_PTHREAD: Cell<u64> = const { Cell::new(0) };
204 }
205
206 pub(super) fn linux_spawn(start: GuestThreadStart) -> Result<(), i64> {
207 if start.entry == 0 || start.sp == 0 || !start.sp.is_multiple_of(16) {
208 return Err(crate::syscall::EINVAL);
209 }
210
211 let builder = std::thread::Builder::new().name("kh-guest".into());
212 match builder.spawn(move || guest_worker_main(start)) {
213 Ok(_handle) => {
214 LIVE_WORKERS.fetch_add(1, Ordering::SeqCst);
215 Ok(())
216 }
217 Err(_) => Err(EAGAIN),
218 }
219 }
220
221 fn guest_worker_main(start: GuestThreadStart) {
222 let _ = thread_selfid();
223 GUEST_PTHREAD.set(start.pthread);
224
225 unblock_sigtrap();
229
230 let host_sp = read_sp();
231 let frame_sp = host_sp.saturating_sub(4096) & !0xF;
233 set_host_exit_frame(host_thread_exit_pc(), frame_sp);
234
235 let result = unsafe {
238 crate::entry::jump_to_guest_args(
239 start.entry,
240 start.sp,
241 start.pthread,
242 start.port,
243 start.func,
244 start.func_arg,
245 )
246 };
247 if let Err(err) = result {
248 tracing::error!(?err, "guest worker failed to enter");
249 }
250 finish_worker_on_host();
252 }
253
254 fn unblock_sigtrap() {
255 unsafe {
258 let mut set: libc::sigset_t = std::mem::zeroed();
259 libc::sigemptyset(std::ptr::addr_of_mut!(set));
260 libc::sigaddset(std::ptr::addr_of_mut!(set), libc::SIGTRAP);
261 libc::pthread_sigmask(
262 libc::SIG_UNBLOCK,
263 std::ptr::addr_of!(set),
264 std::ptr::null_mut(),
265 );
266 }
267 }
268
269 fn read_sp() -> u64 {
270 let sp: u64;
271 unsafe {
273 std::arch::asm!("mov {}, sp", out(reg) sp, options(nomem, nostack, preserves_flags));
274 }
275 sp
276 }
277
278 #[allow(clippy::as_conversions, function_casts_as_integer)]
279 fn host_thread_exit_pc() -> u64 {
280 u64::try_from(host_thread_exit as usize).unwrap_or(0)
281 }
282
283 unsafe extern "C" fn host_thread_exit() -> ! {
285 finish_worker_on_host();
286 }
287
288 fn finish_worker_on_host() -> ! {
293 clear_host_exit_frame();
294 publish_guest_join_done();
295 GUEST_PTHREAD.set(0);
296 super::drop_host_alt_stack();
297 worker_finished();
298 unsafe {
300 libc::pthread_exit(std::ptr::null_mut());
301 }
302 }
303
304 fn publish_guest_join_done() {
311 let pthread = GUEST_PTHREAD.get();
312 if pthread == 0 {
313 return;
314 }
315 let base = usize::try_from(pthread).unwrap_or(0);
316 if base == 0 {
317 return;
318 }
319 let magic_ptr = std::ptr::with_exposed_provenance::<u64>(base);
321 let magic = unsafe { core::ptr::read_volatile(magic_ptr) };
322 if magic != KH_THREAD_MAGIC {
323 tracing::warn!(
324 pthread = format_args!("{pthread:#x}"),
325 magic = format_args!("{magic:#x}"),
326 "guest pthread magic mismatch; skip join publish"
327 );
328 return;
329 }
330 let done_addr = base.saturating_add(KH_THREAD_DONE_OFF);
331 let done_ptr = std::ptr::with_exposed_provenance_mut::<AtomicU32>(done_addr);
332 unsafe {
334 (*done_ptr).store(1, Ordering::Release);
335 }
336 futex_wake(done_ptr.cast::<u32>(), i32::MAX);
337 }
338
339 fn futex_wake(addr: *mut u32, n: i32) {
340 let _ = unsafe {
342 libc::syscall(
343 libc::SYS_futex,
344 addr,
345 1_i32, n,
347 core::ptr::null::<libc::timespec>(),
348 core::ptr::null_mut::<u32>(),
349 0_i32,
350 )
351 };
352 }
353
354 fn worker_finished() {
355 let _ = LIVE_WORKERS.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| {
356 Some(n.saturating_sub(1))
357 });
358 }
359
360 pub(crate) fn exit_worker_now() -> ! {
366 let frame = HOST_EXIT.take();
367 if let Some(frame) = frame {
368 unsafe {
371 std::arch::asm!(
372 "mov sp, {sp}",
373 "br {pc}",
374 sp = in(reg) frame.sp,
375 pc = in(reg) frame.pc,
376 options(noreturn),
377 );
378 }
379 }
380 finish_worker_on_host();
382 }
383
384 fn set_host_exit_frame(pc: u64, sp: u64) {
385 HOST_EXIT.set(Some(HostExitFrame { pc, sp }));
386 }
387
388 fn clear_host_exit_frame() {
389 HOST_EXIT.set(None);
390 }
391
392 pub(crate) fn redirect_ucontext_to_host_exit(m: &mut libc::mcontext_t) -> bool {
394 HOST_EXIT.with(|cell| {
395 let Some(frame) = cell.get() else {
396 return false;
397 };
398 cell.set(None);
400 m.pc = frame.pc;
401 m.sp = frame.sp;
402 true
403 })
404 }
405}
406
407#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
408use linux::linux_spawn;
409
410#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
414pub(crate) use linux::redirect_ucontext_to_host_exit;
415
416#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
420pub(crate) fn exit_current_guest_worker() -> ! {
421 linux::exit_worker_now();
422}
423
424#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
425#[allow(dead_code)] pub(crate) fn redirect_ucontext_to_host_exit() -> bool {
427 false
428}
429
430#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
431#[allow(dead_code)] pub(crate) fn exit_current_guest_worker() -> ! {
433 loop {
434 core::hint::spin_loop();
435 }
436}
437
438#[cfg(test)]
439mod tests {
440 use super::*;
441
442 #[test]
443 fn selfid_stable_on_thread() {
444 let a = thread_selfid();
445 let b = thread_selfid();
446 assert_eq!(a, b);
447 assert!(a > 0);
448 }
449
450 #[test]
451 fn alloc_tid_monotonic() {
452 let a = alloc_tid();
453 let b = alloc_tid();
454 assert!(b > a);
455 }
456}