1#![allow(unsafe_code)]
13
14use std::io::{self, Write};
15use std::sync::Mutex;
16use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering};
17
18use thiserror::Error;
19
20use crate::mem::{
21 MapError, MappedRegion, VM_PROT_EXECUTE, VM_PROT_READ, VM_PROT_WRITE, mprotect_darwin,
22 mprotect_rw,
23};
24use crate::syscall;
25#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
26use crate::syscall::SyscallArgs;
27
28pub const BRK_TRAP_IMM: u32 = 0x0F00;
30
31const SVC_MASK: u32 = 0xFFE0_001F;
33const SVC_BASE: u32 = 0xD400_0001;
34
35const fn brk_encoding(imm16: u32) -> u32 {
37 0xD420_0000 | ((imm16 & 0xFFFF) << 5)
38}
39
40pub const PSTATE_C: u64 = 1 << 29;
44
45#[derive(Debug, Clone, PartialEq, Eq)]
47pub struct TrapEvent {
48 pub pc: u64,
50 pub syscall: Option<u32>,
52 pub name: String,
54 pub arg0: u64,
56 pub arg1: u64,
58 pub arg2: u64,
60 pub retval: Option<u64>,
62 pub error: bool,
64}
65
66#[derive(Debug, Clone, Copy, PartialEq, Eq)]
68pub enum TrapOutcome {
69 Exit {
71 code: i32,
73 },
74 Continue,
76 ThreadExit,
78}
79
80#[derive(Debug, Clone)]
82pub struct TrapConfig {
83 pub max_events: usize,
85 pub max_syscalls: usize,
87}
88
89impl Default for TrapConfig {
90 fn default() -> Self {
91 Self {
92 max_events: 256,
93 max_syscalls: 256,
94 }
95 }
96}
97
98#[derive(Debug, Error)]
100pub enum TrapError {
101 #[error("failed to install trap signal handler: {0}")]
103 SignalSetup(#[source] io::Error),
104
105 #[error(transparent)]
107 Map(#[from] MapError),
108
109 #[error("trap backend requires Linux aarch64 for live execution")]
111 UnsupportedArch,
112}
113
114#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
115static HANDLER_INSTALLED: AtomicBool = AtomicBool::new(false);
116static EXIT_CODE: AtomicI32 = AtomicI32::new(0);
117static EXIT_REQUESTED: AtomicBool = AtomicBool::new(false);
118static EVENT_COUNT: AtomicU64 = AtomicU64::new(0);
119static MAX_EVENTS: AtomicU64 = AtomicU64::new(256);
120static TRACE_ON_EXIT: AtomicBool = AtomicBool::new(false);
121static TRACE_JSON: AtomicBool = AtomicBool::new(false);
122static EXPECT_CODE: AtomicI32 = AtomicI32::new(i32::MIN);
124static EVENTS: Mutex<Vec<TrapEvent>> = Mutex::new(Vec::new());
125
126pub fn patch_svc_to_brk(regions: &mut [MappedRegion]) -> Result<usize, TrapError> {
130 let mut patched = 0_usize;
131 for region in regions.iter_mut() {
132 if region.prot & VM_PROT_EXECUTE == 0 {
133 continue;
134 }
135 mprotect_rw(region)?;
136 let bytes = region.host_bytes_mut();
137 let mut off = 0_usize;
138 while off.saturating_add(4) <= bytes.len() {
139 let end = off.saturating_add(4);
140 let Some(word) = bytes.get(off..end) else {
141 break;
142 };
143 let Ok(word_bytes) = <[u8; 4]>::try_from(word) else {
144 break;
145 };
146 let insn = u32::from_le_bytes(word_bytes);
147 if is_svc(insn) {
148 let brk = brk_encoding(BRK_TRAP_IMM).to_le_bytes();
149 if let Some(slot) = bytes.get_mut(off..end) {
150 slot.copy_from_slice(&brk);
151 patched = patched.saturating_add(1);
152 }
153 }
154 off = end;
155 }
156 let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
157 mprotect_darwin(region, restore)?;
158 region.prot = restore;
159 }
160 Ok(patched)
161}
162
163const fn is_svc(insn: u32) -> bool {
164 (insn & SVC_MASK) == SVC_BASE
165}
166
167#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
169#[inline]
170fn align_up_to_page(len: usize, page: usize) -> usize {
171 if page <= 1 {
172 return len.max(1);
173 }
174 let mask = page.saturating_sub(1);
175 len.saturating_add(mask) & !mask
176}
177
178#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
180unsafe extern "C" {
181 fn __clear_cache(start: *mut libc::c_void, end: *mut libc::c_void);
182}
183
184#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
186fn clear_icache(ptr: *mut u8, len: usize) {
187 if ptr.is_null() || len == 0 {
188 return;
189 }
190 unsafe {
192 let start = ptr.cast::<libc::c_void>();
193 let end = ptr.wrapping_add(len).cast::<libc::c_void>();
194 __clear_cache(start, end);
195 }
196}
197
198#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
199#[allow(clippy::as_conversions, function_casts_as_integer)]
200fn trampoline_rust_entry_addr() -> u64 {
201 u64::try_from(kh_trampoline_dispatch as usize).unwrap_or(0)
202}
203
204#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
206#[repr(C)]
207pub struct TrampRet {
208 pub retval: u64,
210 pub error: u64,
212}
213
214#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
226#[unsafe(no_mangle)]
227pub unsafe extern "C" fn kh_trampoline_dispatch(
228 x0: u64,
229 x1: u64,
230 x2: u64,
231 x3: u64,
232 x4: u64,
233 x5: u64,
234 x6: u64,
235 x16: u64,
236) -> TrampRet {
237 let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
238 let result = syscall::dispatch(SyscallArgs {
239 pc: 0,
240 number: sys_no,
241 x0,
242 x1,
243 x2,
244 x3,
245 x4,
246 x5,
247 x6,
248 });
249 if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
250 maybe_push_event(
251 0,
252 sys_no,
253 result.name,
254 x0,
255 x1,
256 x2,
257 result.retval,
258 result.error,
259 );
260 }
261 match result.outcome {
262 TrapOutcome::Exit { code } => finish_with_exit_code(code),
263 TrapOutcome::ThreadExit => {
264 crate::thread::exit_current_guest_worker();
266 }
267 TrapOutcome::Continue => {}
268 }
269 TrampRet {
270 retval: result.retval.unwrap_or(0),
271 error: u64::from(result.error),
272 }
273}
274
275#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
294std::arch::global_asm!(
295 r#"
296 .text
297 .align 2
298 .global kh_hypercall_entry
299 .type kh_hypercall_entry, %function
300kh_hypercall_entry:
301 // Guest prolog: GPRs + full NEON *before* any bl (NEON must survive host TLS).
302 sub sp, sp, #640
303 stp x29, x30, [sp, #0]
304 stp x0, x1, [sp, #16]
305 stp x2, x3, [sp, #32]
306 stp x4, x5, [sp, #48]
307 stp x6, x7, [sp, #64]
308 mov x29, sp
309 stp q0, q1, [sp, #96]
310 stp q2, q3, [sp, #128]
311 stp q4, q5, [sp, #160]
312 stp q6, q7, [sp, #192]
313 stp q8, q9, [sp, #224]
314 stp q10, q11, [sp, #256]
315 stp q12, q13, [sp, #288]
316 stp q14, q15, [sp, #320]
317 stp q16, q17, [sp, #352]
318 stp q18, q19, [sp, #384]
319 stp q20, q21, [sp, #416]
320 stp q22, q23, [sp, #448]
321 stp q24, q25, [sp, #480]
322 stp q26, q27, [sp, #512]
323 stp q28, q29, [sp, #544]
324 stp q30, q31, [sp, #576]
325 mrs x9, fpcr
326 mrs x10, fpsr
327 // #608 exceeds signed LDP imm7*8 range (±512); use base+add.
328 add x11, sp, #608
329 stp x9, x10, [x11]
330
331 // A1/A2: enter returns alt_top in x0, guest_tpidr in x1 (HyperEnterRet).
332 // Park guest_va at prolog+80 (hole before NEON) so leave needs no gettid.
333 bl kh_tls_enter_host
334 mov x9, sp // guest prolog frame
335 str x1, [x9, #80] // guest_tpidr for leave
336 cbz x0, 1f
337 mov sp, x0 // host alt stack
338 ldr x30, [x9, #8] // reload LR → freestanding thin caller
339 // Host frame: [guest_frame, lr]
340 stp x9, x30, [sp, #-16]!
341 // Reload guest frame ptr / args (enter_host clobbered temps).
342 ldr x9, [sp]
343 ldp x0, x1, [x9, #16]
344 ldp x2, x3, [x9, #32]
345 ldp x4, x5, [x9, #48]
346 ldp x6, x7, [x9, #64]
347 mov x16, x7
348 bl kh_neon_tramp_entry
349 // Preserve TrampRet (x0/x1) across TLS restore.
350 stp x0, x1, [sp, #-16]!
351 // guest_va at host_frame.guest_frame + 80; host_frame at sp+16.
352 ldr x9, [sp, #16]
353 ldr x0, [x9, #80]
354 bl kh_tls_leave_host
355 ldp x0, x1, [sp], #16
356 ldp x9, x30, [sp], #16
357 mov sp, x9
358 // Restore guest NEON from prolog (pre-bl snapshot).
359 ldp q0, q1, [sp, #96]
360 ldp q2, q3, [sp, #128]
361 ldp q4, q5, [sp, #160]
362 ldp q6, q7, [sp, #192]
363 ldp q8, q9, [sp, #224]
364 ldp q10, q11, [sp, #256]
365 ldp q12, q13, [sp, #288]
366 ldp q14, q15, [sp, #320]
367 ldp q16, q17, [sp, #352]
368 ldp q18, q19, [sp, #384]
369 ldp q20, q21, [sp, #416]
370 ldp q22, q23, [sp, #448]
371 ldp q24, q25, [sp, #480]
372 ldp q26, q27, [sp, #512]
373 ldp q28, q29, [sp, #544]
374 ldp q30, q31, [sp, #576]
375 add x11, sp, #608
376 ldp x9, x10, [x11]
377 msr fpcr, x9
378 msr fpsr, x10
379 ldr x29, [sp, #0]
380 add sp, sp, #640
381 ret
3821:
383 // No alt stack: dispatch on the guest stack (ST / fallback only).
384 // Already on host TLS from the enter above; guest_va at [sp, #80].
385 ldp x0, x1, [sp, #16]
386 ldp x2, x3, [sp, #32]
387 ldp x4, x5, [sp, #48]
388 ldp x6, x7, [sp, #64]
389 mov x16, x7
390 // Preserve real LR across the call (still on guest prolog frame).
391 ldr x30, [sp, #8]
392 bl kh_neon_tramp_entry
393 stp x0, x1, [sp, #-16]!
394 ldr x0, [sp, #16+80]
395 bl kh_tls_leave_host
396 ldp x0, x1, [sp], #16
397 // Restore guest NEON from prolog.
398 ldp q0, q1, [sp, #96]
399 ldp q2, q3, [sp, #128]
400 ldp q4, q5, [sp, #160]
401 ldp q6, q7, [sp, #192]
402 ldp q8, q9, [sp, #224]
403 ldp q10, q11, [sp, #256]
404 ldp q12, q13, [sp, #288]
405 ldp q14, q15, [sp, #320]
406 ldp q16, q17, [sp, #352]
407 ldp q18, q19, [sp, #384]
408 ldp q20, q21, [sp, #416]
409 ldp q22, q23, [sp, #448]
410 ldp q24, q25, [sp, #480]
411 ldp q26, q27, [sp, #512]
412 ldp q28, q29, [sp, #544]
413 ldp q30, q31, [sp, #576]
414 add x11, sp, #608
415 ldp x9, x10, [x11]
416 msr fpcr, x9
417 msr fpsr, x10
418 ldp x29, x30, [sp, #0]
419 add sp, sp, #640
420 ret
421 .size kh_hypercall_entry, .-kh_hypercall_entry
422 "#
423);
424
425#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
429std::arch::global_asm!(
430 r#"
431 .text
432 .align 2
433 .global kh_neon_tramp_entry
434 .type kh_neon_tramp_entry, %function
435kh_neon_tramp_entry:
436 adrp x17, KH_NEON_TRAMP_VA
437 ldr x17, [x17, :lo12:KH_NEON_TRAMP_VA]
438 cbz x17, 1f
439 br x17
4401:
441 // No-tramp fallback: number in x16 → AAPCS 8th arg x7; tail call.
442 mov x7, x16
443 b kh_trampoline_dispatch
444 .size kh_neon_tramp_entry, .-kh_neon_tramp_entry
445
446 .data
447 .align 3
448 .global KH_NEON_TRAMP_VA
449KH_NEON_TRAMP_VA:
450 .quad 0
451 "#
452);
453
454#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
455unsafe extern "C" {
456 fn kh_hypercall_entry();
457 static mut KH_NEON_TRAMP_VA: u64;
458}
459
460#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
462#[must_use]
463pub fn hypercall_entry_addr() -> u64 {
464 ensure_neon_tramp();
465 #[allow(clippy::as_conversions, function_casts_as_integer)]
466 {
467 u64::try_from(kh_hypercall_entry as usize).unwrap_or(0)
468 }
469}
470
471#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
473fn ensure_neon_tramp() {
474 static ONCE: AtomicU64 = AtomicU64::new(0);
475 static LOCK: Mutex<()> = Mutex::new(());
476
477 if ONCE.load(Ordering::Acquire) != 0 {
478 return;
479 }
480 let _g = LOCK
481 .lock()
482 .unwrap_or_else(std::sync::PoisonError::into_inner);
483 if ONCE.load(Ordering::Acquire) != 0 {
484 return;
485 }
486
487 let code = TRAMP_BYTES;
488 let page = crate::host::page_size().unwrap_or(4096);
489 let map_len = align_up_to_page(code.len(), page).max(page);
490 let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
491 let prot_rw = libc::PROT_READ | libc::PROT_WRITE;
492 let Some(ptr) = crate::host::mmap(None, map_len, prot_rw, flags, -1, 0) else {
493 tracing::error!("NEON tramp mmap failed");
494 return;
495 };
496 let slice = unsafe { std::slice::from_raw_parts_mut(ptr, map_len) };
498 slice.fill(0);
499 if let Some(dst) = slice.get_mut(..code.len()) {
500 dst.copy_from_slice(code);
501 } else {
502 let _ = crate::host::munmap(ptr, map_len);
503 return;
504 }
505 if let Some(slot) = slice.get_mut(TRAMP_RELOC_OFF..TRAMP_RELOC_OFF.saturating_add(8)) {
506 slot.copy_from_slice(&trampoline_rust_entry_addr().to_le_bytes());
507 } else {
508 let _ = crate::host::munmap(ptr, map_len);
509 return;
510 }
511 let prot_rx = libc::PROT_READ | libc::PROT_EXEC;
512 if !crate::host::mprotect(ptr, map_len, prot_rx) {
513 tracing::error!("NEON tramp mprotect RX failed");
514 let _ = crate::host::munmap(ptr, map_len);
515 return;
516 }
517 clear_icache(ptr, map_len);
518 let va = crate::host::ptr_addr_u64(ptr);
519 unsafe {
521 KH_NEON_TRAMP_VA = va;
522 }
523 ONCE.store(va, Ordering::Release);
524 tracing::info!(
525 va = format_args!("{va:#x}"),
526 "mapped NEON tramp for hypercall"
527 );
528}
529
530#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
531#[must_use]
532pub fn hypercall_entry_addr() -> u64 {
533 0
534}
535
536#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
543const TRAMP_RELOC_OFF: usize = 0x100;
544
545#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
551#[rustfmt::skip]
552const TRAMP_BYTES: &[u8] = &[
553 0x5f, 0x24, 0x03, 0xd5, 0xff, 0x03, 0x0a, 0xd1, 0xe8, 0x27, 0x00, 0xa9, 0xea, 0x2f, 0x01, 0xa9,
554 0xec, 0x37, 0x02, 0xa9, 0xee, 0x3f, 0x03, 0xa9, 0xf1, 0x4b, 0x04, 0xa9, 0xfe, 0x2b, 0x00, 0xf9,
555 0x08, 0x44, 0x3b, 0xd5, 0x29, 0x44, 0x3b, 0xd5, 0xe8, 0x27, 0x06, 0xa9, 0xe0, 0x87, 0x03, 0xad,
556 0xe2, 0x8f, 0x04, 0xad, 0xe4, 0x97, 0x05, 0xad, 0xe6, 0x9f, 0x06, 0xad, 0xe8, 0xa7, 0x07, 0xad,
557 0xea, 0xaf, 0x08, 0xad, 0xec, 0xb7, 0x09, 0xad, 0xee, 0xbf, 0x0a, 0xad, 0xf0, 0xc7, 0x0b, 0xad,
558 0xf2, 0xcf, 0x0c, 0xad, 0xf4, 0xd7, 0x0d, 0xad, 0xf6, 0xdf, 0x0e, 0xad, 0xf8, 0xe7, 0x0f, 0xad,
559 0xfa, 0xef, 0x10, 0xad, 0xfc, 0xf7, 0x11, 0xad, 0xfe, 0xff, 0x12, 0xad, 0xe7, 0x03, 0x10, 0xaa,
560 0x88, 0x04, 0x00, 0x58, 0x00, 0x01, 0x3f, 0xd6, 0x02, 0x42, 0x3b, 0xd5, 0x23, 0x00, 0x80, 0x52,
561 0x63, 0x88, 0x63, 0xd3, 0x42, 0x00, 0x23, 0x8a, 0x41, 0x00, 0x00, 0xb4, 0x42, 0x00, 0x03, 0xaa,
562 0x02, 0x42, 0x1b, 0xd5, 0xe8, 0x27, 0x46, 0xa9, 0x08, 0x44, 0x1b, 0xd5, 0x29, 0x44, 0x1b, 0xd5,
563 0xe0, 0x87, 0x43, 0xad, 0xe2, 0x8f, 0x44, 0xad, 0xe4, 0x97, 0x45, 0xad, 0xe6, 0x9f, 0x46, 0xad,
564 0xe8, 0xa7, 0x47, 0xad, 0xea, 0xaf, 0x48, 0xad, 0xec, 0xb7, 0x49, 0xad, 0xee, 0xbf, 0x4a, 0xad,
565 0xf0, 0xc7, 0x4b, 0xad, 0xf2, 0xcf, 0x4c, 0xad, 0xf4, 0xd7, 0x4d, 0xad, 0xf6, 0xdf, 0x4e, 0xad,
566 0xf8, 0xe7, 0x4f, 0xad, 0xfa, 0xef, 0x50, 0xad, 0xfc, 0xf7, 0x51, 0xad, 0xfe, 0xff, 0x52, 0xad,
567 0xfe, 0x2b, 0x40, 0xf9, 0xf1, 0x4b, 0x44, 0xa9, 0xee, 0x3f, 0x43, 0xa9, 0xec, 0x37, 0x42, 0xa9,
568 0xea, 0x2f, 0x41, 0xa9, 0xe8, 0x27, 0x40, 0xa9, 0xff, 0x03, 0x0a, 0x91, 0xc0, 0x03, 0x5f, 0xd6,
569 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ];
571pub fn install_trap_handlers(config: &TrapConfig) -> Result<(), TrapError> {
573 MAX_EVENTS.store(
574 u64::try_from(config.max_events).unwrap_or(256),
575 Ordering::SeqCst,
576 );
577 syscall::reset_syscall_state(config.max_syscalls);
578
579 #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
580 {
581 let _ = config;
582 Err(TrapError::UnsupportedArch)
583 }
584
585 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
586 {
587 if HANDLER_INSTALLED.swap(true, Ordering::SeqCst) {
588 reset_trap_state();
589 syscall::reset_syscall_state(config.max_syscalls);
590 return Ok(());
591 }
592
593 disable_core_dumps();
596
597 unsafe {
599 let mut sa: libc::sigaction = std::mem::zeroed();
600 sa.sa_flags = libc::SA_SIGINFO;
601 #[allow(clippy::as_conversions, function_casts_as_integer)]
602 {
603 sa.sa_sigaction = trap_sigaction as *const () as usize;
604 }
605 libc::sigemptyset(std::ptr::addr_of_mut!(sa.sa_mask));
606 if libc::sigaction(libc::SIGTRAP, std::ptr::addr_of!(sa), std::ptr::null_mut()) != 0 {
607 HANDLER_INSTALLED.store(false, Ordering::SeqCst);
608 return Err(TrapError::SignalSetup(io::Error::last_os_error()));
609 }
610
611 let mut fault: libc::sigaction = std::mem::zeroed();
613 fault.sa_flags = libc::SA_SIGINFO;
614 #[allow(clippy::as_conversions, function_casts_as_integer)]
615 {
616 fault.sa_sigaction = guest_fault_sigaction as *const () as usize;
617 }
618 libc::sigemptyset(std::ptr::addr_of_mut!(fault.sa_mask));
619 for sig in [libc::SIGSEGV, libc::SIGBUS] {
620 if libc::sigaction(sig, std::ptr::addr_of!(fault), std::ptr::null_mut()) != 0 {
621 HANDLER_INSTALLED.store(false, Ordering::SeqCst);
622 return Err(TrapError::SignalSetup(io::Error::last_os_error()));
623 }
624 }
625 }
626 reset_trap_state();
627 Ok(())
628 }
629}
630
631#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
634fn disable_core_dumps() {
635 unsafe {
637 let lim = libc::rlimit {
638 rlim_cur: 0,
639 rlim_max: 0,
640 };
641 let _ = libc::setrlimit(libc::RLIMIT_CORE, std::ptr::addr_of!(lim));
642 }
643}
644
645#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
647unsafe extern "C" fn guest_fault_sigaction(
648 signo: libc::c_int,
649 info: *mut libc::siginfo_t,
650 ctx: *mut libc::c_void,
651) {
652 crate::tls::enter_host_tls();
654 let name = match signo {
655 libc::SIGSEGV => "SIGSEGV",
656 libc::SIGBUS => "SIGBUS",
657 _ => "FAULT",
658 };
659 let mut pc = 0_u64;
660 let mut sp = 0_u64;
661 let mut x0 = 0_u64;
662 let mut x1 = 0_u64;
663 let mut x8 = 0_u64;
664 let mut x16 = 0_u64;
665 let mut lr = 0_u64;
666 if !ctx.is_null() {
667 let uctx = unsafe { &*ctx.cast::<libc::ucontext_t>() };
669 let m = &uctx.uc_mcontext;
670 pc = m.pc;
671 sp = m.sp;
672 x0 = m.regs[0];
673 x1 = m.regs[1];
674 x8 = m.regs[8];
675 x16 = m.regs[16];
676 lr = m.regs[30];
677 }
678 let addr = if info.is_null() {
679 0_u64
680 } else {
681 let p = unsafe { (*info).si_addr() };
683 u64::try_from(p.addr()).unwrap_or(0)
684 };
685 let msg = format!(
686 "error: guest {name} pc={pc:#x} addr={addr:#x} sp={sp:#x} lr={lr:#x} \
687 x0={x0:#x} x1={x1:#x} x8={x8:#x} x16={x16:#x}\n"
688 );
689 drop(io::stderr().write_all(msg.as_bytes()));
690 if let Ok(maps) = std::fs::read_to_string("/proc/self/maps") {
692 for &va in &[pc, addr, lr, x16] {
693 for line in maps.lines() {
694 let Some((range, rest)) = line.split_once(' ') else {
695 continue;
696 };
697 let Some((a, b)) = range.split_once('-') else {
698 continue;
699 };
700 let (Ok(lo), Ok(hi)) = (u64::from_str_radix(a, 16), u64::from_str_radix(b, 16))
701 else {
702 continue;
703 };
704 if va >= lo && va < hi {
705 let off = va.saturating_sub(lo);
706 let m = format!(" map {va:#x} = {lo:#x}+{off:#x} {rest}\n");
707 drop(io::stderr().write_all(m.as_bytes()));
708 break;
709 }
710 }
711 }
712 }
713 drop(io::stderr().flush());
714 if TRACE_ON_EXIT.load(Ordering::SeqCst) {
715 dump_events_before_exit();
716 }
717 unsafe { libc::_exit(128_i32.saturating_add(signo)) };
719}
720
721#[must_use]
723pub fn take_trace_events() -> Vec<TrapEvent> {
724 EVENTS
725 .lock()
726 .map(|mut g| std::mem::take(&mut *g))
727 .unwrap_or_default()
728}
729
730#[must_use]
732pub fn exit_requested() -> bool {
733 EXIT_REQUESTED.load(Ordering::SeqCst)
734}
735
736#[must_use]
738pub fn take_exit_code() -> Option<i32> {
739 if EXIT_REQUESTED.load(Ordering::SeqCst) {
740 Some(EXIT_CODE.load(Ordering::SeqCst))
741 } else {
742 None
743 }
744}
745
746pub fn reset_trap_state() {
748 EXIT_REQUESTED.store(false, Ordering::SeqCst);
749 EXIT_CODE.store(0, Ordering::SeqCst);
750 EVENT_COUNT.store(0, Ordering::SeqCst);
751 if let Ok(mut guard) = EVENTS.lock() {
752 guard.clear();
753 }
754}
755
756pub fn set_trace_on_exit(json: bool) {
758 TRACE_ON_EXIT.store(true, Ordering::SeqCst);
759 TRACE_JSON.store(json, Ordering::SeqCst);
760}
761
762pub fn clear_trace_on_exit() {
764 TRACE_ON_EXIT.store(false, Ordering::SeqCst);
765 TRACE_JSON.store(false, Ordering::SeqCst);
766}
767
768pub fn set_expect_code(code: i32) {
770 EXPECT_CODE.store(code, Ordering::SeqCst);
771}
772
773pub fn clear_expect_code() {
775 EXPECT_CODE.store(i32::MIN, Ordering::SeqCst);
776}
777
778pub fn finish_with_exit_code(code: i32) -> ! {
783 EXIT_CODE.store(code, Ordering::SeqCst);
784 EXIT_REQUESTED.store(true, Ordering::SeqCst);
785 if TRACE_ON_EXIT.load(Ordering::SeqCst) {
786 dump_events_before_exit();
787 }
788 crate::syscall::dump_futex_stats_if_enabled();
790 let expect = EXPECT_CODE.load(Ordering::SeqCst);
791 let status = if expect == i32::MIN || expect == code {
792 code
793 } else {
794 let msg = format!("error: guest exit code {code} != expected {expect}\n");
795 drop(io::stderr().write_all(msg.as_bytes()));
796 1
797 };
798 unsafe { libc::_exit(status) };
800}
801
802#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
803unsafe extern "C" fn trap_sigaction(
804 signo: libc::c_int,
805 _info: *mut libc::siginfo_t,
806 ctx: *mut libc::c_void,
807) {
808 if signo != libc::SIGTRAP || ctx.is_null() {
809 return;
810 }
811
812 crate::tls::enter_host_tls();
814
815 let uctx = unsafe { &mut *ctx.cast::<libc::ucontext_t>() };
817 let m = &mut uctx.uc_mcontext;
818 let pc = m.pc;
819 let x0 = m.regs[0];
821 let x1 = m.regs[1];
822 let x2 = m.regs[2];
823 let x3 = m.regs[3];
824 let x4 = m.regs[4];
825 let x5 = m.regs[5];
826 let x6 = m.regs[6];
827 let x16 = m.regs[16];
828
829 let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
830 let result = syscall::dispatch(SyscallArgs {
831 pc,
832 number: sys_no,
833 x0,
834 x1,
835 x2,
836 x3,
837 x4,
838 x5,
839 x6,
840 });
841
842 if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
844 maybe_push_event(
845 pc,
846 sys_no,
847 result.name,
848 x0,
849 x1,
850 x2,
851 result.retval,
852 result.error,
853 );
854 }
855
856 if let Some(ret) = result.retval {
857 m.regs[0] = ret;
858 }
859 if result.error {
861 m.pstate |= PSTATE_C;
862 } else if result.retval.is_some() {
863 m.pstate &= !PSTATE_C;
864 }
865
866 match result.outcome {
867 TrapOutcome::Exit { code } => {
868 finish_with_exit_code(code);
869 }
870 TrapOutcome::Continue => {
871 m.pc = pc.wrapping_add(4);
872 crate::tls::leave_host_tls();
874 }
875 TrapOutcome::ThreadExit => {
876 if !crate::thread::redirect_ucontext_to_host_exit(m) {
881 finish_with_exit_code(0);
882 }
883 }
884 }
885}
886
887fn dump_events_before_exit() {
888 use std::fmt::Write as _;
889
890 let json = TRACE_JSON.load(Ordering::SeqCst);
891 let events = EVENTS.lock().map(|g| g.clone()).unwrap_or_default();
892 if json {
893 let mut out = String::from("{\"events\":[");
894 for (i, e) in events.iter().enumerate() {
895 if i > 0 {
896 out.push(',');
897 }
898 let sys = e
899 .syscall
900 .map_or_else(|| "null".to_owned(), |s| s.to_string());
901 let ret = e
902 .retval
903 .map_or_else(|| "null".to_owned(), |v| v.to_string());
904 let _ = write!(
905 out,
906 "{{\"pc\":{},\"syscall\":{},\"name\":\"{}\",\"arg0\":{},\"arg1\":{},\"arg2\":{},\"retval\":{},\"error\":{}}}",
907 e.pc, sys, e.name, e.arg0, e.arg1, e.arg2, ret, e.error
908 );
909 }
910 out.push_str("]}\n");
911 drop(io::stdout().write_all(out.as_bytes()));
912 drop(io::stdout().flush());
913 } else {
914 let header = format!("trace events: {}\n", events.len());
915 drop(io::stdout().write_all(header.as_bytes()));
916 for (i, e) in events.iter().enumerate() {
917 let ret = e
918 .retval
919 .map_or_else(|| "-".to_owned(), |v| format!("{v:#x}"));
920 let err = if e.error { " err" } else { "" };
921 let line = format!(
922 " [{i}] pc={:#x} {} sys={:?} args=({:#x}, {:#x}, {:#x}) ret={ret}{err}\n",
923 e.pc, e.name, e.syscall, e.arg0, e.arg1, e.arg2
924 );
925 drop(io::stdout().write_all(line.as_bytes()));
926 }
927 drop(io::stdout().flush());
928 }
929}
930
931#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
932#[allow(clippy::too_many_arguments)]
933fn maybe_push_event(
934 pc: u64,
935 sys_no: u32,
936 name: &'static str,
937 arg0: u64,
938 arg1: u64,
939 arg2: u64,
940 retval: Option<u64>,
941 error: bool,
942) {
943 let max = MAX_EVENTS.load(Ordering::Relaxed);
944 if max == 0 {
946 return;
947 }
948 let n = EVENT_COUNT.fetch_add(1, Ordering::Relaxed);
949 if n >= max {
950 return;
951 }
952 if let Ok(mut guard) = EVENTS.lock()
953 && u64::try_from(guard.len()).unwrap_or(u64::MAX) < max
954 {
955 guard.push(TrapEvent {
956 pc,
957 syscall: Some(sys_no),
958 name: name.to_owned(),
959 arg0,
960 arg1,
961 arg2,
962 retval,
963 error,
964 });
965 }
966}
967
968#[cfg(test)]
969mod tests {
970 use super::*;
971
972 #[test]
973 fn detects_svc_encoding() {
974 assert!(is_svc(0xD400_0001));
975 let svc80 = 0xD400_0001 | (0x80_u32 << 5);
976 assert!(is_svc(svc80));
977 assert!(!is_svc(0xD503_201F));
978 assert!(!is_svc(brk_encoding(BRK_TRAP_IMM)));
979 }
980
981 #[test]
982 fn brk_encoding_roundtrip_imm() {
983 let enc = brk_encoding(BRK_TRAP_IMM);
984 let imm = (enc >> 5) & 0xFFFF;
985 assert_eq!(imm, BRK_TRAP_IMM);
986 }
987}