1#![allow(unsafe_code)]
14
15use std::io::{self, Write};
16use std::sync::Mutex;
17use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering};
18
19use thiserror::Error;
20
21use crate::mem::{
22 MapError, MappedRegion, VM_PROT_EXECUTE, VM_PROT_READ, VM_PROT_WRITE, mprotect_darwin,
23 mprotect_rw,
24};
25use crate::syscall;
26#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
27use crate::syscall::SyscallArgs;
28
29pub const BRK_TRAP_IMM: u32 = 0x0F00;
31
32const SVC_MASK: u32 = 0xFFE0_001F;
34const SVC_BASE: u32 = 0xD400_0001;
35
36const fn brk_encoding(imm16: u32) -> u32 {
38 0xD420_0000 | ((imm16 & 0xFFFF) << 5)
39}
40
41pub const PSTATE_C: u64 = 1 << 29;
45
46#[derive(Debug, Clone, PartialEq, Eq)]
48pub struct TrapEvent {
49 pub pc: u64,
51 pub syscall: Option<u32>,
53 pub name: String,
55 pub arg0: u64,
57 pub arg1: u64,
59 pub arg2: u64,
61 pub retval: Option<u64>,
63 pub error: bool,
65}
66
67#[derive(Debug, Clone, Copy, PartialEq, Eq)]
69pub enum TrapOutcome {
70 Exit {
72 code: i32,
74 },
75 Continue,
77 ThreadExit,
79}
80
81#[derive(Debug, Clone)]
83pub struct TrapConfig {
84 pub max_events: usize,
86 pub max_syscalls: usize,
88}
89
90impl Default for TrapConfig {
91 fn default() -> Self {
92 Self {
93 max_events: 256,
94 max_syscalls: 256,
95 }
96 }
97}
98
99#[derive(Debug, Error)]
101pub enum TrapError {
102 #[error("failed to install trap signal handler: {0}")]
104 SignalSetup(#[source] io::Error),
105
106 #[error(transparent)]
108 Map(#[from] MapError),
109
110 #[error("trap backend requires Linux aarch64 for live execution")]
112 UnsupportedArch,
113}
114
115#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
116static HANDLER_INSTALLED: AtomicBool = AtomicBool::new(false);
117static EXIT_CODE: AtomicI32 = AtomicI32::new(0);
118static EXIT_REQUESTED: AtomicBool = AtomicBool::new(false);
119static EVENT_COUNT: AtomicU64 = AtomicU64::new(0);
120static MAX_EVENTS: AtomicU64 = AtomicU64::new(256);
121static TRACE_ON_EXIT: AtomicBool = AtomicBool::new(false);
122static TRACE_JSON: AtomicBool = AtomicBool::new(false);
123static EXPECT_CODE: AtomicI32 = AtomicI32::new(i32::MIN);
125static EVENTS: Mutex<Vec<TrapEvent>> = Mutex::new(Vec::new());
126
127pub fn patch_svc_to_brk(regions: &mut [MappedRegion]) -> Result<usize, TrapError> {
131 let mut patched = 0_usize;
132 for region in regions.iter_mut() {
133 if region.prot & VM_PROT_EXECUTE == 0 {
134 continue;
135 }
136 mprotect_rw(region)?;
137 let bytes = region.host_bytes_mut();
138 let mut off = 0_usize;
139 while off.saturating_add(4) <= bytes.len() {
140 let end = off.saturating_add(4);
141 let Some(word) = bytes.get(off..end) else {
142 break;
143 };
144 let Ok(word_bytes) = <[u8; 4]>::try_from(word) else {
145 break;
146 };
147 let insn = u32::from_le_bytes(word_bytes);
148 if is_svc(insn) {
149 let brk = brk_encoding(BRK_TRAP_IMM).to_le_bytes();
150 if let Some(slot) = bytes.get_mut(off..end) {
151 slot.copy_from_slice(&brk);
152 patched = patched.saturating_add(1);
153 }
154 }
155 off = end;
156 }
157 let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
158 mprotect_darwin(region, restore)?;
159 region.prot = restore;
160 }
161 Ok(patched)
162}
163
164const fn is_svc(insn: u32) -> bool {
165 (insn & SVC_MASK) == SVC_BASE
166}
167
168#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
170#[inline]
171fn align_up_to_page(len: usize, page: usize) -> usize {
172 if page <= 1 {
173 return len.max(1);
174 }
175 let mask = page.saturating_sub(1);
176 len.saturating_add(mask) & !mask
177}
178
179#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
181unsafe extern "C" {
182 fn __clear_cache(start: *mut libc::c_void, end: *mut libc::c_void);
183}
184
185#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
187fn clear_icache(ptr: *mut u8, len: usize) {
188 if ptr.is_null() || len == 0 {
189 return;
190 }
191 unsafe {
193 let start = ptr.cast::<libc::c_void>();
194 let end = ptr.wrapping_add(len).cast::<libc::c_void>();
195 __clear_cache(start, end);
196 }
197}
198
199#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
200#[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
201fn trampoline_rust_entry_addr() -> u64 {
202 u64::try_from(kh_trampoline_dispatch as usize).unwrap_or(0)
203}
204
205#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
207#[repr(C)]
208pub struct TrampRet {
209 pub retval: u64,
211 pub error: u64,
213}
214
215#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
227#[unsafe(no_mangle)]
228pub unsafe extern "C" fn kh_trampoline_dispatch(
229 x0: u64,
230 x1: u64,
231 x2: u64,
232 x3: u64,
233 x4: u64,
234 x5: u64,
235 x6: u64,
236 x16: u64,
237) -> TrampRet {
238 let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
239 let result = syscall::dispatch(SyscallArgs {
240 pc: 0,
241 number: sys_no,
242 x0,
243 x1,
244 x2,
245 x3,
246 x4,
247 x5,
248 x6,
249 });
250 if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
251 maybe_push_event(
252 0,
253 sys_no,
254 result.name,
255 x0,
256 x1,
257 x2,
258 result.retval,
259 result.error,
260 );
261 }
262 match result.outcome {
263 TrapOutcome::Exit { code } => finish_with_exit_code(code),
264 TrapOutcome::ThreadExit => {
265 crate::thread::exit_current_guest_worker();
267 }
268 TrapOutcome::Continue => {}
269 }
270 TrampRet {
271 retval: result.retval.unwrap_or(0),
272 error: u64::from(result.error),
273 }
274}
275
276#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
295std::arch::global_asm!(
296 r#"
297 .text
298 .align 2
299 .global kh_hypercall_entry
300 .type kh_hypercall_entry, %function
301kh_hypercall_entry:
302 // Guest prolog: GPRs + full NEON *before* any bl (NEON must survive host TLS).
303 sub sp, sp, #640
304 stp x29, x30, [sp, #0]
305 stp x0, x1, [sp, #16]
306 stp x2, x3, [sp, #32]
307 stp x4, x5, [sp, #48]
308 stp x6, x7, [sp, #64]
309 mov x29, sp
310 stp q0, q1, [sp, #96]
311 stp q2, q3, [sp, #128]
312 stp q4, q5, [sp, #160]
313 stp q6, q7, [sp, #192]
314 stp q8, q9, [sp, #224]
315 stp q10, q11, [sp, #256]
316 stp q12, q13, [sp, #288]
317 stp q14, q15, [sp, #320]
318 stp q16, q17, [sp, #352]
319 stp q18, q19, [sp, #384]
320 stp q20, q21, [sp, #416]
321 stp q22, q23, [sp, #448]
322 stp q24, q25, [sp, #480]
323 stp q26, q27, [sp, #512]
324 stp q28, q29, [sp, #544]
325 stp q30, q31, [sp, #576]
326 mrs x9, fpcr
327 mrs x10, fpsr
328 // #608 exceeds signed LDP imm7*8 range (±512); use base+add.
329 add x11, sp, #608
330 stp x9, x10, [x11]
331
332 // A1/A2: enter returns alt_top in x0, guest_tpidr in x1 (HyperEnterRet).
333 // Park guest_va at prolog+80 (hole before NEON) so leave needs no gettid.
334 bl kh_tls_enter_host
335 mov x9, sp // guest prolog frame
336 str x1, [x9, #80] // guest_tpidr for leave
337 cbz x0, 1f
338 mov sp, x0 // host alt stack
339 ldr x30, [x9, #8] // reload LR → freestanding thin caller
340 // Host frame: [guest_frame, lr]
341 stp x9, x30, [sp, #-16]!
342 // Reload guest frame ptr / args (enter_host clobbered temps).
343 ldr x9, [sp]
344 ldp x0, x1, [x9, #16]
345 ldp x2, x3, [x9, #32]
346 ldp x4, x5, [x9, #48]
347 ldp x6, x7, [x9, #64]
348 mov x16, x7
349 bl kh_neon_tramp_entry
350 // Preserve TrampRet (x0/x1) across TLS restore.
351 stp x0, x1, [sp, #-16]!
352 // guest_va at host_frame.guest_frame + 80; host_frame at sp+16.
353 ldr x9, [sp, #16]
354 ldr x0, [x9, #80]
355 bl kh_tls_leave_host
356 ldp x0, x1, [sp], #16
357 ldp x9, x30, [sp], #16
358 mov sp, x9
359 // Restore guest NEON from prolog (pre-bl snapshot).
360 ldp q0, q1, [sp, #96]
361 ldp q2, q3, [sp, #128]
362 ldp q4, q5, [sp, #160]
363 ldp q6, q7, [sp, #192]
364 ldp q8, q9, [sp, #224]
365 ldp q10, q11, [sp, #256]
366 ldp q12, q13, [sp, #288]
367 ldp q14, q15, [sp, #320]
368 ldp q16, q17, [sp, #352]
369 ldp q18, q19, [sp, #384]
370 ldp q20, q21, [sp, #416]
371 ldp q22, q23, [sp, #448]
372 ldp q24, q25, [sp, #480]
373 ldp q26, q27, [sp, #512]
374 ldp q28, q29, [sp, #544]
375 ldp q30, q31, [sp, #576]
376 add x11, sp, #608
377 ldp x9, x10, [x11]
378 msr fpcr, x9
379 msr fpsr, x10
380 ldr x29, [sp, #0]
381 add sp, sp, #640
382 ret
3831:
384 // No alt stack: dispatch on the guest stack (ST / fallback only).
385 // Already on host TLS from the enter above; guest_va at [sp, #80].
386 ldp x0, x1, [sp, #16]
387 ldp x2, x3, [sp, #32]
388 ldp x4, x5, [sp, #48]
389 ldp x6, x7, [sp, #64]
390 mov x16, x7
391 // Preserve real LR across the call (still on guest prolog frame).
392 ldr x30, [sp, #8]
393 bl kh_neon_tramp_entry
394 stp x0, x1, [sp, #-16]!
395 ldr x0, [sp, #16+80]
396 bl kh_tls_leave_host
397 ldp x0, x1, [sp], #16
398 // Restore guest NEON from prolog.
399 ldp q0, q1, [sp, #96]
400 ldp q2, q3, [sp, #128]
401 ldp q4, q5, [sp, #160]
402 ldp q6, q7, [sp, #192]
403 ldp q8, q9, [sp, #224]
404 ldp q10, q11, [sp, #256]
405 ldp q12, q13, [sp, #288]
406 ldp q14, q15, [sp, #320]
407 ldp q16, q17, [sp, #352]
408 ldp q18, q19, [sp, #384]
409 ldp q20, q21, [sp, #416]
410 ldp q22, q23, [sp, #448]
411 ldp q24, q25, [sp, #480]
412 ldp q26, q27, [sp, #512]
413 ldp q28, q29, [sp, #544]
414 ldp q30, q31, [sp, #576]
415 add x11, sp, #608
416 ldp x9, x10, [x11]
417 msr fpcr, x9
418 msr fpsr, x10
419 ldp x29, x30, [sp, #0]
420 add sp, sp, #640
421 ret
422 .size kh_hypercall_entry, .-kh_hypercall_entry
423 "#
424);
425
426#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
430std::arch::global_asm!(
431 r#"
432 .text
433 .align 2
434 .global kh_neon_tramp_entry
435 .type kh_neon_tramp_entry, %function
436kh_neon_tramp_entry:
437 adrp x17, KH_NEON_TRAMP_VA
438 ldr x17, [x17, :lo12:KH_NEON_TRAMP_VA]
439 cbz x17, 1f
440 br x17
4411:
442 // No-tramp fallback: number in x16 → AAPCS 8th arg x7; tail call.
443 mov x7, x16
444 b kh_trampoline_dispatch
445 .size kh_neon_tramp_entry, .-kh_neon_tramp_entry
446
447 .data
448 .align 3
449 .global KH_NEON_TRAMP_VA
450KH_NEON_TRAMP_VA:
451 .quad 0
452 "#
453);
454
455#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
456unsafe extern "C" {
457 fn kh_hypercall_entry();
458 static mut KH_NEON_TRAMP_VA: u64;
459}
460
461#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
463#[must_use]
464pub fn hypercall_entry_addr() -> u64 {
465 ensure_neon_tramp();
466 #[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
467 {
468 u64::try_from(kh_hypercall_entry as usize).unwrap_or(0)
469 }
470}
471
472#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
474fn ensure_neon_tramp() {
475 static ONCE: AtomicU64 = AtomicU64::new(0);
476 static LOCK: Mutex<()> = Mutex::new(());
477
478 if ONCE.load(Ordering::Acquire) != 0 {
479 return;
480 }
481 let _g = LOCK
482 .lock()
483 .unwrap_or_else(std::sync::PoisonError::into_inner);
484 if ONCE.load(Ordering::Acquire) != 0 {
485 return;
486 }
487
488 let code = TRAMP_BYTES;
489 let page = crate::host::page_size().unwrap_or(4096);
490 let map_len = align_up_to_page(code.len(), page).max(page);
491 let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
492 let prot_rw = libc::PROT_READ | libc::PROT_WRITE;
493 let Some(ptr) = crate::host::mmap(None, map_len, prot_rw, flags, -1, 0) else {
494 tracing::error!("NEON tramp mmap failed");
495 return;
496 };
497 let slice = unsafe { std::slice::from_raw_parts_mut(ptr, map_len) };
499 slice.fill(0);
500 if let Some(dst) = slice.get_mut(..code.len()) {
501 dst.copy_from_slice(code);
502 } else {
503 let _ = crate::host::munmap(ptr, map_len);
504 return;
505 }
506 if let Some(slot) = slice.get_mut(TRAMP_RELOC_OFF..TRAMP_RELOC_OFF.saturating_add(8)) {
507 slot.copy_from_slice(&trampoline_rust_entry_addr().to_le_bytes());
508 } else {
509 let _ = crate::host::munmap(ptr, map_len);
510 return;
511 }
512 let prot_rx = libc::PROT_READ | libc::PROT_EXEC;
513 if !crate::host::mprotect(ptr, map_len, prot_rx) {
514 tracing::error!("NEON tramp mprotect RX failed");
515 let _ = crate::host::munmap(ptr, map_len);
516 return;
517 }
518 clear_icache(ptr, map_len);
519 let va = crate::host::ptr_addr_u64(ptr);
520 unsafe {
522 KH_NEON_TRAMP_VA = va;
523 }
524 ONCE.store(va, Ordering::Release);
525 tracing::info!(
526 va = format_args!("{va:#x}"),
527 "mapped NEON tramp for hypercall"
528 );
529}
530
531#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
532#[must_use]
533pub fn hypercall_entry_addr() -> u64 {
534 0
535}
536
537#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
544const TRAMP_RELOC_OFF: usize = 0x100;
545
546#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
552#[rustfmt::skip]
553const TRAMP_BYTES: &[u8] = &[
554 0x5f, 0x24, 0x03, 0xd5, 0xff, 0x03, 0x0a, 0xd1, 0xe8, 0x27, 0x00, 0xa9, 0xea, 0x2f, 0x01, 0xa9,
555 0xec, 0x37, 0x02, 0xa9, 0xee, 0x3f, 0x03, 0xa9, 0xf1, 0x4b, 0x04, 0xa9, 0xfe, 0x2b, 0x00, 0xf9,
556 0x08, 0x44, 0x3b, 0xd5, 0x29, 0x44, 0x3b, 0xd5, 0xe8, 0x27, 0x06, 0xa9, 0xe0, 0x87, 0x03, 0xad,
557 0xe2, 0x8f, 0x04, 0xad, 0xe4, 0x97, 0x05, 0xad, 0xe6, 0x9f, 0x06, 0xad, 0xe8, 0xa7, 0x07, 0xad,
558 0xea, 0xaf, 0x08, 0xad, 0xec, 0xb7, 0x09, 0xad, 0xee, 0xbf, 0x0a, 0xad, 0xf0, 0xc7, 0x0b, 0xad,
559 0xf2, 0xcf, 0x0c, 0xad, 0xf4, 0xd7, 0x0d, 0xad, 0xf6, 0xdf, 0x0e, 0xad, 0xf8, 0xe7, 0x0f, 0xad,
560 0xfa, 0xef, 0x10, 0xad, 0xfc, 0xf7, 0x11, 0xad, 0xfe, 0xff, 0x12, 0xad, 0xe7, 0x03, 0x10, 0xaa,
561 0x88, 0x04, 0x00, 0x58, 0x00, 0x01, 0x3f, 0xd6, 0x02, 0x42, 0x3b, 0xd5, 0x23, 0x00, 0x80, 0x52,
562 0x63, 0x88, 0x63, 0xd3, 0x42, 0x00, 0x23, 0x8a, 0x41, 0x00, 0x00, 0xb4, 0x42, 0x00, 0x03, 0xaa,
563 0x02, 0x42, 0x1b, 0xd5, 0xe8, 0x27, 0x46, 0xa9, 0x08, 0x44, 0x1b, 0xd5, 0x29, 0x44, 0x1b, 0xd5,
564 0xe0, 0x87, 0x43, 0xad, 0xe2, 0x8f, 0x44, 0xad, 0xe4, 0x97, 0x45, 0xad, 0xe6, 0x9f, 0x46, 0xad,
565 0xe8, 0xa7, 0x47, 0xad, 0xea, 0xaf, 0x48, 0xad, 0xec, 0xb7, 0x49, 0xad, 0xee, 0xbf, 0x4a, 0xad,
566 0xf0, 0xc7, 0x4b, 0xad, 0xf2, 0xcf, 0x4c, 0xad, 0xf4, 0xd7, 0x4d, 0xad, 0xf6, 0xdf, 0x4e, 0xad,
567 0xf8, 0xe7, 0x4f, 0xad, 0xfa, 0xef, 0x50, 0xad, 0xfc, 0xf7, 0x51, 0xad, 0xfe, 0xff, 0x52, 0xad,
568 0xfe, 0x2b, 0x40, 0xf9, 0xf1, 0x4b, 0x44, 0xa9, 0xee, 0x3f, 0x43, 0xa9, 0xec, 0x37, 0x42, 0xa9,
569 0xea, 0x2f, 0x41, 0xa9, 0xe8, 0x27, 0x40, 0xa9, 0xff, 0x03, 0x0a, 0x91, 0xc0, 0x03, 0x5f, 0xd6,
570 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ];
572pub fn install_trap_handlers(config: &TrapConfig) -> Result<(), TrapError> {
574 MAX_EVENTS.store(
575 u64::try_from(config.max_events).unwrap_or(256),
576 Ordering::SeqCst,
577 );
578 syscall::reset_syscall_state(config.max_syscalls);
579
580 #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
581 {
582 let _ = config;
583 Err(TrapError::UnsupportedArch)
584 }
585
586 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
587 {
588 if HANDLER_INSTALLED.swap(true, Ordering::SeqCst) {
589 reset_trap_state();
590 syscall::reset_syscall_state(config.max_syscalls);
591 return Ok(());
592 }
593
594 disable_core_dumps();
597
598 unsafe {
600 let mut sa: libc::sigaction = std::mem::zeroed();
601 sa.sa_flags = libc::SA_SIGINFO;
602 #[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
603 {
604 sa.sa_sigaction = trap_sigaction as *const () as usize;
605 }
606 libc::sigemptyset(std::ptr::addr_of_mut!(sa.sa_mask));
607 if libc::sigaction(libc::SIGTRAP, std::ptr::addr_of!(sa), std::ptr::null_mut()) != 0 {
608 HANDLER_INSTALLED.store(false, Ordering::SeqCst);
609 return Err(TrapError::SignalSetup(io::Error::last_os_error()));
610 }
611
612 let mut fault: libc::sigaction = std::mem::zeroed();
614 fault.sa_flags = libc::SA_SIGINFO;
615 #[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
616 {
617 fault.sa_sigaction = guest_fault_sigaction as *const () as usize;
618 }
619 libc::sigemptyset(std::ptr::addr_of_mut!(fault.sa_mask));
620 for sig in [libc::SIGSEGV, libc::SIGBUS] {
621 if libc::sigaction(sig, std::ptr::addr_of!(fault), std::ptr::null_mut()) != 0 {
622 HANDLER_INSTALLED.store(false, Ordering::SeqCst);
623 return Err(TrapError::SignalSetup(io::Error::last_os_error()));
624 }
625 }
626 }
627 reset_trap_state();
628 Ok(())
629 }
630}
631
632#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
635fn disable_core_dumps() {
636 unsafe {
638 let lim = libc::rlimit {
639 rlim_cur: 0,
640 rlim_max: 0,
641 };
642 let _ = libc::setrlimit(libc::RLIMIT_CORE, std::ptr::addr_of!(lim));
643 }
644}
645
646#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
648unsafe extern "C" fn guest_fault_sigaction(
649 signo: libc::c_int,
650 info: *mut libc::siginfo_t,
651 ctx: *mut libc::c_void,
652) {
653 crate::tls::enter_host_tls();
655 let name = match signo {
656 libc::SIGSEGV => "SIGSEGV",
657 libc::SIGBUS => "SIGBUS",
658 _ => "FAULT",
659 };
660 let mut pc = 0_u64;
661 let mut sp = 0_u64;
662 let mut x0 = 0_u64;
663 let mut x1 = 0_u64;
664 let mut x2 = 0_u64;
665 let mut x8 = 0_u64;
666 let mut x16 = 0_u64;
667 let mut x19 = 0_u64;
668 let mut x3 = 0_u64;
669 let mut x4 = 0_u64;
670 let mut x5 = 0_u64;
671 let mut x6 = 0_u64;
672 let mut x7 = 0_u64;
673 let mut x9 = 0_u64;
674 let mut x20 = 0_u64;
675 let mut x21 = 0_u64;
676 let mut x22 = 0_u64;
677 let mut x23 = 0_u64;
678 let mut x24 = 0_u64;
679 let mut x25 = 0_u64;
680 let mut x26 = 0_u64;
681 let mut x27 = 0_u64;
682 let mut x28 = 0_u64;
683 let mut x29 = 0_u64;
684 let mut lr = 0_u64;
685 if !ctx.is_null() {
686 let uctx = unsafe { &*ctx.cast::<libc::ucontext_t>() };
688 let m = &uctx.uc_mcontext;
689 pc = m.pc;
690 sp = m.sp;
691 x0 = m.regs[0];
692 x1 = m.regs[1];
693 x2 = m.regs[2];
694 x3 = m.regs[3];
695 x4 = m.regs[4];
696 x5 = m.regs[5];
697 x6 = m.regs[6];
698 x7 = m.regs[7];
699 x8 = m.regs[8];
700 x9 = m.regs[9];
701 x16 = m.regs[16];
702 x19 = m.regs[19];
703 x20 = m.regs[20];
704 x21 = m.regs[21];
705 x22 = m.regs[22];
706 x23 = m.regs[23];
707 x24 = m.regs[24];
708 x25 = m.regs[25];
709 x26 = m.regs[26];
710 x27 = m.regs[27];
711 x28 = m.regs[28];
712 x29 = m.regs[29];
713 lr = m.regs[30];
714 }
715 let addr = if info.is_null() {
716 0_u64
717 } else {
718 let p = unsafe { (*info).si_addr() };
720 u64::try_from(p.addr()).unwrap_or(0)
721 };
722 let msg = format!(
723 "error: guest {name} pc={pc:#x} addr={addr:#x} sp={sp:#x} lr={lr:#x} \
724 x0={x0:#x} x1={x1:#x} x2={x2:#x} x3={x3:#x} x4={x4:#x} x5={x5:#x} \
725 x6={x6:#x} x7={x7:#x} x8={x8:#x} x9={x9:#x} x16={x16:#x} \
726 x19={x19:#x} x20={x20:#x} x21={x21:#x} x22={x22:#x} x23={x23:#x} \
727 x24={x24:#x} x25={x25:#x} x26={x26:#x} x27={x27:#x} x28={x28:#x} x29={x29:#x}\n"
728 );
729 drop(io::stderr().write_all(msg.as_bytes()));
730 if pc != 0 {
732 if let Ok(mut f) = std::fs::File::open("/proc/self/mem") {
733 use std::io::{Read, Seek, SeekFrom};
734 let mut insn = [0_u8; 4];
735 if f.seek(SeekFrom::Start(pc)).is_ok() && f.read_exact(&mut insn).is_ok() {
736 let w = u32::from_le_bytes(insn);
737 let m = format!(" insn@{pc:#x} = {w:#010x} bytes={insn:02x?}\n");
738 drop(io::stderr().write_all(m.as_bytes()));
739 }
740 }
741 }
742 if let Ok(maps) = std::fs::read_to_string("/proc/self/maps") {
744 for &va in &[pc, addr, lr, x0, x1, x2, x16, x19, x25] {
745 if va == 0 {
746 continue;
747 }
748 for line in maps.lines() {
749 let Some((range, rest)) = line.split_once(' ') else {
750 continue;
751 };
752 let Some((a, b)) = range.split_once('-') else {
753 continue;
754 };
755 let (Ok(lo), Ok(hi)) = (u64::from_str_radix(a, 16), u64::from_str_radix(b, 16))
756 else {
757 continue;
758 };
759 if va >= lo && va < hi {
760 let off = va.saturating_sub(lo);
761 let m = format!(" map {va:#x} = {lo:#x}+{off:#x} {rest}\n");
762 drop(io::stderr().write_all(m.as_bytes()));
763 break;
764 }
765 }
766 }
767 }
768 drop(io::stderr().flush());
769 if TRACE_ON_EXIT.load(Ordering::SeqCst) {
770 dump_events_before_exit();
771 }
772 unsafe { libc::_exit(128_i32.saturating_add(signo)) };
774}
775
776#[must_use]
778pub fn take_trace_events() -> Vec<TrapEvent> {
779 EVENTS
780 .lock()
781 .map(|mut g| std::mem::take(&mut *g))
782 .unwrap_or_default()
783}
784
785#[must_use]
787pub fn exit_requested() -> bool {
788 EXIT_REQUESTED.load(Ordering::SeqCst)
789}
790
791#[must_use]
793pub fn take_exit_code() -> Option<i32> {
794 if EXIT_REQUESTED.load(Ordering::SeqCst) {
795 Some(EXIT_CODE.load(Ordering::SeqCst))
796 } else {
797 None
798 }
799}
800
801pub fn reset_trap_state() {
803 EXIT_REQUESTED.store(false, Ordering::SeqCst);
804 EXIT_CODE.store(0, Ordering::SeqCst);
805 EVENT_COUNT.store(0, Ordering::SeqCst);
806 if let Ok(mut guard) = EVENTS.lock() {
807 guard.clear();
808 }
809}
810
811pub fn set_trace_on_exit(json: bool) {
813 TRACE_ON_EXIT.store(true, Ordering::SeqCst);
814 TRACE_JSON.store(json, Ordering::SeqCst);
815}
816
817pub fn clear_trace_on_exit() {
819 TRACE_ON_EXIT.store(false, Ordering::SeqCst);
820 TRACE_JSON.store(false, Ordering::SeqCst);
821}
822
823pub fn set_expect_code(code: i32) {
825 EXPECT_CODE.store(code, Ordering::SeqCst);
826}
827
828pub fn clear_expect_code() {
830 EXPECT_CODE.store(i32::MIN, Ordering::SeqCst);
831}
832
833pub fn finish_with_exit_code(code: i32) -> ! {
838 EXIT_CODE.store(code, Ordering::SeqCst);
839 EXIT_REQUESTED.store(true, Ordering::SeqCst);
840 if TRACE_ON_EXIT.load(Ordering::SeqCst) {
841 dump_events_before_exit();
842 }
843 crate::syscall::dump_futex_stats_if_enabled();
845 crate::syscall::dump_boundary_stats_if_enabled();
846 let expect = EXPECT_CODE.load(Ordering::SeqCst);
847 let status = if expect == i32::MIN || expect == code {
848 code
849 } else {
850 let msg = format!("error: guest exit code {code} != expected {expect}\n");
851 drop(io::stderr().write_all(msg.as_bytes()));
852 1
853 };
854 unsafe { libc::_exit(status) };
856}
857
858#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
859unsafe extern "C" fn trap_sigaction(
860 signo: libc::c_int,
861 _info: *mut libc::siginfo_t,
862 ctx: *mut libc::c_void,
863) {
864 if signo != libc::SIGTRAP || ctx.is_null() {
865 return;
866 }
867
868 crate::tls::enter_host_tls();
870
871 let uctx = unsafe { &mut *ctx.cast::<libc::ucontext_t>() };
873 let m = &mut uctx.uc_mcontext;
874 let pc = m.pc;
875 let x0 = m.regs[0];
877 let x1 = m.regs[1];
878 let x2 = m.regs[2];
879 let x3 = m.regs[3];
880 let x4 = m.regs[4];
881 let x5 = m.regs[5];
882 let x6 = m.regs[6];
883 let x16 = m.regs[16];
884
885 let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
886 let result = syscall::dispatch(SyscallArgs {
887 pc,
888 number: sys_no,
889 x0,
890 x1,
891 x2,
892 x3,
893 x4,
894 x5,
895 x6,
896 });
897
898 if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
900 maybe_push_event(
901 pc,
902 sys_no,
903 result.name,
904 x0,
905 x1,
906 x2,
907 result.retval,
908 result.error,
909 );
910 }
911
912 if let Some(ret) = result.retval {
913 m.regs[0] = ret;
914 }
915 if result.error {
917 m.pstate |= PSTATE_C;
918 } else if result.retval.is_some() {
919 m.pstate &= !PSTATE_C;
920 }
921
922 match result.outcome {
923 TrapOutcome::Exit { code } => {
924 finish_with_exit_code(code);
925 }
926 TrapOutcome::Continue => {
927 m.pc = pc.wrapping_add(4);
928 crate::tls::leave_host_tls();
930 }
931 TrapOutcome::ThreadExit => {
932 if !crate::thread::redirect_ucontext_to_host_exit(m) {
937 finish_with_exit_code(0);
938 }
939 }
940 }
941}
942
943fn dump_events_before_exit() {
944 use std::fmt::Write as _;
945
946 let json = TRACE_JSON.load(Ordering::SeqCst);
947 let events = EVENTS.lock().map(|g| g.clone()).unwrap_or_default();
948 if json {
949 let mut out = String::from("{\"events\":[");
950 for (i, e) in events.iter().enumerate() {
951 if i > 0 {
952 out.push(',');
953 }
954 let sys = e
955 .syscall
956 .map_or_else(|| "null".to_owned(), |s| s.to_string());
957 let ret = e
958 .retval
959 .map_or_else(|| "null".to_owned(), |v| v.to_string());
960 let _ = write!(
961 out,
962 "{{\"pc\":{},\"syscall\":{},\"name\":\"{}\",\"arg0\":{},\"arg1\":{},\"arg2\":{},\"retval\":{},\"error\":{}}}",
963 e.pc, sys, e.name, e.arg0, e.arg1, e.arg2, ret, e.error
964 );
965 }
966 out.push_str("]}\n");
967 drop(io::stdout().write_all(out.as_bytes()));
968 drop(io::stdout().flush());
969 } else {
970 let header = format!("trace events: {}\n", events.len());
971 drop(io::stdout().write_all(header.as_bytes()));
972 for (i, e) in events.iter().enumerate() {
973 let ret = e
974 .retval
975 .map_or_else(|| "-".to_owned(), |v| format!("{v:#x}"));
976 let err = if e.error { " err" } else { "" };
977 let line = format!(
978 " [{i}] pc={:#x} {} sys={:?} args=({:#x}, {:#x}, {:#x}) ret={ret}{err}\n",
979 e.pc, e.name, e.syscall, e.arg0, e.arg1, e.arg2
980 );
981 drop(io::stdout().write_all(line.as_bytes()));
982 }
983 drop(io::stdout().flush());
984 }
985}
986
987#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
988#[allow(clippy::too_many_arguments)]
989fn maybe_push_event(
990 pc: u64,
991 sys_no: u32,
992 name: &'static str,
993 arg0: u64,
994 arg1: u64,
995 arg2: u64,
996 retval: Option<u64>,
997 error: bool,
998) {
999 let max = MAX_EVENTS.load(Ordering::Relaxed);
1000 if max == 0 {
1002 return;
1003 }
1004 let n = EVENT_COUNT.fetch_add(1, Ordering::Relaxed);
1005 if n >= max {
1006 return;
1007 }
1008 if let Ok(mut guard) = EVENTS.lock()
1009 && u64::try_from(guard.len()).unwrap_or(u64::MAX) < max
1010 {
1011 guard.push(TrapEvent {
1012 pc,
1013 syscall: Some(sys_no),
1014 name: name.to_owned(),
1015 arg0,
1016 arg1,
1017 arg2,
1018 retval,
1019 error,
1020 });
1021 }
1022}
1023
1024#[cfg(test)]
1025mod tests {
1026 use super::*;
1027
1028 #[test]
1029 fn detects_svc_encoding() {
1030 assert!(is_svc(0xD400_0001));
1031 let svc80 = 0xD400_0001 | (0x80_u32 << 5);
1032 assert!(is_svc(svc80));
1033 assert!(!is_svc(0xD503_201F));
1034 assert!(!is_svc(brk_encoding(BRK_TRAP_IMM)));
1035 }
1036
1037 #[test]
1038 fn brk_encoding_roundtrip_imm() {
1039 let enc = brk_encoding(BRK_TRAP_IMM);
1040 let imm = (enc >> 5) & 0xFFFF;
1041 assert_eq!(imm, BRK_TRAP_IMM);
1042 }
1043}