1#![allow(unsafe_code)]
14
15use std::io::{self, Write};
16use std::sync::Mutex;
17use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering};
18
19use thiserror::Error;
20
21use crate::mem::{
22 MapError, MappedRegion, VM_PROT_EXECUTE, VM_PROT_READ, VM_PROT_WRITE, mprotect_darwin,
23 mprotect_rw,
24};
25use crate::syscall;
26#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
27use crate::syscall::SyscallArgs;
28
29pub const BRK_TRAP_IMM: u32 = 0x0F00;
31
32const SVC_MASK: u32 = 0xFFE0_001F;
34const SVC_BASE: u32 = 0xD400_0001;
35
36const fn brk_encoding(imm16: u32) -> u32 {
38 0xD420_0000 | ((imm16 & 0xFFFF) << 5)
39}
40
41pub const PSTATE_C: u64 = 1 << 29;
45
46#[derive(Debug, Clone, PartialEq, Eq)]
48pub struct TrapEvent {
49 pub pc: u64,
51 pub syscall: Option<u32>,
53 pub name: String,
55 pub arg0: u64,
57 pub arg1: u64,
59 pub arg2: u64,
61 pub retval: Option<u64>,
63 pub error: bool,
65}
66
67#[derive(Debug, Clone, Copy, PartialEq, Eq)]
69pub enum TrapOutcome {
70 Exit {
72 code: i32,
74 },
75 Continue,
77 ThreadExit,
79}
80
81#[derive(Debug, Clone)]
83pub struct TrapConfig {
84 pub max_events: usize,
86 pub max_syscalls: usize,
88}
89
90impl Default for TrapConfig {
91 fn default() -> Self {
92 Self {
93 max_events: 256,
94 max_syscalls: 256,
95 }
96 }
97}
98
99#[derive(Debug, Error)]
101pub enum TrapError {
102 #[error("failed to install trap signal handler: {0}")]
104 SignalSetup(#[source] io::Error),
105
106 #[error(transparent)]
108 Map(#[from] MapError),
109
110 #[error("trap backend requires Linux aarch64 for live execution")]
112 UnsupportedArch,
113}
114
115#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
116static HANDLER_INSTALLED: AtomicBool = AtomicBool::new(false);
117static EXIT_CODE: AtomicI32 = AtomicI32::new(0);
118static EXIT_REQUESTED: AtomicBool = AtomicBool::new(false);
119static EVENT_COUNT: AtomicU64 = AtomicU64::new(0);
120static MAX_EVENTS: AtomicU64 = AtomicU64::new(256);
121static TRACE_ON_EXIT: AtomicBool = AtomicBool::new(false);
122static TRACE_JSON: AtomicBool = AtomicBool::new(false);
123static EXPECT_CODE: AtomicI32 = AtomicI32::new(i32::MIN);
125static EVENTS: Mutex<Vec<TrapEvent>> = Mutex::new(Vec::new());
126
127pub fn patch_svc_to_brk(regions: &mut [MappedRegion]) -> Result<usize, TrapError> {
134 let mut patched = 0_usize;
135 for region in regions.iter_mut() {
136 if region.prot & VM_PROT_EXECUTE == 0 {
137 continue;
138 }
139 let file_len = usize::try_from(region.file_bytes)
141 .unwrap_or(0)
142 .min(region.host_len());
143 if file_len < 4 {
144 continue;
145 }
146
147 let ranges = resolve_svc_scan_ranges(region, file_len);
151 if ranges.is_empty() {
152 continue;
153 }
154
155 let hits = {
157 let bytes = region.host_bytes();
158 let mut hits = Vec::new();
159 for &(start, len) in &ranges {
160 let end = start.saturating_add(len).min(file_len);
161 if end.saturating_sub(start) < 4 {
162 continue;
163 }
164 let Some(scan) = bytes.get(start..end) else {
165 continue;
166 };
167 hits.extend(find_svc_offsets(scan).into_iter().map(|o| start.saturating_add(o)));
168 }
169 hits
170 };
171 if hits.is_empty() {
172 continue;
173 }
174
175 mprotect_rw(region)?;
177 let bytes = region.host_bytes_mut();
178 let brk = brk_encoding(BRK_TRAP_IMM).to_le_bytes();
179 for off in &hits {
180 let end = off.saturating_add(4);
181 if let Some(slot) = bytes.get_mut(*off..end) {
182 slot.copy_from_slice(&brk);
183 patched = patched.saturating_add(1);
184 }
185 }
186 let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
187 mprotect_darwin(region, restore)?;
188 region.prot = restore;
189 let base = region.host_bytes().as_ptr();
192 for off in hits {
193 let ptr = unsafe { base.add(off).cast_mut() };
195 clear_icache(ptr, 4);
196 }
197 }
198 Ok(patched)
199}
200
201fn resolve_svc_scan_ranges(region: &MappedRegion, file_len: usize) -> Vec<(usize, usize)> {
203 if region.svc_scan_ranges.is_empty() {
204 return vec![(0, file_len)];
205 }
206 let mut out = Vec::with_capacity(region.svc_scan_ranges.len());
207 for &(off, len) in ®ion.svc_scan_ranges {
208 let start = usize::try_from(off).unwrap_or(usize::MAX);
209 let len = usize::try_from(len).unwrap_or(0);
210 if start >= file_len || len == 0 {
211 continue;
212 }
213 let len = len.min(file_len.saturating_sub(start));
214 if len >= 4 {
215 out.push((start, len));
216 }
217 }
218 out
219}
220
221const fn is_svc(insn: u32) -> bool {
222 (insn & SVC_MASK) == SVC_BASE
223}
224
225#[allow(clippy::integer_division)] fn find_svc_offsets(scan: &[u8]) -> Vec<usize> {
231 const PARALLEL_MIN: usize = 8 << 20; if scan.len() >= PARALLEL_MIN {
233 return find_svc_offsets_parallel(scan);
234 }
235 find_svc_offsets_range(scan, 0)
236}
237
238#[allow(clippy::integer_division)]
240fn find_svc_offsets_parallel(scan: &[u8]) -> Vec<usize> {
241 let nthreads = std::thread::available_parallelism()
242 .map_or(1, std::num::NonZeroUsize::get)
243 .clamp(1, 8);
244 if nthreads <= 1 {
245 return find_svc_offsets_range(scan, 0);
246 }
247 let total = scan.len() & !3_usize;
248 if total < 4 {
249 return Vec::new();
250 }
251 let chunk = (total
252 .checked_div(nthreads)
253 .unwrap_or(total)
254 & !3_usize)
255 .max(4);
256 let mut hits = Vec::new();
257 std::thread::scope(|scope| {
258 let mut handles = Vec::with_capacity(nthreads);
259 let mut start = 0_usize;
260 while start < total {
261 let end = (start.saturating_add(chunk)).min(total);
262 let Some(part) = scan.get(start..end) else {
263 break;
264 };
265 let base_off = start;
266 handles.push(scope.spawn(move || find_svc_offsets_range(part, base_off)));
267 start = end;
268 }
269 for h in handles {
270 if let Ok(mut part) = h.join() {
271 hits.append(&mut part);
272 }
273 }
274 });
275 hits.sort_unstable();
276 hits
277}
278
279#[allow(clippy::integer_division)]
282fn find_svc_offsets_range(scan: &[u8], base_off: usize) -> Vec<usize> {
283 let mut hits = Vec::new();
284 let words = scan.len() / 4;
285 if words == 0 {
286 return hits;
287 }
288 let base = scan.as_ptr();
291 let mut i = 0_usize;
292 while i.saturating_add(4) <= words {
293 let w0 = unsafe { base.add(i.saturating_mul(4)).cast::<u32>().read_unaligned() };
295 let w1 = unsafe {
296 base.add(i.saturating_add(1).saturating_mul(4))
297 .cast::<u32>()
298 .read_unaligned()
299 };
300 let w2 = unsafe {
301 base.add(i.saturating_add(2).saturating_mul(4))
302 .cast::<u32>()
303 .read_unaligned()
304 };
305 let w3 = unsafe {
306 base.add(i.saturating_add(3).saturating_mul(4))
307 .cast::<u32>()
308 .read_unaligned()
309 };
310 if is_svc(u32::from_le(w0)) {
311 hits.push(base_off.saturating_add(i.saturating_mul(4)));
312 }
313 if is_svc(u32::from_le(w1)) {
314 hits.push(
315 base_off.saturating_add(i.saturating_add(1).saturating_mul(4)),
316 );
317 }
318 if is_svc(u32::from_le(w2)) {
319 hits.push(
320 base_off.saturating_add(i.saturating_add(2).saturating_mul(4)),
321 );
322 }
323 if is_svc(u32::from_le(w3)) {
324 hits.push(
325 base_off.saturating_add(i.saturating_add(3).saturating_mul(4)),
326 );
327 }
328 i = i.saturating_add(4);
329 }
330 while i < words {
331 let w = unsafe { base.add(i.saturating_mul(4)).cast::<u32>().read_unaligned() };
332 if is_svc(u32::from_le(w)) {
333 hits.push(base_off.saturating_add(i.saturating_mul(4)));
334 }
335 i = i.saturating_add(1);
336 }
337 hits
338}
339
340#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
342#[inline]
343fn align_up_to_page(len: usize, page: usize) -> usize {
344 if page <= 1 {
345 return len.max(1);
346 }
347 let mask = page.saturating_sub(1);
348 len.saturating_add(mask) & !mask
349}
350
351#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
353unsafe extern "C" {
354 fn __clear_cache(start: *mut libc::c_void, end: *mut libc::c_void);
355}
356
357#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
359fn clear_icache(ptr: *mut u8, len: usize) {
360 if ptr.is_null() || len == 0 {
361 return;
362 }
363 unsafe {
365 let start = ptr.cast::<libc::c_void>();
366 let end = ptr.wrapping_add(len).cast::<libc::c_void>();
367 __clear_cache(start, end);
368 }
369}
370
371#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
372fn clear_icache(_ptr: *mut u8, _len: usize) {}
373
374#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
375#[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
376fn trampoline_rust_entry_addr() -> u64 {
377 u64::try_from(kh_trampoline_dispatch as usize).unwrap_or(0)
378}
379
380#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
382#[repr(C)]
383pub struct TrampRet {
384 pub retval: u64,
386 pub error: u64,
388}
389
390#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
402#[unsafe(no_mangle)]
403pub unsafe extern "C" fn kh_trampoline_dispatch(
404 x0: u64,
405 x1: u64,
406 x2: u64,
407 x3: u64,
408 x4: u64,
409 x5: u64,
410 x6: u64,
411 x16: u64,
412) -> TrampRet {
413 let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
414 let result = syscall::dispatch(SyscallArgs {
415 pc: 0,
416 number: sys_no,
417 x0,
418 x1,
419 x2,
420 x3,
421 x4,
422 x5,
423 x6,
424 });
425 if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
426 maybe_push_event(
427 0,
428 sys_no,
429 result.name,
430 x0,
431 x1,
432 x2,
433 result.retval,
434 result.error,
435 );
436 }
437 match result.outcome {
438 TrapOutcome::Exit { code } => finish_with_exit_code(code),
439 TrapOutcome::ThreadExit => {
440 crate::thread::exit_current_guest_worker();
442 }
443 TrapOutcome::Continue => {}
444 }
445 TrampRet {
446 retval: result.retval.unwrap_or(0),
447 error: u64::from(result.error),
448 }
449}
450
451#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
470std::arch::global_asm!(
471 r#"
472 .text
473 .align 2
474 .global kh_hypercall_entry
475 .type kh_hypercall_entry, %function
476kh_hypercall_entry:
477 // Guest prolog: GPRs + full NEON *before* any bl (NEON must survive host TLS).
478 sub sp, sp, #640
479 stp x29, x30, [sp, #0]
480 stp x0, x1, [sp, #16]
481 stp x2, x3, [sp, #32]
482 stp x4, x5, [sp, #48]
483 stp x6, x7, [sp, #64]
484 mov x29, sp
485 stp q0, q1, [sp, #96]
486 stp q2, q3, [sp, #128]
487 stp q4, q5, [sp, #160]
488 stp q6, q7, [sp, #192]
489 stp q8, q9, [sp, #224]
490 stp q10, q11, [sp, #256]
491 stp q12, q13, [sp, #288]
492 stp q14, q15, [sp, #320]
493 stp q16, q17, [sp, #352]
494 stp q18, q19, [sp, #384]
495 stp q20, q21, [sp, #416]
496 stp q22, q23, [sp, #448]
497 stp q24, q25, [sp, #480]
498 stp q26, q27, [sp, #512]
499 stp q28, q29, [sp, #544]
500 stp q30, q31, [sp, #576]
501 mrs x9, fpcr
502 mrs x10, fpsr
503 // #608 exceeds signed LDP imm7*8 range (±512); use base+add.
504 add x11, sp, #608
505 stp x9, x10, [x11]
506
507 // A1/A2: enter returns alt_top in x0, guest_tpidr in x1 (HyperEnterRet).
508 // Park guest_va at prolog+80 (hole before NEON) so leave needs no gettid.
509 bl kh_tls_enter_host
510 mov x9, sp // guest prolog frame
511 str x1, [x9, #80] // guest_tpidr for leave
512 cbz x0, 1f
513 mov sp, x0 // host alt stack
514 ldr x30, [x9, #8] // reload LR → freestanding thin caller
515 // Host frame: [guest_frame, lr]
516 stp x9, x30, [sp, #-16]!
517 // Reload guest frame ptr / args (enter_host clobbered temps).
518 ldr x9, [sp]
519 ldp x0, x1, [x9, #16]
520 ldp x2, x3, [x9, #32]
521 ldp x4, x5, [x9, #48]
522 ldp x6, x7, [x9, #64]
523 mov x16, x7
524 bl kh_neon_tramp_entry
525 // Preserve TrampRet (x0/x1) across TLS restore.
526 stp x0, x1, [sp, #-16]!
527 // guest_va at host_frame.guest_frame + 80; host_frame at sp+16.
528 ldr x9, [sp, #16]
529 ldr x0, [x9, #80]
530 bl kh_tls_leave_host
531 ldp x0, x1, [sp], #16
532 ldp x9, x30, [sp], #16
533 mov sp, x9
534 // Restore guest NEON from prolog (pre-bl snapshot).
535 ldp q0, q1, [sp, #96]
536 ldp q2, q3, [sp, #128]
537 ldp q4, q5, [sp, #160]
538 ldp q6, q7, [sp, #192]
539 ldp q8, q9, [sp, #224]
540 ldp q10, q11, [sp, #256]
541 ldp q12, q13, [sp, #288]
542 ldp q14, q15, [sp, #320]
543 ldp q16, q17, [sp, #352]
544 ldp q18, q19, [sp, #384]
545 ldp q20, q21, [sp, #416]
546 ldp q22, q23, [sp, #448]
547 ldp q24, q25, [sp, #480]
548 ldp q26, q27, [sp, #512]
549 ldp q28, q29, [sp, #544]
550 ldp q30, q31, [sp, #576]
551 add x11, sp, #608
552 ldp x9, x10, [x11]
553 msr fpcr, x9
554 msr fpsr, x10
555 ldr x29, [sp, #0]
556 add sp, sp, #640
557 ret
5581:
559 // No alt stack: dispatch on the guest stack (ST / fallback only).
560 // Already on host TLS from the enter above; guest_va at [sp, #80].
561 ldp x0, x1, [sp, #16]
562 ldp x2, x3, [sp, #32]
563 ldp x4, x5, [sp, #48]
564 ldp x6, x7, [sp, #64]
565 mov x16, x7
566 // Preserve real LR across the call (still on guest prolog frame).
567 ldr x30, [sp, #8]
568 bl kh_neon_tramp_entry
569 stp x0, x1, [sp, #-16]!
570 ldr x0, [sp, #16+80]
571 bl kh_tls_leave_host
572 ldp x0, x1, [sp], #16
573 // Restore guest NEON from prolog.
574 ldp q0, q1, [sp, #96]
575 ldp q2, q3, [sp, #128]
576 ldp q4, q5, [sp, #160]
577 ldp q6, q7, [sp, #192]
578 ldp q8, q9, [sp, #224]
579 ldp q10, q11, [sp, #256]
580 ldp q12, q13, [sp, #288]
581 ldp q14, q15, [sp, #320]
582 ldp q16, q17, [sp, #352]
583 ldp q18, q19, [sp, #384]
584 ldp q20, q21, [sp, #416]
585 ldp q22, q23, [sp, #448]
586 ldp q24, q25, [sp, #480]
587 ldp q26, q27, [sp, #512]
588 ldp q28, q29, [sp, #544]
589 ldp q30, q31, [sp, #576]
590 add x11, sp, #608
591 ldp x9, x10, [x11]
592 msr fpcr, x9
593 msr fpsr, x10
594 ldp x29, x30, [sp, #0]
595 add sp, sp, #640
596 ret
597 .size kh_hypercall_entry, .-kh_hypercall_entry
598 "#
599);
600
601#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
605std::arch::global_asm!(
606 r#"
607 .text
608 .align 2
609 .global kh_neon_tramp_entry
610 .type kh_neon_tramp_entry, %function
611kh_neon_tramp_entry:
612 adrp x17, KH_NEON_TRAMP_VA
613 ldr x17, [x17, :lo12:KH_NEON_TRAMP_VA]
614 cbz x17, 1f
615 br x17
6161:
617 // No-tramp fallback: number in x16 → AAPCS 8th arg x7; tail call.
618 mov x7, x16
619 b kh_trampoline_dispatch
620 .size kh_neon_tramp_entry, .-kh_neon_tramp_entry
621
622 .data
623 .align 3
624 .global KH_NEON_TRAMP_VA
625KH_NEON_TRAMP_VA:
626 .quad 0
627 "#
628);
629
630#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
631unsafe extern "C" {
632 fn kh_hypercall_entry();
633 static mut KH_NEON_TRAMP_VA: u64;
634}
635
636#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
638#[must_use]
639pub fn hypercall_entry_addr() -> u64 {
640 ensure_neon_tramp();
641 #[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
642 {
643 u64::try_from(kh_hypercall_entry as usize).unwrap_or(0)
644 }
645}
646
647#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
649fn ensure_neon_tramp() {
650 static ONCE: AtomicU64 = AtomicU64::new(0);
651 static LOCK: Mutex<()> = Mutex::new(());
652
653 if ONCE.load(Ordering::Acquire) != 0 {
654 return;
655 }
656 let _g = LOCK
657 .lock()
658 .unwrap_or_else(std::sync::PoisonError::into_inner);
659 if ONCE.load(Ordering::Acquire) != 0 {
660 return;
661 }
662
663 let code = TRAMP_BYTES;
664 let page = crate::host::page_size().unwrap_or(4096);
665 let map_len = align_up_to_page(code.len(), page).max(page);
666 let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
667 let prot_rw = libc::PROT_READ | libc::PROT_WRITE;
668 let Some(ptr) = crate::host::mmap(None, map_len, prot_rw, flags, -1, 0) else {
669 tracing::error!("NEON tramp mmap failed");
670 return;
671 };
672 let slice = unsafe { std::slice::from_raw_parts_mut(ptr, map_len) };
674 slice.fill(0);
675 if let Some(dst) = slice.get_mut(..code.len()) {
676 dst.copy_from_slice(code);
677 } else {
678 let _ = crate::host::munmap(ptr, map_len);
679 return;
680 }
681 if let Some(slot) = slice.get_mut(TRAMP_RELOC_OFF..TRAMP_RELOC_OFF.saturating_add(8)) {
682 slot.copy_from_slice(&trampoline_rust_entry_addr().to_le_bytes());
683 } else {
684 let _ = crate::host::munmap(ptr, map_len);
685 return;
686 }
687 let prot_rx = libc::PROT_READ | libc::PROT_EXEC;
688 if !crate::host::mprotect(ptr, map_len, prot_rx) {
689 tracing::error!("NEON tramp mprotect RX failed");
690 let _ = crate::host::munmap(ptr, map_len);
691 return;
692 }
693 clear_icache(ptr, map_len);
694 let va = crate::host::ptr_addr_u64(ptr);
695 unsafe {
697 KH_NEON_TRAMP_VA = va;
698 }
699 ONCE.store(va, Ordering::Release);
700 tracing::info!(
701 va = format_args!("{va:#x}"),
702 "mapped NEON tramp for hypercall"
703 );
704}
705
706#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
707#[must_use]
708pub fn hypercall_entry_addr() -> u64 {
709 0
710}
711
712#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
719const TRAMP_RELOC_OFF: usize = 0x100;
720
721#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
727#[rustfmt::skip]
728const TRAMP_BYTES: &[u8] = &[
729 0x5f, 0x24, 0x03, 0xd5, 0xff, 0x03, 0x0a, 0xd1, 0xe8, 0x27, 0x00, 0xa9, 0xea, 0x2f, 0x01, 0xa9,
730 0xec, 0x37, 0x02, 0xa9, 0xee, 0x3f, 0x03, 0xa9, 0xf1, 0x4b, 0x04, 0xa9, 0xfe, 0x2b, 0x00, 0xf9,
731 0x08, 0x44, 0x3b, 0xd5, 0x29, 0x44, 0x3b, 0xd5, 0xe8, 0x27, 0x06, 0xa9, 0xe0, 0x87, 0x03, 0xad,
732 0xe2, 0x8f, 0x04, 0xad, 0xe4, 0x97, 0x05, 0xad, 0xe6, 0x9f, 0x06, 0xad, 0xe8, 0xa7, 0x07, 0xad,
733 0xea, 0xaf, 0x08, 0xad, 0xec, 0xb7, 0x09, 0xad, 0xee, 0xbf, 0x0a, 0xad, 0xf0, 0xc7, 0x0b, 0xad,
734 0xf2, 0xcf, 0x0c, 0xad, 0xf4, 0xd7, 0x0d, 0xad, 0xf6, 0xdf, 0x0e, 0xad, 0xf8, 0xe7, 0x0f, 0xad,
735 0xfa, 0xef, 0x10, 0xad, 0xfc, 0xf7, 0x11, 0xad, 0xfe, 0xff, 0x12, 0xad, 0xe7, 0x03, 0x10, 0xaa,
736 0x88, 0x04, 0x00, 0x58, 0x00, 0x01, 0x3f, 0xd6, 0x02, 0x42, 0x3b, 0xd5, 0x23, 0x00, 0x80, 0x52,
737 0x63, 0x88, 0x63, 0xd3, 0x42, 0x00, 0x23, 0x8a, 0x41, 0x00, 0x00, 0xb4, 0x42, 0x00, 0x03, 0xaa,
738 0x02, 0x42, 0x1b, 0xd5, 0xe8, 0x27, 0x46, 0xa9, 0x08, 0x44, 0x1b, 0xd5, 0x29, 0x44, 0x1b, 0xd5,
739 0xe0, 0x87, 0x43, 0xad, 0xe2, 0x8f, 0x44, 0xad, 0xe4, 0x97, 0x45, 0xad, 0xe6, 0x9f, 0x46, 0xad,
740 0xe8, 0xa7, 0x47, 0xad, 0xea, 0xaf, 0x48, 0xad, 0xec, 0xb7, 0x49, 0xad, 0xee, 0xbf, 0x4a, 0xad,
741 0xf0, 0xc7, 0x4b, 0xad, 0xf2, 0xcf, 0x4c, 0xad, 0xf4, 0xd7, 0x4d, 0xad, 0xf6, 0xdf, 0x4e, 0xad,
742 0xf8, 0xe7, 0x4f, 0xad, 0xfa, 0xef, 0x50, 0xad, 0xfc, 0xf7, 0x51, 0xad, 0xfe, 0xff, 0x52, 0xad,
743 0xfe, 0x2b, 0x40, 0xf9, 0xf1, 0x4b, 0x44, 0xa9, 0xee, 0x3f, 0x43, 0xa9, 0xec, 0x37, 0x42, 0xa9,
744 0xea, 0x2f, 0x41, 0xa9, 0xe8, 0x27, 0x40, 0xa9, 0xff, 0x03, 0x0a, 0x91, 0xc0, 0x03, 0x5f, 0xd6,
745 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ];
747pub fn install_trap_handlers(config: &TrapConfig) -> Result<(), TrapError> {
749 MAX_EVENTS.store(
750 u64::try_from(config.max_events).unwrap_or(256),
751 Ordering::SeqCst,
752 );
753 syscall::reset_syscall_state(config.max_syscalls);
754
755 #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
756 {
757 let _ = config;
758 Err(TrapError::UnsupportedArch)
759 }
760
761 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
762 {
763 if HANDLER_INSTALLED.swap(true, Ordering::SeqCst) {
764 reset_trap_state();
765 syscall::reset_syscall_state(config.max_syscalls);
766 return Ok(());
767 }
768
769 disable_core_dumps();
772
773 unsafe {
775 let mut sa: libc::sigaction = std::mem::zeroed();
776 sa.sa_flags = libc::SA_SIGINFO;
777 #[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
778 {
779 sa.sa_sigaction = trap_sigaction as *const () as usize;
780 }
781 libc::sigemptyset(std::ptr::addr_of_mut!(sa.sa_mask));
782 if libc::sigaction(libc::SIGTRAP, std::ptr::addr_of!(sa), std::ptr::null_mut()) != 0 {
783 HANDLER_INSTALLED.store(false, Ordering::SeqCst);
784 return Err(TrapError::SignalSetup(io::Error::last_os_error()));
785 }
786
787 let mut fault: libc::sigaction = std::mem::zeroed();
789 fault.sa_flags = libc::SA_SIGINFO;
790 #[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
791 {
792 fault.sa_sigaction = guest_fault_sigaction as *const () as usize;
793 }
794 libc::sigemptyset(std::ptr::addr_of_mut!(fault.sa_mask));
795 for sig in [libc::SIGSEGV, libc::SIGBUS] {
796 if libc::sigaction(sig, std::ptr::addr_of!(fault), std::ptr::null_mut()) != 0 {
797 HANDLER_INSTALLED.store(false, Ordering::SeqCst);
798 return Err(TrapError::SignalSetup(io::Error::last_os_error()));
799 }
800 }
801 }
802 reset_trap_state();
803 Ok(())
804 }
805}
806
807#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
810fn disable_core_dumps() {
811 unsafe {
813 let lim = libc::rlimit {
814 rlim_cur: 0,
815 rlim_max: 0,
816 };
817 let _ = libc::setrlimit(libc::RLIMIT_CORE, std::ptr::addr_of!(lim));
818 }
819}
820
821#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
823unsafe extern "C" fn guest_fault_sigaction(
824 signo: libc::c_int,
825 info: *mut libc::siginfo_t,
826 ctx: *mut libc::c_void,
827) {
828 crate::tls::enter_host_tls();
830 let name = match signo {
831 libc::SIGSEGV => "SIGSEGV",
832 libc::SIGBUS => "SIGBUS",
833 _ => "FAULT",
834 };
835 let mut pc = 0_u64;
836 let mut sp = 0_u64;
837 let mut x0 = 0_u64;
838 let mut x1 = 0_u64;
839 let mut x2 = 0_u64;
840 let mut x8 = 0_u64;
841 let mut x16 = 0_u64;
842 let mut x19 = 0_u64;
843 let mut x3 = 0_u64;
844 let mut x4 = 0_u64;
845 let mut x5 = 0_u64;
846 let mut x6 = 0_u64;
847 let mut x7 = 0_u64;
848 let mut x9 = 0_u64;
849 let mut x20 = 0_u64;
850 let mut x21 = 0_u64;
851 let mut x22 = 0_u64;
852 let mut x23 = 0_u64;
853 let mut x24 = 0_u64;
854 let mut x25 = 0_u64;
855 let mut x26 = 0_u64;
856 let mut x27 = 0_u64;
857 let mut x28 = 0_u64;
858 let mut x29 = 0_u64;
859 let mut lr = 0_u64;
860 if !ctx.is_null() {
861 let uctx = unsafe { &*ctx.cast::<libc::ucontext_t>() };
863 let m = &uctx.uc_mcontext;
864 pc = m.pc;
865 sp = m.sp;
866 x0 = m.regs[0];
867 x1 = m.regs[1];
868 x2 = m.regs[2];
869 x3 = m.regs[3];
870 x4 = m.regs[4];
871 x5 = m.regs[5];
872 x6 = m.regs[6];
873 x7 = m.regs[7];
874 x8 = m.regs[8];
875 x9 = m.regs[9];
876 x16 = m.regs[16];
877 x19 = m.regs[19];
878 x20 = m.regs[20];
879 x21 = m.regs[21];
880 x22 = m.regs[22];
881 x23 = m.regs[23];
882 x24 = m.regs[24];
883 x25 = m.regs[25];
884 x26 = m.regs[26];
885 x27 = m.regs[27];
886 x28 = m.regs[28];
887 x29 = m.regs[29];
888 lr = m.regs[30];
889 }
890 let addr = if info.is_null() {
891 0_u64
892 } else {
893 let p = unsafe { (*info).si_addr() };
895 u64::try_from(p.addr()).unwrap_or(0)
896 };
897 let msg = format!(
898 "error: guest {name} pc={pc:#x} addr={addr:#x} sp={sp:#x} lr={lr:#x} \
899 x0={x0:#x} x1={x1:#x} x2={x2:#x} x3={x3:#x} x4={x4:#x} x5={x5:#x} \
900 x6={x6:#x} x7={x7:#x} x8={x8:#x} x9={x9:#x} x16={x16:#x} \
901 x19={x19:#x} x20={x20:#x} x21={x21:#x} x22={x22:#x} x23={x23:#x} \
902 x24={x24:#x} x25={x25:#x} x26={x26:#x} x27={x27:#x} x28={x28:#x} x29={x29:#x}\n"
903 );
904 drop(io::stderr().write_all(msg.as_bytes()));
905 if pc != 0 {
907 if let Ok(mut f) = std::fs::File::open("/proc/self/mem") {
908 use std::io::{Read, Seek, SeekFrom};
909 let mut insn = [0_u8; 4];
910 if f.seek(SeekFrom::Start(pc)).is_ok() && f.read_exact(&mut insn).is_ok() {
911 let w = u32::from_le_bytes(insn);
912 let m = format!(" insn@{pc:#x} = {w:#010x} bytes={insn:02x?}\n");
913 drop(io::stderr().write_all(m.as_bytes()));
914 }
915 }
916 }
917 if let Ok(maps) = std::fs::read_to_string("/proc/self/maps") {
919 for &va in &[pc, addr, lr, x0, x1, x2, x16, x19, x25] {
920 if va == 0 {
921 continue;
922 }
923 for line in maps.lines() {
924 let Some((range, rest)) = line.split_once(' ') else {
925 continue;
926 };
927 let Some((a, b)) = range.split_once('-') else {
928 continue;
929 };
930 let (Ok(lo), Ok(hi)) = (u64::from_str_radix(a, 16), u64::from_str_radix(b, 16))
931 else {
932 continue;
933 };
934 if va >= lo && va < hi {
935 let off = va.saturating_sub(lo);
936 let m = format!(" map {va:#x} = {lo:#x}+{off:#x} {rest}\n");
937 drop(io::stderr().write_all(m.as_bytes()));
938 break;
939 }
940 }
941 }
942 }
943 drop(io::stderr().flush());
944 if TRACE_ON_EXIT.load(Ordering::SeqCst) {
945 dump_events_before_exit();
946 }
947 unsafe { libc::_exit(128_i32.saturating_add(signo)) };
949}
950
951#[must_use]
953pub fn take_trace_events() -> Vec<TrapEvent> {
954 EVENTS
955 .lock()
956 .map(|mut g| std::mem::take(&mut *g))
957 .unwrap_or_default()
958}
959
960#[must_use]
962pub fn exit_requested() -> bool {
963 EXIT_REQUESTED.load(Ordering::SeqCst)
964}
965
966#[must_use]
968pub fn take_exit_code() -> Option<i32> {
969 if EXIT_REQUESTED.load(Ordering::SeqCst) {
970 Some(EXIT_CODE.load(Ordering::SeqCst))
971 } else {
972 None
973 }
974}
975
976pub fn reset_trap_state() {
978 EXIT_REQUESTED.store(false, Ordering::SeqCst);
979 EXIT_CODE.store(0, Ordering::SeqCst);
980 EVENT_COUNT.store(0, Ordering::SeqCst);
981 if let Ok(mut guard) = EVENTS.lock() {
982 guard.clear();
983 }
984}
985
986pub fn set_trace_on_exit(json: bool) {
988 TRACE_ON_EXIT.store(true, Ordering::SeqCst);
989 TRACE_JSON.store(json, Ordering::SeqCst);
990}
991
992pub fn clear_trace_on_exit() {
994 TRACE_ON_EXIT.store(false, Ordering::SeqCst);
995 TRACE_JSON.store(false, Ordering::SeqCst);
996}
997
998pub fn set_expect_code(code: i32) {
1000 EXPECT_CODE.store(code, Ordering::SeqCst);
1001}
1002
1003pub fn clear_expect_code() {
1005 EXPECT_CODE.store(i32::MIN, Ordering::SeqCst);
1006}
1007
1008pub fn finish_with_exit_code(code: i32) -> ! {
1013 EXIT_CODE.store(code, Ordering::SeqCst);
1014 EXIT_REQUESTED.store(true, Ordering::SeqCst);
1015 if TRACE_ON_EXIT.load(Ordering::SeqCst) {
1016 dump_events_before_exit();
1017 }
1018 crate::syscall::dump_futex_stats_if_enabled();
1020 crate::syscall::dump_boundary_stats_if_enabled();
1021 let expect = EXPECT_CODE.load(Ordering::SeqCst);
1022 let status = if expect == i32::MIN || expect == code {
1023 code
1024 } else {
1025 let msg = format!("error: guest exit code {code} != expected {expect}\n");
1026 drop(io::stderr().write_all(msg.as_bytes()));
1027 1
1028 };
1029 unsafe { libc::_exit(status) };
1031}
1032
1033#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
1034unsafe extern "C" fn trap_sigaction(
1035 signo: libc::c_int,
1036 _info: *mut libc::siginfo_t,
1037 ctx: *mut libc::c_void,
1038) {
1039 if signo != libc::SIGTRAP || ctx.is_null() {
1040 return;
1041 }
1042
1043 crate::tls::enter_host_tls();
1045
1046 let uctx = unsafe { &mut *ctx.cast::<libc::ucontext_t>() };
1048 let m = &mut uctx.uc_mcontext;
1049 let pc = m.pc;
1050 let x0 = m.regs[0];
1052 let x1 = m.regs[1];
1053 let x2 = m.regs[2];
1054 let x3 = m.regs[3];
1055 let x4 = m.regs[4];
1056 let x5 = m.regs[5];
1057 let x6 = m.regs[6];
1058 let x16 = m.regs[16];
1059
1060 let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
1061 let result = syscall::dispatch(SyscallArgs {
1062 pc,
1063 number: sys_no,
1064 x0,
1065 x1,
1066 x2,
1067 x3,
1068 x4,
1069 x5,
1070 x6,
1071 });
1072
1073 if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
1075 maybe_push_event(
1076 pc,
1077 sys_no,
1078 result.name,
1079 x0,
1080 x1,
1081 x2,
1082 result.retval,
1083 result.error,
1084 );
1085 }
1086
1087 if let Some(ret) = result.retval {
1088 m.regs[0] = ret;
1089 }
1090 if result.error {
1092 m.pstate |= PSTATE_C;
1093 } else if result.retval.is_some() {
1094 m.pstate &= !PSTATE_C;
1095 }
1096
1097 match result.outcome {
1098 TrapOutcome::Exit { code } => {
1099 finish_with_exit_code(code);
1100 }
1101 TrapOutcome::Continue => {
1102 m.pc = pc.wrapping_add(4);
1103 crate::tls::leave_host_tls();
1105 }
1106 TrapOutcome::ThreadExit => {
1107 if !crate::thread::redirect_ucontext_to_host_exit(m) {
1112 finish_with_exit_code(0);
1113 }
1114 }
1115 }
1116}
1117
1118fn dump_events_before_exit() {
1119 use std::fmt::Write as _;
1120
1121 let json = TRACE_JSON.load(Ordering::SeqCst);
1122 let events = EVENTS.lock().map(|g| g.clone()).unwrap_or_default();
1123 if json {
1124 let mut out = String::from("{\"events\":[");
1125 for (i, e) in events.iter().enumerate() {
1126 if i > 0 {
1127 out.push(',');
1128 }
1129 let sys = e
1130 .syscall
1131 .map_or_else(|| "null".to_owned(), |s| s.to_string());
1132 let ret = e
1133 .retval
1134 .map_or_else(|| "null".to_owned(), |v| v.to_string());
1135 let _ = write!(
1136 out,
1137 "{{\"pc\":{},\"syscall\":{},\"name\":\"{}\",\"arg0\":{},\"arg1\":{},\"arg2\":{},\"retval\":{},\"error\":{}}}",
1138 e.pc, sys, e.name, e.arg0, e.arg1, e.arg2, ret, e.error
1139 );
1140 }
1141 out.push_str("]}\n");
1142 drop(io::stdout().write_all(out.as_bytes()));
1143 drop(io::stdout().flush());
1144 } else {
1145 let header = format!("trace events: {}\n", events.len());
1146 drop(io::stdout().write_all(header.as_bytes()));
1147 for (i, e) in events.iter().enumerate() {
1148 let ret = e
1149 .retval
1150 .map_or_else(|| "-".to_owned(), |v| format!("{v:#x}"));
1151 let err = if e.error { " err" } else { "" };
1152 let line = format!(
1153 " [{i}] pc={:#x} {} sys={:?} args=({:#x}, {:#x}, {:#x}) ret={ret}{err}\n",
1154 e.pc, e.name, e.syscall, e.arg0, e.arg1, e.arg2
1155 );
1156 drop(io::stdout().write_all(line.as_bytes()));
1157 }
1158 drop(io::stdout().flush());
1159 }
1160}
1161
1162#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
1163#[allow(clippy::too_many_arguments)]
1164fn maybe_push_event(
1165 pc: u64,
1166 sys_no: u32,
1167 name: &'static str,
1168 arg0: u64,
1169 arg1: u64,
1170 arg2: u64,
1171 retval: Option<u64>,
1172 error: bool,
1173) {
1174 let max = MAX_EVENTS.load(Ordering::Relaxed);
1175 if max == 0 {
1177 return;
1178 }
1179 let n = EVENT_COUNT.fetch_add(1, Ordering::Relaxed);
1180 if n >= max {
1181 return;
1182 }
1183 if let Ok(mut guard) = EVENTS.lock()
1184 && u64::try_from(guard.len()).unwrap_or(u64::MAX) < max
1185 {
1186 guard.push(TrapEvent {
1187 pc,
1188 syscall: Some(sys_no),
1189 name: name.to_owned(),
1190 arg0,
1191 arg1,
1192 arg2,
1193 retval,
1194 error,
1195 });
1196 }
1197}
1198
1199#[cfg(test)]
1200mod tests {
1201 use super::*;
1202
1203 #[test]
1204 fn detects_svc_encoding() {
1205 assert!(is_svc(0xD400_0001));
1206 let svc80 = 0xD400_0001 | (0x80_u32 << 5);
1207 assert!(is_svc(svc80));
1208 assert!(!is_svc(0xD503_201F));
1209 assert!(!is_svc(brk_encoding(BRK_TRAP_IMM)));
1210 }
1211
1212 #[test]
1213 fn brk_encoding_roundtrip_imm() {
1214 let enc = brk_encoding(BRK_TRAP_IMM);
1215 let imm = (enc >> 5) & 0xFFFF;
1216 assert_eq!(imm, BRK_TRAP_IMM);
1217 }
1218}