1#![allow(unsafe_code)]
8
9use std::io::{self, Write};
10use std::sync::Mutex;
11use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering};
12
13use thiserror::Error;
14
15use crate::mem::{
16 MapError, MappedRegion, VM_PROT_EXECUTE, VM_PROT_READ, VM_PROT_WRITE, mprotect_darwin,
17 mprotect_rw,
18};
19use crate::syscall;
20#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
21use crate::syscall::SyscallArgs;
22
23pub const BRK_TRAP_IMM: u32 = 0x0F00;
25
26const SVC_MASK: u32 = 0xFFE0_001F;
28const SVC_BASE: u32 = 0xD400_0001;
29
30const fn brk_encoding(imm16: u32) -> u32 {
32 0xD420_0000 | ((imm16 & 0xFFFF) << 5)
33}
34
35pub const PSTATE_C: u64 = 1 << 29;
39
40#[derive(Debug, Clone, PartialEq, Eq)]
42pub struct TrapEvent {
43 pub pc: u64,
45 pub syscall: Option<u32>,
47 pub name: String,
49 pub arg0: u64,
51 pub arg1: u64,
53 pub arg2: u64,
55 pub retval: Option<u64>,
57 pub error: bool,
59}
60
61#[derive(Debug, Clone, Copy, PartialEq, Eq)]
63pub enum TrapOutcome {
64 Exit {
66 code: i32,
68 },
69 Continue,
71 ThreadExit,
73}
74
75#[derive(Debug, Clone)]
77pub struct TrapConfig {
78 pub max_events: usize,
80 pub max_syscalls: usize,
82}
83
84impl Default for TrapConfig {
85 fn default() -> Self {
86 Self {
87 max_events: 256,
88 max_syscalls: 256,
89 }
90 }
91}
92
93#[derive(Debug, Error)]
95pub enum TrapError {
96 #[error("failed to install trap signal handler: {0}")]
98 SignalSetup(#[source] io::Error),
99
100 #[error(transparent)]
102 Map(#[from] MapError),
103
104 #[error("trap backend requires Linux aarch64 for live execution")]
106 UnsupportedArch,
107}
108
109#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
110static HANDLER_INSTALLED: AtomicBool = AtomicBool::new(false);
111static EXIT_CODE: AtomicI32 = AtomicI32::new(0);
112static EXIT_REQUESTED: AtomicBool = AtomicBool::new(false);
113static EVENT_COUNT: AtomicU64 = AtomicU64::new(0);
114static MAX_EVENTS: AtomicU64 = AtomicU64::new(256);
115static TRACE_ON_EXIT: AtomicBool = AtomicBool::new(false);
116static TRACE_JSON: AtomicBool = AtomicBool::new(false);
117static EXPECT_CODE: AtomicI32 = AtomicI32::new(i32::MIN);
119static EVENTS: Mutex<Vec<TrapEvent>> = Mutex::new(Vec::new());
120
121pub fn patch_svc_to_brk(regions: &mut [MappedRegion]) -> Result<usize, TrapError> {
128 let mut patched = 0_usize;
129 for region in regions.iter_mut() {
130 if region.prot & VM_PROT_EXECUTE == 0 {
131 continue;
132 }
133 let file_len = usize::try_from(region.file_bytes)
135 .unwrap_or(0)
136 .min(region.host_len());
137 if file_len < 4 {
138 continue;
139 }
140
141 let ranges = resolve_svc_scan_ranges(region, file_len);
143 if ranges.is_empty() {
144 continue;
145 }
146
147 let hits = {
149 let bytes = region.host_bytes();
150 let mut hits = Vec::new();
151 for &(start, len) in &ranges {
152 let end = start.saturating_add(len).min(file_len);
153 if end.saturating_sub(start) < 4 {
154 continue;
155 }
156 let Some(scan) = bytes.get(start..end) else {
157 continue;
158 };
159 hits.extend(find_svc_offsets(scan).into_iter().map(|o| start.saturating_add(o)));
160 }
161 hits
162 };
163 if hits.is_empty() {
164 continue;
165 }
166
167 mprotect_rw(region)?;
169 let bytes = region.host_bytes_mut();
170 let brk = brk_encoding(BRK_TRAP_IMM).to_le_bytes();
171 for off in &hits {
172 let end = off.saturating_add(4);
173 if let Some(slot) = bytes.get_mut(*off..end) {
174 slot.copy_from_slice(&brk);
175 patched = patched.saturating_add(1);
176 }
177 }
178 let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
179 mprotect_darwin(region, restore)?;
180 region.prot = restore;
181 let base = region.host_bytes().as_ptr();
183 for off in hits {
184 let ptr = unsafe { base.add(off).cast_mut() };
186 clear_icache(ptr, 4);
187 }
188 }
189 Ok(patched)
190}
191
192fn resolve_svc_scan_ranges(region: &MappedRegion, file_len: usize) -> Vec<(usize, usize)> {
194 if region.svc_scan_ranges.is_empty() {
195 return vec![(0, file_len)];
196 }
197 let mut out = Vec::with_capacity(region.svc_scan_ranges.len());
198 for &(off, len) in ®ion.svc_scan_ranges {
199 let start = usize::try_from(off).unwrap_or(usize::MAX);
200 let len = usize::try_from(len).unwrap_or(0);
201 if start >= file_len || len == 0 {
202 continue;
203 }
204 let len = len.min(file_len.saturating_sub(start));
205 if len >= 4 {
206 out.push((start, len));
207 }
208 }
209 out
210}
211
212const fn is_svc(insn: u32) -> bool {
213 (insn & SVC_MASK) == SVC_BASE
214}
215
216#[allow(clippy::integer_division)] fn find_svc_offsets(scan: &[u8]) -> Vec<usize> {
222 const PARALLEL_MIN: usize = 8 << 20; if scan.len() >= PARALLEL_MIN {
224 return find_svc_offsets_parallel(scan);
225 }
226 find_svc_offsets_range(scan, 0)
227}
228
229#[allow(clippy::integer_division)]
231fn find_svc_offsets_parallel(scan: &[u8]) -> Vec<usize> {
232 let nthreads = std::thread::available_parallelism()
233 .map_or(1, std::num::NonZeroUsize::get)
234 .clamp(1, 8);
235 if nthreads <= 1 {
236 return find_svc_offsets_range(scan, 0);
237 }
238 let total = scan.len() & !3_usize;
239 if total < 4 {
240 return Vec::new();
241 }
242 let chunk = (total
243 .checked_div(nthreads)
244 .unwrap_or(total)
245 & !3_usize)
246 .max(4);
247 let mut hits = Vec::new();
248 std::thread::scope(|scope| {
249 let mut handles = Vec::with_capacity(nthreads);
250 let mut start = 0_usize;
251 while start < total {
252 let end = (start.saturating_add(chunk)).min(total);
253 let Some(part) = scan.get(start..end) else {
254 break;
255 };
256 let base_off = start;
257 handles.push(scope.spawn(move || find_svc_offsets_range(part, base_off)));
258 start = end;
259 }
260 for h in handles {
261 if let Ok(mut part) = h.join() {
262 hits.append(&mut part);
263 }
264 }
265 });
266 hits.sort_unstable();
267 hits
268}
269
270#[allow(clippy::integer_division)]
273fn find_svc_offsets_range(scan: &[u8], base_off: usize) -> Vec<usize> {
274 let mut hits = Vec::new();
275 let words = scan.len() / 4;
276 if words == 0 {
277 return hits;
278 }
279 let base = scan.as_ptr();
282 let mut i = 0_usize;
283 while i.saturating_add(4) <= words {
284 let w0 = unsafe { base.add(i.saturating_mul(4)).cast::<u32>().read_unaligned() };
286 let w1 = unsafe {
287 base.add(i.saturating_add(1).saturating_mul(4))
288 .cast::<u32>()
289 .read_unaligned()
290 };
291 let w2 = unsafe {
292 base.add(i.saturating_add(2).saturating_mul(4))
293 .cast::<u32>()
294 .read_unaligned()
295 };
296 let w3 = unsafe {
297 base.add(i.saturating_add(3).saturating_mul(4))
298 .cast::<u32>()
299 .read_unaligned()
300 };
301 if is_svc(u32::from_le(w0)) {
302 hits.push(base_off.saturating_add(i.saturating_mul(4)));
303 }
304 if is_svc(u32::from_le(w1)) {
305 hits.push(
306 base_off.saturating_add(i.saturating_add(1).saturating_mul(4)),
307 );
308 }
309 if is_svc(u32::from_le(w2)) {
310 hits.push(
311 base_off.saturating_add(i.saturating_add(2).saturating_mul(4)),
312 );
313 }
314 if is_svc(u32::from_le(w3)) {
315 hits.push(
316 base_off.saturating_add(i.saturating_add(3).saturating_mul(4)),
317 );
318 }
319 i = i.saturating_add(4);
320 }
321 while i < words {
322 let w = unsafe { base.add(i.saturating_mul(4)).cast::<u32>().read_unaligned() };
323 if is_svc(u32::from_le(w)) {
324 hits.push(base_off.saturating_add(i.saturating_mul(4)));
325 }
326 i = i.saturating_add(1);
327 }
328 hits
329}
330
331#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
333#[inline]
334fn align_up_to_page(len: usize, page: usize) -> usize {
335 if page <= 1 {
336 return len.max(1);
337 }
338 let mask = page.saturating_sub(1);
339 len.saturating_add(mask) & !mask
340}
341
342#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
344unsafe extern "C" {
345 fn __clear_cache(start: *mut libc::c_void, end: *mut libc::c_void);
346}
347
348#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
350fn clear_icache(ptr: *mut u8, len: usize) {
351 if ptr.is_null() || len == 0 {
352 return;
353 }
354 unsafe {
356 let start = ptr.cast::<libc::c_void>();
357 let end = ptr.wrapping_add(len).cast::<libc::c_void>();
358 __clear_cache(start, end);
359 }
360}
361
362#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
363fn clear_icache(_ptr: *mut u8, _len: usize) {}
364
365#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
366#[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
367fn trampoline_rust_entry_addr() -> u64 {
368 u64::try_from(kh_trampoline_dispatch as usize).unwrap_or(0)
369}
370
371#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
373#[repr(C)]
374pub struct TrampRet {
375 pub retval: u64,
377 pub error: u64,
379}
380
381#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
393#[unsafe(no_mangle)]
394pub unsafe extern "C" fn kh_trampoline_dispatch(
395 x0: u64,
396 x1: u64,
397 x2: u64,
398 x3: u64,
399 x4: u64,
400 x5: u64,
401 x6: u64,
402 x16: u64,
403) -> TrampRet {
404 let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
405 let result = syscall::dispatch(SyscallArgs {
406 pc: 0,
407 number: sys_no,
408 x0,
409 x1,
410 x2,
411 x3,
412 x4,
413 x5,
414 x6,
415 });
416 if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
417 maybe_push_event(
418 0,
419 sys_no,
420 result.name,
421 x0,
422 x1,
423 x2,
424 result.retval,
425 result.error,
426 );
427 }
428 match result.outcome {
429 TrapOutcome::Exit { code } => finish_with_exit_code(code),
430 TrapOutcome::ThreadExit => {
431 crate::thread::exit_current_guest_worker();
433 }
434 TrapOutcome::Continue => {}
435 }
436 TrampRet {
437 retval: result.retval.unwrap_or(0),
438 error: u64::from(result.error),
439 }
440}
441
442#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
450std::arch::global_asm!(
451 r#"
452 .text
453 .align 2
454 .global kh_hypercall_entry
455 .type kh_hypercall_entry, %function
456kh_hypercall_entry:
457 // Guest prolog: GPRs + full NEON *before* any bl (NEON must survive host TLS).
458 sub sp, sp, #640
459 stp x29, x30, [sp, #0]
460 stp x0, x1, [sp, #16]
461 stp x2, x3, [sp, #32]
462 stp x4, x5, [sp, #48]
463 stp x6, x7, [sp, #64]
464 mov x29, sp
465 stp q0, q1, [sp, #96]
466 stp q2, q3, [sp, #128]
467 stp q4, q5, [sp, #160]
468 stp q6, q7, [sp, #192]
469 stp q8, q9, [sp, #224]
470 stp q10, q11, [sp, #256]
471 stp q12, q13, [sp, #288]
472 stp q14, q15, [sp, #320]
473 stp q16, q17, [sp, #352]
474 stp q18, q19, [sp, #384]
475 stp q20, q21, [sp, #416]
476 stp q22, q23, [sp, #448]
477 stp q24, q25, [sp, #480]
478 stp q26, q27, [sp, #512]
479 stp q28, q29, [sp, #544]
480 stp q30, q31, [sp, #576]
481 mrs x9, fpcr
482 mrs x10, fpsr
483 // #608 exceeds signed LDP imm7*8 range (±512); use base+add.
484 add x11, sp, #608
485 stp x9, x10, [x11]
486
487 // A1/A2: enter returns alt_top in x0, guest_tpidr in x1 (HyperEnterRet).
488 // Park guest_va at prolog+80 (hole before NEON) so leave needs no gettid.
489 bl kh_tls_enter_host
490 mov x9, sp // guest prolog frame
491 str x1, [x9, #80] // guest_tpidr for leave
492 cbz x0, 1f
493 mov sp, x0 // host alt stack
494 ldr x30, [x9, #8] // reload LR → freestanding thin caller
495 // Host frame: [guest_frame, lr]
496 stp x9, x30, [sp, #-16]!
497 // Reload guest frame ptr / args (enter_host clobbered temps).
498 ldr x9, [sp]
499 ldp x0, x1, [x9, #16]
500 ldp x2, x3, [x9, #32]
501 ldp x4, x5, [x9, #48]
502 ldp x6, x7, [x9, #64]
503 mov x16, x7
504 bl kh_neon_tramp_entry
505 // Preserve TrampRet (x0/x1) across TLS restore.
506 stp x0, x1, [sp, #-16]!
507 // guest_va at host_frame.guest_frame + 80; host_frame at sp+16.
508 ldr x9, [sp, #16]
509 ldr x0, [x9, #80]
510 bl kh_tls_leave_host
511 ldp x0, x1, [sp], #16
512 ldp x9, x30, [sp], #16
513 mov sp, x9
514 // Restore guest NEON from prolog (pre-bl snapshot).
515 ldp q0, q1, [sp, #96]
516 ldp q2, q3, [sp, #128]
517 ldp q4, q5, [sp, #160]
518 ldp q6, q7, [sp, #192]
519 ldp q8, q9, [sp, #224]
520 ldp q10, q11, [sp, #256]
521 ldp q12, q13, [sp, #288]
522 ldp q14, q15, [sp, #320]
523 ldp q16, q17, [sp, #352]
524 ldp q18, q19, [sp, #384]
525 ldp q20, q21, [sp, #416]
526 ldp q22, q23, [sp, #448]
527 ldp q24, q25, [sp, #480]
528 ldp q26, q27, [sp, #512]
529 ldp q28, q29, [sp, #544]
530 ldp q30, q31, [sp, #576]
531 add x11, sp, #608
532 ldp x9, x10, [x11]
533 msr fpcr, x9
534 msr fpsr, x10
535 ldr x29, [sp, #0]
536 add sp, sp, #640
537 ret
5381:
539 // No alt stack: dispatch on the guest stack (ST / fallback only).
540 // Already on host TLS from the enter above; guest_va at [sp, #80].
541 ldp x0, x1, [sp, #16]
542 ldp x2, x3, [sp, #32]
543 ldp x4, x5, [sp, #48]
544 ldp x6, x7, [sp, #64]
545 mov x16, x7
546 // Preserve real LR across the call (still on guest prolog frame).
547 ldr x30, [sp, #8]
548 bl kh_neon_tramp_entry
549 stp x0, x1, [sp, #-16]!
550 ldr x0, [sp, #16+80]
551 bl kh_tls_leave_host
552 ldp x0, x1, [sp], #16
553 // Restore guest NEON from prolog.
554 ldp q0, q1, [sp, #96]
555 ldp q2, q3, [sp, #128]
556 ldp q4, q5, [sp, #160]
557 ldp q6, q7, [sp, #192]
558 ldp q8, q9, [sp, #224]
559 ldp q10, q11, [sp, #256]
560 ldp q12, q13, [sp, #288]
561 ldp q14, q15, [sp, #320]
562 ldp q16, q17, [sp, #352]
563 ldp q18, q19, [sp, #384]
564 ldp q20, q21, [sp, #416]
565 ldp q22, q23, [sp, #448]
566 ldp q24, q25, [sp, #480]
567 ldp q26, q27, [sp, #512]
568 ldp q28, q29, [sp, #544]
569 ldp q30, q31, [sp, #576]
570 add x11, sp, #608
571 ldp x9, x10, [x11]
572 msr fpcr, x9
573 msr fpsr, x10
574 ldp x29, x30, [sp, #0]
575 add sp, sp, #640
576 ret
577 .size kh_hypercall_entry, .-kh_hypercall_entry
578 "#
579);
580
581#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
585std::arch::global_asm!(
586 r#"
587 .text
588 .align 2
589 .global kh_neon_tramp_entry
590 .type kh_neon_tramp_entry, %function
591kh_neon_tramp_entry:
592 adrp x17, KH_NEON_TRAMP_VA
593 ldr x17, [x17, :lo12:KH_NEON_TRAMP_VA]
594 cbz x17, 1f
595 br x17
5961:
597 // No-tramp fallback: number in x16 → AAPCS 8th arg x7; tail call.
598 mov x7, x16
599 b kh_trampoline_dispatch
600 .size kh_neon_tramp_entry, .-kh_neon_tramp_entry
601
602 .data
603 .align 3
604 .global KH_NEON_TRAMP_VA
605KH_NEON_TRAMP_VA:
606 .quad 0
607 "#
608);
609
610#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
611unsafe extern "C" {
612 fn kh_hypercall_entry();
613 static mut KH_NEON_TRAMP_VA: u64;
614}
615
616#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
618#[must_use]
619pub fn hypercall_entry_addr() -> u64 {
620 ensure_neon_tramp();
621 #[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
622 {
623 u64::try_from(kh_hypercall_entry as usize).unwrap_or(0)
624 }
625}
626
627#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
629fn ensure_neon_tramp() {
630 static ONCE: AtomicU64 = AtomicU64::new(0);
631 static LOCK: Mutex<()> = Mutex::new(());
632
633 if ONCE.load(Ordering::Acquire) != 0 {
634 return;
635 }
636 let _g = LOCK
637 .lock()
638 .unwrap_or_else(std::sync::PoisonError::into_inner);
639 if ONCE.load(Ordering::Acquire) != 0 {
640 return;
641 }
642
643 let code = TRAMP_BYTES;
644 let page = crate::host::page_size().unwrap_or(4096);
645 let map_len = align_up_to_page(code.len(), page).max(page);
646 let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
647 let prot_rw = libc::PROT_READ | libc::PROT_WRITE;
648 let Some(ptr) = crate::host::mmap(None, map_len, prot_rw, flags, -1, 0) else {
649 tracing::error!("NEON tramp mmap failed");
650 return;
651 };
652 let slice = unsafe { std::slice::from_raw_parts_mut(ptr, map_len) };
654 slice.fill(0);
655 if let Some(dst) = slice.get_mut(..code.len()) {
656 dst.copy_from_slice(code);
657 } else {
658 let _ = crate::host::munmap(ptr, map_len);
659 return;
660 }
661 if let Some(slot) = slice.get_mut(TRAMP_RELOC_OFF..TRAMP_RELOC_OFF.saturating_add(8)) {
662 slot.copy_from_slice(&trampoline_rust_entry_addr().to_le_bytes());
663 } else {
664 let _ = crate::host::munmap(ptr, map_len);
665 return;
666 }
667 let prot_rx = libc::PROT_READ | libc::PROT_EXEC;
668 if !crate::host::mprotect(ptr, map_len, prot_rx) {
669 tracing::error!("NEON tramp mprotect RX failed");
670 let _ = crate::host::munmap(ptr, map_len);
671 return;
672 }
673 clear_icache(ptr, map_len);
674 let va = crate::host::ptr_addr_u64(ptr);
675 unsafe {
677 KH_NEON_TRAMP_VA = va;
678 }
679 ONCE.store(va, Ordering::Release);
680 tracing::info!(
681 va = format_args!("{va:#x}"),
682 "mapped NEON tramp for hypercall"
683 );
684}
685
686#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
687#[must_use]
688pub fn hypercall_entry_addr() -> u64 {
689 0
690}
691
692#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
699const TRAMP_RELOC_OFF: usize = 0x100;
700
701#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
707#[rustfmt::skip]
708const TRAMP_BYTES: &[u8] = &[
709 0x5f, 0x24, 0x03, 0xd5, 0xff, 0x03, 0x0a, 0xd1, 0xe8, 0x27, 0x00, 0xa9, 0xea, 0x2f, 0x01, 0xa9,
710 0xec, 0x37, 0x02, 0xa9, 0xee, 0x3f, 0x03, 0xa9, 0xf1, 0x4b, 0x04, 0xa9, 0xfe, 0x2b, 0x00, 0xf9,
711 0x08, 0x44, 0x3b, 0xd5, 0x29, 0x44, 0x3b, 0xd5, 0xe8, 0x27, 0x06, 0xa9, 0xe0, 0x87, 0x03, 0xad,
712 0xe2, 0x8f, 0x04, 0xad, 0xe4, 0x97, 0x05, 0xad, 0xe6, 0x9f, 0x06, 0xad, 0xe8, 0xa7, 0x07, 0xad,
713 0xea, 0xaf, 0x08, 0xad, 0xec, 0xb7, 0x09, 0xad, 0xee, 0xbf, 0x0a, 0xad, 0xf0, 0xc7, 0x0b, 0xad,
714 0xf2, 0xcf, 0x0c, 0xad, 0xf4, 0xd7, 0x0d, 0xad, 0xf6, 0xdf, 0x0e, 0xad, 0xf8, 0xe7, 0x0f, 0xad,
715 0xfa, 0xef, 0x10, 0xad, 0xfc, 0xf7, 0x11, 0xad, 0xfe, 0xff, 0x12, 0xad, 0xe7, 0x03, 0x10, 0xaa,
716 0x88, 0x04, 0x00, 0x58, 0x00, 0x01, 0x3f, 0xd6, 0x02, 0x42, 0x3b, 0xd5, 0x23, 0x00, 0x80, 0x52,
717 0x63, 0x88, 0x63, 0xd3, 0x42, 0x00, 0x23, 0x8a, 0x41, 0x00, 0x00, 0xb4, 0x42, 0x00, 0x03, 0xaa,
718 0x02, 0x42, 0x1b, 0xd5, 0xe8, 0x27, 0x46, 0xa9, 0x08, 0x44, 0x1b, 0xd5, 0x29, 0x44, 0x1b, 0xd5,
719 0xe0, 0x87, 0x43, 0xad, 0xe2, 0x8f, 0x44, 0xad, 0xe4, 0x97, 0x45, 0xad, 0xe6, 0x9f, 0x46, 0xad,
720 0xe8, 0xa7, 0x47, 0xad, 0xea, 0xaf, 0x48, 0xad, 0xec, 0xb7, 0x49, 0xad, 0xee, 0xbf, 0x4a, 0xad,
721 0xf0, 0xc7, 0x4b, 0xad, 0xf2, 0xcf, 0x4c, 0xad, 0xf4, 0xd7, 0x4d, 0xad, 0xf6, 0xdf, 0x4e, 0xad,
722 0xf8, 0xe7, 0x4f, 0xad, 0xfa, 0xef, 0x50, 0xad, 0xfc, 0xf7, 0x51, 0xad, 0xfe, 0xff, 0x52, 0xad,
723 0xfe, 0x2b, 0x40, 0xf9, 0xf1, 0x4b, 0x44, 0xa9, 0xee, 0x3f, 0x43, 0xa9, 0xec, 0x37, 0x42, 0xa9,
724 0xea, 0x2f, 0x41, 0xa9, 0xe8, 0x27, 0x40, 0xa9, 0xff, 0x03, 0x0a, 0x91, 0xc0, 0x03, 0x5f, 0xd6,
725 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ];
727pub fn install_trap_handlers(config: &TrapConfig) -> Result<(), TrapError> {
729 MAX_EVENTS.store(
730 u64::try_from(config.max_events).unwrap_or(256),
731 Ordering::SeqCst,
732 );
733 syscall::reset_syscall_state(config.max_syscalls);
734
735 #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
736 {
737 let _ = config;
738 Err(TrapError::UnsupportedArch)
739 }
740
741 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
742 {
743 if HANDLER_INSTALLED.swap(true, Ordering::SeqCst) {
744 reset_trap_state();
745 syscall::reset_syscall_state(config.max_syscalls);
746 return Ok(());
747 }
748
749 disable_core_dumps();
752
753 unsafe {
755 let mut sa: libc::sigaction = std::mem::zeroed();
756 sa.sa_flags = libc::SA_SIGINFO;
757 #[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
758 {
759 sa.sa_sigaction = trap_sigaction as *const () as usize;
760 }
761 libc::sigemptyset(std::ptr::addr_of_mut!(sa.sa_mask));
762 if libc::sigaction(libc::SIGTRAP, std::ptr::addr_of!(sa), std::ptr::null_mut()) != 0 {
763 HANDLER_INSTALLED.store(false, Ordering::SeqCst);
764 return Err(TrapError::SignalSetup(io::Error::last_os_error()));
765 }
766
767 let mut fault: libc::sigaction = std::mem::zeroed();
769 fault.sa_flags = libc::SA_SIGINFO;
770 #[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
771 {
772 fault.sa_sigaction = guest_fault_sigaction as *const () as usize;
773 }
774 libc::sigemptyset(std::ptr::addr_of_mut!(fault.sa_mask));
775 for sig in [libc::SIGSEGV, libc::SIGBUS] {
776 if libc::sigaction(sig, std::ptr::addr_of!(fault), std::ptr::null_mut()) != 0 {
777 HANDLER_INSTALLED.store(false, Ordering::SeqCst);
778 return Err(TrapError::SignalSetup(io::Error::last_os_error()));
779 }
780 }
781 }
782 reset_trap_state();
783 Ok(())
784 }
785}
786
787#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
790fn disable_core_dumps() {
791 unsafe {
793 let lim = libc::rlimit {
794 rlim_cur: 0,
795 rlim_max: 0,
796 };
797 let _ = libc::setrlimit(libc::RLIMIT_CORE, std::ptr::addr_of!(lim));
798 }
799}
800
801#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
803unsafe extern "C" fn guest_fault_sigaction(
804 signo: libc::c_int,
805 info: *mut libc::siginfo_t,
806 ctx: *mut libc::c_void,
807) {
808 crate::tls::enter_host_tls();
810 let name = match signo {
811 libc::SIGSEGV => "SIGSEGV",
812 libc::SIGBUS => "SIGBUS",
813 _ => "FAULT",
814 };
815 let mut pc = 0_u64;
816 let mut sp = 0_u64;
817 let mut x0 = 0_u64;
818 let mut x1 = 0_u64;
819 let mut x2 = 0_u64;
820 let mut x8 = 0_u64;
821 let mut x16 = 0_u64;
822 let mut x19 = 0_u64;
823 let mut x3 = 0_u64;
824 let mut x4 = 0_u64;
825 let mut x5 = 0_u64;
826 let mut x6 = 0_u64;
827 let mut x7 = 0_u64;
828 let mut x9 = 0_u64;
829 let mut x20 = 0_u64;
830 let mut x21 = 0_u64;
831 let mut x22 = 0_u64;
832 let mut x23 = 0_u64;
833 let mut x24 = 0_u64;
834 let mut x25 = 0_u64;
835 let mut x26 = 0_u64;
836 let mut x27 = 0_u64;
837 let mut x28 = 0_u64;
838 let mut x29 = 0_u64;
839 let mut lr = 0_u64;
840 if !ctx.is_null() {
841 let uctx = unsafe { &*ctx.cast::<libc::ucontext_t>() };
843 let m = &uctx.uc_mcontext;
844 pc = m.pc;
845 sp = m.sp;
846 x0 = m.regs[0];
847 x1 = m.regs[1];
848 x2 = m.regs[2];
849 x3 = m.regs[3];
850 x4 = m.regs[4];
851 x5 = m.regs[5];
852 x6 = m.regs[6];
853 x7 = m.regs[7];
854 x8 = m.regs[8];
855 x9 = m.regs[9];
856 x16 = m.regs[16];
857 x19 = m.regs[19];
858 x20 = m.regs[20];
859 x21 = m.regs[21];
860 x22 = m.regs[22];
861 x23 = m.regs[23];
862 x24 = m.regs[24];
863 x25 = m.regs[25];
864 x26 = m.regs[26];
865 x27 = m.regs[27];
866 x28 = m.regs[28];
867 x29 = m.regs[29];
868 lr = m.regs[30];
869 }
870 let addr = if info.is_null() {
871 0_u64
872 } else {
873 let p = unsafe { (*info).si_addr() };
875 u64::try_from(p.addr()).unwrap_or(0)
876 };
877 let msg = format!(
878 "error: guest {name} pc={pc:#x} addr={addr:#x} sp={sp:#x} lr={lr:#x} \
879 x0={x0:#x} x1={x1:#x} x2={x2:#x} x3={x3:#x} x4={x4:#x} x5={x5:#x} \
880 x6={x6:#x} x7={x7:#x} x8={x8:#x} x9={x9:#x} x16={x16:#x} \
881 x19={x19:#x} x20={x20:#x} x21={x21:#x} x22={x22:#x} x23={x23:#x} \
882 x24={x24:#x} x25={x25:#x} x26={x26:#x} x27={x27:#x} x28={x28:#x} x29={x29:#x}\n"
883 );
884 drop(io::stderr().write_all(msg.as_bytes()));
885 if pc != 0 {
887 if let Ok(mut f) = std::fs::File::open("/proc/self/mem") {
888 use std::io::{Read, Seek, SeekFrom};
889 let mut insn = [0_u8; 4];
890 if f.seek(SeekFrom::Start(pc)).is_ok() && f.read_exact(&mut insn).is_ok() {
891 let w = u32::from_le_bytes(insn);
892 let m = format!(" insn@{pc:#x} = {w:#010x} bytes={insn:02x?}\n");
893 drop(io::stderr().write_all(m.as_bytes()));
894 }
895 }
896 }
897 if let Ok(maps) = std::fs::read_to_string("/proc/self/maps") {
899 for &va in &[pc, addr, lr, x0, x1, x2, x16, x19, x25] {
900 if va == 0 {
901 continue;
902 }
903 for line in maps.lines() {
904 let Some((range, rest)) = line.split_once(' ') else {
905 continue;
906 };
907 let Some((a, b)) = range.split_once('-') else {
908 continue;
909 };
910 let (Ok(lo), Ok(hi)) = (u64::from_str_radix(a, 16), u64::from_str_radix(b, 16))
911 else {
912 continue;
913 };
914 if va >= lo && va < hi {
915 let off = va.saturating_sub(lo);
916 let m = format!(" map {va:#x} = {lo:#x}+{off:#x} {rest}\n");
917 drop(io::stderr().write_all(m.as_bytes()));
918 break;
919 }
920 }
921 }
922 }
923 drop(io::stderr().flush());
924 if TRACE_ON_EXIT.load(Ordering::SeqCst) {
925 dump_events_before_exit();
926 }
927 unsafe { libc::_exit(128_i32.saturating_add(signo)) };
929}
930
931#[must_use]
933pub fn take_trace_events() -> Vec<TrapEvent> {
934 EVENTS
935 .lock()
936 .map(|mut g| std::mem::take(&mut *g))
937 .unwrap_or_default()
938}
939
940#[must_use]
942pub fn exit_requested() -> bool {
943 EXIT_REQUESTED.load(Ordering::SeqCst)
944}
945
946#[must_use]
948pub fn take_exit_code() -> Option<i32> {
949 if EXIT_REQUESTED.load(Ordering::SeqCst) {
950 Some(EXIT_CODE.load(Ordering::SeqCst))
951 } else {
952 None
953 }
954}
955
956pub fn reset_trap_state() {
958 EXIT_REQUESTED.store(false, Ordering::SeqCst);
959 EXIT_CODE.store(0, Ordering::SeqCst);
960 EVENT_COUNT.store(0, Ordering::SeqCst);
961 if let Ok(mut guard) = EVENTS.lock() {
962 guard.clear();
963 }
964}
965
966pub fn set_trace_on_exit(json: bool) {
968 TRACE_ON_EXIT.store(true, Ordering::SeqCst);
969 TRACE_JSON.store(json, Ordering::SeqCst);
970}
971
972pub fn clear_trace_on_exit() {
974 TRACE_ON_EXIT.store(false, Ordering::SeqCst);
975 TRACE_JSON.store(false, Ordering::SeqCst);
976}
977
978pub fn set_expect_code(code: i32) {
980 EXPECT_CODE.store(code, Ordering::SeqCst);
981}
982
983pub fn clear_expect_code() {
985 EXPECT_CODE.store(i32::MIN, Ordering::SeqCst);
986}
987
988pub fn finish_with_exit_code(code: i32) -> ! {
993 EXIT_CODE.store(code, Ordering::SeqCst);
994 EXIT_REQUESTED.store(true, Ordering::SeqCst);
995 if TRACE_ON_EXIT.load(Ordering::SeqCst) {
996 dump_events_before_exit();
997 }
998 crate::syscall::dump_futex_stats_if_enabled();
1000 crate::syscall::dump_boundary_stats_if_enabled();
1001 let expect = EXPECT_CODE.load(Ordering::SeqCst);
1002 let status = if expect == i32::MIN || expect == code {
1003 code
1004 } else {
1005 let msg = format!("error: guest exit code {code} != expected {expect}\n");
1006 drop(io::stderr().write_all(msg.as_bytes()));
1007 1
1008 };
1009 unsafe { libc::_exit(status) };
1011}
1012
1013#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
1014unsafe extern "C" fn trap_sigaction(
1015 signo: libc::c_int,
1016 _info: *mut libc::siginfo_t,
1017 ctx: *mut libc::c_void,
1018) {
1019 if signo != libc::SIGTRAP || ctx.is_null() {
1020 return;
1021 }
1022
1023 crate::tls::enter_host_tls();
1025
1026 let uctx = unsafe { &mut *ctx.cast::<libc::ucontext_t>() };
1028 let m = &mut uctx.uc_mcontext;
1029 let pc = m.pc;
1030 let x0 = m.regs[0];
1032 let x1 = m.regs[1];
1033 let x2 = m.regs[2];
1034 let x3 = m.regs[3];
1035 let x4 = m.regs[4];
1036 let x5 = m.regs[5];
1037 let x6 = m.regs[6];
1038 let x16 = m.regs[16];
1039
1040 let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
1041 let result = syscall::dispatch(SyscallArgs {
1042 pc,
1043 number: sys_no,
1044 x0,
1045 x1,
1046 x2,
1047 x3,
1048 x4,
1049 x5,
1050 x6,
1051 });
1052
1053 if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
1055 maybe_push_event(
1056 pc,
1057 sys_no,
1058 result.name,
1059 x0,
1060 x1,
1061 x2,
1062 result.retval,
1063 result.error,
1064 );
1065 }
1066
1067 if let Some(ret) = result.retval {
1068 m.regs[0] = ret;
1069 }
1070 if result.error {
1072 m.pstate |= PSTATE_C;
1073 } else if result.retval.is_some() {
1074 m.pstate &= !PSTATE_C;
1075 }
1076
1077 match result.outcome {
1078 TrapOutcome::Exit { code } => {
1079 finish_with_exit_code(code);
1080 }
1081 TrapOutcome::Continue => {
1082 m.pc = pc.wrapping_add(4);
1083 crate::tls::leave_host_tls();
1085 }
1086 TrapOutcome::ThreadExit => {
1087 if !crate::thread::redirect_ucontext_to_host_exit(m) {
1092 finish_with_exit_code(0);
1093 }
1094 }
1095 }
1096}
1097
1098fn dump_events_before_exit() {
1099 use std::fmt::Write as _;
1100
1101 let json = TRACE_JSON.load(Ordering::SeqCst);
1102 let events = EVENTS.lock().map(|g| g.clone()).unwrap_or_default();
1103 if json {
1104 let mut out = String::from("{\"events\":[");
1105 for (i, e) in events.iter().enumerate() {
1106 if i > 0 {
1107 out.push(',');
1108 }
1109 let sys = e
1110 .syscall
1111 .map_or_else(|| "null".to_owned(), |s| s.to_string());
1112 let ret = e
1113 .retval
1114 .map_or_else(|| "null".to_owned(), |v| v.to_string());
1115 let _ = write!(
1116 out,
1117 "{{\"pc\":{},\"syscall\":{},\"name\":\"{}\",\"arg0\":{},\"arg1\":{},\"arg2\":{},\"retval\":{},\"error\":{}}}",
1118 e.pc, sys, e.name, e.arg0, e.arg1, e.arg2, ret, e.error
1119 );
1120 }
1121 out.push_str("]}\n");
1122 drop(io::stdout().write_all(out.as_bytes()));
1123 drop(io::stdout().flush());
1124 } else {
1125 let header = format!("trace events: {}\n", events.len());
1126 drop(io::stdout().write_all(header.as_bytes()));
1127 for (i, e) in events.iter().enumerate() {
1128 let ret = e
1129 .retval
1130 .map_or_else(|| "-".to_owned(), |v| format!("{v:#x}"));
1131 let err = if e.error { " err" } else { "" };
1132 let line = format!(
1133 " [{i}] pc={:#x} {} sys={:?} args=({:#x}, {:#x}, {:#x}) ret={ret}{err}\n",
1134 e.pc, e.name, e.syscall, e.arg0, e.arg1, e.arg2
1135 );
1136 drop(io::stdout().write_all(line.as_bytes()));
1137 }
1138 drop(io::stdout().flush());
1139 }
1140}
1141
1142#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
1143#[allow(clippy::too_many_arguments)]
1144fn maybe_push_event(
1145 pc: u64,
1146 sys_no: u32,
1147 name: &'static str,
1148 arg0: u64,
1149 arg1: u64,
1150 arg2: u64,
1151 retval: Option<u64>,
1152 error: bool,
1153) {
1154 let max = MAX_EVENTS.load(Ordering::Relaxed);
1155 if max == 0 {
1157 return;
1158 }
1159 let n = EVENT_COUNT.fetch_add(1, Ordering::Relaxed);
1160 if n >= max {
1161 return;
1162 }
1163 if let Ok(mut guard) = EVENTS.lock()
1164 && u64::try_from(guard.len()).unwrap_or(u64::MAX) < max
1165 {
1166 guard.push(TrapEvent {
1167 pc,
1168 syscall: Some(sys_no),
1169 name: name.to_owned(),
1170 arg0,
1171 arg1,
1172 arg2,
1173 retval,
1174 error,
1175 });
1176 }
1177}
1178
1179#[cfg(test)]
1180mod tests {
1181 use super::*;
1182
1183 #[test]
1184 fn detects_svc_encoding() {
1185 assert!(is_svc(0xD400_0001));
1186 let svc80 = 0xD400_0001 | (0x80_u32 << 5);
1187 assert!(is_svc(svc80));
1188 assert!(!is_svc(0xD503_201F));
1189 assert!(!is_svc(brk_encoding(BRK_TRAP_IMM)));
1190 }
1191
1192 #[test]
1193 fn brk_encoding_roundtrip_imm() {
1194 let enc = brk_encoding(BRK_TRAP_IMM);
1195 let imm = (enc >> 5) & 0xFFFF;
1196 assert_eq!(imm, BRK_TRAP_IMM);
1197 }
1198}