1#![allow(unsafe_code)]
14
15use std::io::{self, Write};
16use std::sync::Mutex;
17use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering};
18
19use thiserror::Error;
20
21use crate::mem::{
22 MapError, MappedRegion, VM_PROT_EXECUTE, VM_PROT_READ, VM_PROT_WRITE, mprotect_darwin,
23 mprotect_rw,
24};
25use crate::syscall;
26#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
27use crate::syscall::SyscallArgs;
28
29pub const BRK_TRAP_IMM: u32 = 0x0F00;
31
32const SVC_MASK: u32 = 0xFFE0_001F;
34const SVC_BASE: u32 = 0xD400_0001;
35
36const fn brk_encoding(imm16: u32) -> u32 {
38 0xD420_0000 | ((imm16 & 0xFFFF) << 5)
39}
40
41#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
43#[allow(
44 clippy::as_conversions,
45 clippy::cast_possible_truncation,
46 clippy::cast_sign_loss,
47 clippy::manual_range_contains
48)]
49fn branch_imm26(from: u64, to: u64) -> Option<u32> {
50 let delta = i64::try_from(to).ok()?.wrapping_sub(i64::try_from(from).ok()?);
51 if delta & 3 != 0 {
52 return None;
53 }
54 let imm = delta >> 2;
55 if imm < -(1_i64 << 25) || imm >= (1_i64 << 25) {
57 return None;
58 }
59 Some((imm as u32) & 0x03FF_FFFF)
61}
62
63#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
65fn bl_encoding(from: u64, to: u64) -> Option<u32> {
66 Some(0x9400_0000 | branch_imm26(from, to)?)
67}
68
69#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
71fn b_encoding(from: u64, to: u64) -> Option<u32> {
72 Some(0x1400_0000 | branch_imm26(from, to)?)
73}
74
75pub const PSTATE_C: u64 = 1 << 29;
79
80#[derive(Debug, Clone, PartialEq, Eq)]
82pub struct TrapEvent {
83 pub pc: u64,
85 pub syscall: Option<u32>,
87 pub name: String,
89 pub arg0: u64,
91 pub arg1: u64,
93 pub arg2: u64,
95 pub retval: Option<u64>,
97 pub error: bool,
99}
100
101#[derive(Debug, Clone, Copy, PartialEq, Eq)]
103pub enum TrapOutcome {
104 Exit {
106 code: i32,
108 },
109 Continue,
111 ThreadExit,
113}
114
115#[derive(Debug, Clone)]
117pub struct TrapConfig {
118 pub max_events: usize,
120 pub max_syscalls: usize,
122}
123
124impl Default for TrapConfig {
125 fn default() -> Self {
126 Self {
127 max_events: 256,
128 max_syscalls: 256,
129 }
130 }
131}
132
133#[derive(Debug, Error)]
135pub enum TrapError {
136 #[error("failed to install trap signal handler: {0}")]
138 SignalSetup(#[source] io::Error),
139
140 #[error(transparent)]
142 Map(#[from] MapError),
143
144 #[error("trap backend requires Linux aarch64 for live execution")]
146 UnsupportedArch,
147}
148
149#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
150static HANDLER_INSTALLED: AtomicBool = AtomicBool::new(false);
151static EXIT_CODE: AtomicI32 = AtomicI32::new(0);
152static EXIT_REQUESTED: AtomicBool = AtomicBool::new(false);
153static EVENT_COUNT: AtomicU64 = AtomicU64::new(0);
154static MAX_EVENTS: AtomicU64 = AtomicU64::new(256);
155static TRACE_ON_EXIT: AtomicBool = AtomicBool::new(false);
156static TRACE_JSON: AtomicBool = AtomicBool::new(false);
157static EXPECT_CODE: AtomicI32 = AtomicI32::new(i32::MIN);
159static EVENTS: Mutex<Vec<TrapEvent>> = Mutex::new(Vec::new());
160
161pub fn patch_svc_to_brk(regions: &mut [MappedRegion]) -> Result<usize, TrapError> {
166 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
167 {
168 patch_svc_linux(regions)
169 }
170 #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
171 {
172 patch_svc_brk_only(regions)
173 }
174}
175
176#[cfg_attr(
177 all(target_arch = "aarch64", target_os = "linux"),
178 allow(dead_code)
179)]
180fn patch_svc_brk_only(regions: &mut [MappedRegion]) -> Result<usize, TrapError> {
181 let mut patched = 0_usize;
182 for region in regions.iter_mut() {
183 if region.prot & VM_PROT_EXECUTE == 0 {
184 continue;
185 }
186 mprotect_rw(region)?;
187 let guest_base = region.guest_addr;
188 let bytes = region.host_bytes_mut();
189 let mut off = 0_usize;
190 while off.saturating_add(4) <= bytes.len() {
191 let end = off.saturating_add(4);
192 let Some(word) = bytes.get(off..end) else {
193 break;
194 };
195 let Ok(word_bytes) = <[u8; 4]>::try_from(word) else {
196 break;
197 };
198 let insn = u32::from_le_bytes(word_bytes);
199 if is_svc(insn) {
200 let brk = brk_encoding(BRK_TRAP_IMM).to_le_bytes();
201 if let Some(slot) = bytes.get_mut(off..end) {
202 slot.copy_from_slice(&brk);
203 patched = patched.saturating_add(1);
204 }
205 }
206 let _ = guest_base;
207 off = end;
208 }
209 let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
210 mprotect_darwin(region, restore)?;
211 region.prot = restore;
212 }
213 Ok(patched)
214}
215
216const fn is_svc(insn: u32) -> bool {
217 (insn & SVC_MASK) == SVC_BASE
218}
219
220#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
233fn patch_svc_linux(regions: &mut [MappedRegion]) -> Result<usize, TrapError> {
234 if !trampoline_enabled() {
235 return patch_svc_brk_only(regions);
236 }
237 let mut patched = 0_usize;
238 for region in regions.iter_mut() {
239 if region.prot & VM_PROT_EXECUTE == 0 {
240 continue;
241 }
242 mprotect_rw(region)?;
243 let guest_base = region.guest_addr;
244 let mut sites = Vec::new();
246 {
247 let bytes = region.host_bytes();
248 let mut off = 0_usize;
249 while off.saturating_add(4) <= bytes.len() {
250 let end = off.saturating_add(4);
251 if let Some(word) = bytes.get(off..end)
252 && let Ok(word_bytes) = <[u8; 4]>::try_from(word)
253 {
254 let insn = u32::from_le_bytes(word_bytes);
255 if is_svc(insn) {
256 sites.push(off);
257 }
258 }
259 off = end;
260 }
261 }
262 if sites.is_empty() {
263 let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
264 mprotect_darwin(region, restore)?;
265 region.prot = restore;
266 continue;
267 }
268
269 patched = patched.saturating_add(patch_region_with_veneers(region, guest_base, &sites));
270 }
271 Ok(patched)
272}
273
274#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
276const VENEER_BYTES: usize = 16;
277
278#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
280const VENEER_PROBE_RADIUS: u64 = 128;
281
282#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
284#[inline]
285fn align_up_to_page(len: usize, page: usize) -> usize {
286 if page <= 1 {
287 return len.max(1);
288 }
289 let mask = page.saturating_sub(1);
290 len.saturating_add(mask) & !mask
291}
292
293#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
295unsafe extern "C" {
296 fn __clear_cache(start: *mut libc::c_void, end: *mut libc::c_void);
297}
298
299#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
301fn trampoline_enabled() -> bool {
302 match std::env::var_os("KAKEHASHI_TRAMPOLINE") {
303 None => false,
304 Some(v) => {
305 v == "1"
306 || v.eq_ignore_ascii_case("true")
307 || v.eq_ignore_ascii_case("yes")
308 || v.eq_ignore_ascii_case("on")
309 }
310 }
311}
312
313#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
315fn clear_icache(ptr: *mut u8, len: usize) {
316 if ptr.is_null() || len == 0 {
317 return;
318 }
319 unsafe {
321 let start = ptr.cast::<libc::c_void>();
322 let end = ptr.wrapping_add(len).cast::<libc::c_void>();
323 __clear_cache(start, end);
324 }
325}
326
327pub fn clear_trampoline_cache() {
329 }
331
332#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
336fn patch_region_with_veneers(
337 region: &mut MappedRegion,
338 guest_base: u64,
339 sites: &[usize],
340) -> usize {
341 let n_sites = sites.len();
342 let Some(hub) = install_veneer_hub(guest_base, region.host_len(), n_sites) else {
343 return patch_sites_brk(region, sites);
344 };
345
346 let mut patched = 0_usize;
347 let slice = unsafe { std::slice::from_raw_parts_mut(hub.ptr, hub.map_len) };
349
350 for (i, &off) in sites.iter().enumerate() {
351 let site_va = guest_base.saturating_add(u64::try_from(off).unwrap_or(0));
352 let site_ret = site_va.saturating_add(4);
353 let veneer_off = hub.veneer0.saturating_add(i.saturating_mul(VENEER_BYTES));
354 let veneer_va = hub.base_va.saturating_add(u64::try_from(veneer_off).unwrap_or(0));
355 let end = off.saturating_add(4);
356
357 let ok = write_veneer(slice, veneer_off, veneer_va, hub.shared_va, site_ret)
358 .and_then(|()| b_encoding(site_va, veneer_va))
359 .and_then(|enc| {
360 let bytes = region.host_bytes_mut();
361 let slot = bytes.get_mut(off..end)?;
362 slot.copy_from_slice(&enc.to_le_bytes());
363 Some(())
364 })
365 .is_some();
366
367 if !ok {
368 let bytes = region.host_bytes_mut();
369 let brk = brk_encoding(BRK_TRAP_IMM).to_le_bytes();
370 if let Some(slot) = bytes.get_mut(off..end) {
371 slot.copy_from_slice(&brk);
372 }
373 }
374 patched = patched.saturating_add(1);
375 }
376
377 let prot_rx = libc::PROT_READ | libc::PROT_EXEC;
379 if !crate::host::mprotect(hub.ptr, hub.map_len, prot_rx) {
380 let _ = crate::host::munmap(hub.ptr, hub.map_len);
381 return patch_sites_brk(region, sites);
382 }
383 clear_icache(hub.ptr, hub.map_len);
384 crate::mem::register_owned(
385 hub.base_va,
386 hub.ptr,
387 hub.map_len,
388 VM_PROT_READ | VM_PROT_EXECUTE,
389 );
390
391 let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
392 drop(mprotect_darwin(region, restore));
393 region.prot = restore;
394 let host = region.host_addr();
395 let len = region.host_len();
396 if let (Ok(base), true) = (usize::try_from(host), len > 0) {
397 let p = std::ptr::with_exposed_provenance_mut::<u8>(base);
398 clear_icache(p, len);
399 }
400 patched
401}
402
403#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
404fn patch_sites_brk(region: &mut MappedRegion, sites: &[usize]) -> usize {
405 let mut patched = 0_usize;
406 let bytes = region.host_bytes_mut();
407 for &off in sites {
408 let end = off.saturating_add(4);
409 let brk = brk_encoding(BRK_TRAP_IMM).to_le_bytes();
410 if let Some(slot) = bytes.get_mut(off..end) {
411 slot.copy_from_slice(&brk);
412 patched = patched.saturating_add(1);
413 }
414 }
415 let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
416 drop(mprotect_darwin(region, restore));
417 region.prot = restore;
418 patched
419}
420
421#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
423struct VeneerHub {
424 ptr: *mut u8,
425 map_len: usize,
426 base_va: u64,
427 shared_va: u64,
428 veneer0: usize,
429}
430
431#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
433fn install_veneer_hub(region_guest: u64, region_len: usize, n_sites: usize) -> Option<VeneerHub> {
434 let region_end = region_guest.saturating_add(u64::try_from(region_len).unwrap_or(0));
435 let site_lo = region_guest;
436 let site_hi = region_end.saturating_sub(4).max(region_guest);
437
438 let page = crate::host::page_size().unwrap_or(4096);
439 let page_u = u64::try_from(page).unwrap_or(4096);
440 let code = trampoline_template();
441 let veneer0 = code.len().saturating_add(15) & !15_usize;
442 let need = veneer0.saturating_add(n_sites.saturating_mul(VENEER_BYTES));
443 let map_len = align_up_to_page(need, page).max(page);
444
445 let flags_fixed = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | crate::host::fixed_map_flag();
446 let flags_free = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
447 let prot_rw = libc::PROT_READ | libc::PROT_WRITE;
448
449 let after0 = region_end.saturating_add(page_u.saturating_sub(1)) & !(page_u.saturating_sub(1));
450 let before0 = region_guest.saturating_sub(page_u) & !(page_u.saturating_sub(1));
451
452 for i in 0..VENEER_PROBE_RADIUS {
453 let mut candidates = [after0.saturating_add(i.saturating_mul(page_u)), 0_u64];
454 if before0 >= i.saturating_mul(page_u) {
455 candidates[1] = before0.saturating_sub(i.saturating_mul(page_u));
456 }
457 for &prefer in &candidates {
458 if prefer == 0 {
459 continue;
460 }
461 if b_encoding(site_lo, prefer).is_none() || b_encoding(site_hi, prefer).is_none() {
463 continue;
464 }
465 if let Some(ptr) =
466 crate::host::mmap(Some(prefer), map_len, prot_rw, flags_fixed, -1, 0)
467 {
468 return init_hub_slice(ptr, map_len, code, veneer0);
469 }
470 }
471 }
472
473 let ptr = crate::host::mmap(None, map_len, prot_rw, flags_free, -1, 0)?;
474 let va = crate::host::ptr_addr_u64(ptr);
475 if b_encoding(site_lo, va).is_none() || b_encoding(site_hi, va).is_none() {
476 let _ = crate::host::munmap(ptr, map_len);
477 return None;
478 }
479 init_hub_slice(ptr, map_len, code, veneer0)
480}
481
482#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
483fn init_hub_slice(ptr: *mut u8, map_len: usize, code: &[u8], veneer0: usize) -> Option<VeneerHub> {
484 let slice = unsafe { std::slice::from_raw_parts_mut(ptr, map_len) };
486 if code.len() > slice.len() || veneer0 > slice.len() {
487 let _ = crate::host::munmap(ptr, map_len);
488 return None;
489 }
490 slice.fill(0);
491 slice.get_mut(..code.len())?.copy_from_slice(code);
492 let reloc_off = trampoline_reloc_offset();
493 if let Some(slot) = slice.get_mut(reloc_off..reloc_off.saturating_add(8)) {
494 slot.copy_from_slice(&trampoline_rust_entry_addr().to_le_bytes());
495 } else {
496 let _ = crate::host::munmap(ptr, map_len);
497 return None;
498 }
499 let base_va = crate::host::ptr_addr_u64(ptr);
500 Some(VeneerHub {
501 ptr,
502 map_len,
503 base_va,
504 shared_va: base_va,
505 veneer0,
506 })
507}
508
509#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
511fn write_veneer(
512 slice: &mut [u8],
513 veneer_off: usize,
514 veneer_va: u64,
515 shared_va: u64,
516 site_ret: u64,
517) -> Option<()> {
518 let str_lr: u32 = 0xf81f_0ffe;
520 let ldr_lr: u32 = 0xf841_07fe;
522 let bl = bl_encoding(veneer_va.saturating_add(4), shared_va)?;
523 let b_back = b_encoding(veneer_va.saturating_add(12), site_ret)?;
524 let words = [str_lr, bl, ldr_lr, b_back];
525 let end = veneer_off.saturating_add(VENEER_BYTES);
526 let slot = slice.get_mut(veneer_off..end)?;
527 for (i, w) in words.iter().enumerate() {
528 let o = i.saturating_mul(4);
529 slot.get_mut(o..o.saturating_add(4))?.copy_from_slice(&w.to_le_bytes());
530 }
531 Some(())
532}
533
534#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
538fn trampoline_template() -> &'static [u8] {
539 TRAMP_BYTES
540}
541
542#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
543fn trampoline_reloc_offset() -> usize {
544 TRAMP_RELOC_OFF
545}
546
547#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
548#[allow(clippy::as_conversions, function_casts_as_integer)]
549fn trampoline_rust_entry_addr() -> u64 {
550 u64::try_from(kh_trampoline_dispatch as usize).unwrap_or(0)
551}
552
553#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
555#[repr(C)]
556pub struct TrampRet {
557 pub retval: u64,
559 pub error: u64,
561}
562
563#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
577#[unsafe(no_mangle)]
578pub unsafe extern "C" fn kh_trampoline_dispatch(
579 x0: u64,
580 x1: u64,
581 x2: u64,
582 x3: u64,
583 x4: u64,
584 x5: u64,
585 x6: u64,
586 x16: u64,
587) -> TrampRet {
588 let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
589 let result = syscall::dispatch(SyscallArgs {
590 pc: 0,
591 number: sys_no,
592 x0,
593 x1,
594 x2,
595 x3,
596 x4,
597 x5,
598 x6,
599 });
600 if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
601 maybe_push_event(0, sys_no, result.name, x0, x1, x2, result.retval, result.error);
602 }
603 match result.outcome {
604 TrapOutcome::Exit { code } => finish_with_exit_code(code),
605 TrapOutcome::ThreadExit => {
606 crate::thread::exit_current_guest_worker();
608 }
609 TrapOutcome::Continue => {}
610 }
611 TrampRet {
612 retval: result.retval.unwrap_or(0),
613 error: u64::from(result.error),
614 }
615}
616
617#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
622std::arch::global_asm!(
623 r#"
624 .text
625 .align 2
626 .global kh_hypercall_entry
627 .type kh_hypercall_entry, %function
628kh_hypercall_entry:
629 // Guest prolog: args + real LR (before any `bl` clobbers x30).
630 sub sp, sp, #96
631 stp x29, x30, [sp, #0]
632 stp x0, x1, [sp, #16]
633 stp x2, x3, [sp, #32]
634 stp x4, x5, [sp, #48]
635 stp x6, x7, [sp, #64]
636 mov x29, sp
637
638 bl kh_host_alt_sp
639 cbz x0, 1f
640 mov x9, sp // guest prolog frame
641 mov sp, x0 // host alt stack
642 ldr x30, [x9, #8] // reload LR → freestanding thin caller
643 stp x9, x30, [sp, #-16]!
644 ldp x0, x1, [x9, #16]
645 ldp x2, x3, [x9, #32]
646 ldp x4, x5, [x9, #48]
647 ldp x6, x7, [x9, #64]
648 mov x16, x7
649 bl kh_neon_tramp_entry
650 ldp x9, x30, [sp], #16
651 mov sp, x9
652 ldr x29, [sp, #0]
653 add sp, sp, #96
654 ret
6551:
656 // No alt stack: NEON tramp on the guest stack (ST / fallback).
657 ldp x0, x1, [sp, #16]
658 ldp x2, x3, [sp, #32]
659 ldp x4, x5, [sp, #48]
660 ldp x6, x7, [sp, #64]
661 ldp x29, x30, [sp, #0]
662 add sp, sp, #96
663 mov x16, x7
664 b kh_neon_tramp_entry
665 .size kh_hypercall_entry, .-kh_hypercall_entry
666 "#
667);
668
669#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
671std::arch::global_asm!(
672 r#"
673 .text
674 .align 2
675 .global kh_neon_tramp_entry
676 .type kh_neon_tramp_entry, %function
677kh_neon_tramp_entry:
678 adrp x17, KH_NEON_TRAMP_VA
679 ldr x17, [x17, :lo12:KH_NEON_TRAMP_VA]
680 cbz x17, 1f
681 br x17
6821:
683 mov x7, x16
684 b kh_trampoline_dispatch
685 .size kh_neon_tramp_entry, .-kh_neon_tramp_entry
686
687 .data
688 .align 3
689 .global KH_NEON_TRAMP_VA
690KH_NEON_TRAMP_VA:
691 .quad 0
692 "#
693);
694
695#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
696unsafe extern "C" {
697 fn kh_hypercall_entry();
698 static mut KH_NEON_TRAMP_VA: u64;
699}
700
701#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
703#[must_use]
704pub fn hypercall_entry_addr() -> u64 {
705 ensure_neon_tramp();
706 #[allow(clippy::as_conversions, function_casts_as_integer)]
707 {
708 u64::try_from(kh_hypercall_entry as usize).unwrap_or(0)
709 }
710}
711
712#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
714fn ensure_neon_tramp() {
715 static ONCE: AtomicU64 = AtomicU64::new(0);
716 static LOCK: Mutex<()> = Mutex::new(());
717
718 if ONCE.load(Ordering::Acquire) != 0 {
719 return;
720 }
721 let _g = LOCK
722 .lock()
723 .unwrap_or_else(std::sync::PoisonError::into_inner);
724 if ONCE.load(Ordering::Acquire) != 0 {
725 return;
726 }
727
728 let code = trampoline_template();
729 let page = crate::host::page_size().unwrap_or(4096);
730 let map_len = align_up_to_page(code.len(), page).max(page);
731 let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
732 let prot_rw = libc::PROT_READ | libc::PROT_WRITE;
733 let Some(ptr) = crate::host::mmap(None, map_len, prot_rw, flags, -1, 0) else {
734 tracing::error!("NEON tramp mmap failed");
735 return;
736 };
737 let slice = unsafe { std::slice::from_raw_parts_mut(ptr, map_len) };
739 slice.fill(0);
740 if let Some(dst) = slice.get_mut(..code.len()) {
741 dst.copy_from_slice(code);
742 } else {
743 let _ = crate::host::munmap(ptr, map_len);
744 return;
745 }
746 let reloc_off = trampoline_reloc_offset();
747 if let Some(slot) = slice.get_mut(reloc_off..reloc_off.saturating_add(8)) {
748 slot.copy_from_slice(&trampoline_rust_entry_addr().to_le_bytes());
749 } else {
750 let _ = crate::host::munmap(ptr, map_len);
751 return;
752 }
753 let prot_rx = libc::PROT_READ | libc::PROT_EXEC;
754 if !crate::host::mprotect(ptr, map_len, prot_rx) {
755 tracing::error!("NEON tramp mprotect RX failed");
756 let _ = crate::host::munmap(ptr, map_len);
757 return;
758 }
759 clear_icache(ptr, map_len);
760 let va = crate::host::ptr_addr_u64(ptr);
761 unsafe {
763 KH_NEON_TRAMP_VA = va;
764 }
765 ONCE.store(va, Ordering::Release);
766 tracing::info!(va = format_args!("{va:#x}"), "mapped NEON tramp for hypercall");
767}
768
769#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
770#[must_use]
771pub fn hypercall_entry_addr() -> u64 {
772 0
773}
774
775#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
782const TRAMP_RELOC_OFF: usize = 0x100;
783
784#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
790#[rustfmt::skip]
791const TRAMP_BYTES: &[u8] = &[
792 0x5f, 0x24, 0x03, 0xd5, 0xff, 0x03, 0x0a, 0xd1, 0xe8, 0x27, 0x00, 0xa9, 0xea, 0x2f, 0x01, 0xa9,
793 0xec, 0x37, 0x02, 0xa9, 0xee, 0x3f, 0x03, 0xa9, 0xf1, 0x4b, 0x04, 0xa9, 0xfe, 0x2b, 0x00, 0xf9,
794 0x08, 0x44, 0x3b, 0xd5, 0x29, 0x44, 0x3b, 0xd5, 0xe8, 0x27, 0x06, 0xa9, 0xe0, 0x87, 0x03, 0xad,
795 0xe2, 0x8f, 0x04, 0xad, 0xe4, 0x97, 0x05, 0xad, 0xe6, 0x9f, 0x06, 0xad, 0xe8, 0xa7, 0x07, 0xad,
796 0xea, 0xaf, 0x08, 0xad, 0xec, 0xb7, 0x09, 0xad, 0xee, 0xbf, 0x0a, 0xad, 0xf0, 0xc7, 0x0b, 0xad,
797 0xf2, 0xcf, 0x0c, 0xad, 0xf4, 0xd7, 0x0d, 0xad, 0xf6, 0xdf, 0x0e, 0xad, 0xf8, 0xe7, 0x0f, 0xad,
798 0xfa, 0xef, 0x10, 0xad, 0xfc, 0xf7, 0x11, 0xad, 0xfe, 0xff, 0x12, 0xad, 0xe7, 0x03, 0x10, 0xaa,
799 0x88, 0x04, 0x00, 0x58, 0x00, 0x01, 0x3f, 0xd6, 0x02, 0x42, 0x3b, 0xd5, 0x23, 0x00, 0x80, 0x52,
800 0x63, 0x88, 0x63, 0xd3, 0x42, 0x00, 0x23, 0x8a, 0x41, 0x00, 0x00, 0xb4, 0x42, 0x00, 0x03, 0xaa,
801 0x02, 0x42, 0x1b, 0xd5, 0xe8, 0x27, 0x46, 0xa9, 0x08, 0x44, 0x1b, 0xd5, 0x29, 0x44, 0x1b, 0xd5,
802 0xe0, 0x87, 0x43, 0xad, 0xe2, 0x8f, 0x44, 0xad, 0xe4, 0x97, 0x45, 0xad, 0xe6, 0x9f, 0x46, 0xad,
803 0xe8, 0xa7, 0x47, 0xad, 0xea, 0xaf, 0x48, 0xad, 0xec, 0xb7, 0x49, 0xad, 0xee, 0xbf, 0x4a, 0xad,
804 0xf0, 0xc7, 0x4b, 0xad, 0xf2, 0xcf, 0x4c, 0xad, 0xf4, 0xd7, 0x4d, 0xad, 0xf6, 0xdf, 0x4e, 0xad,
805 0xf8, 0xe7, 0x4f, 0xad, 0xfa, 0xef, 0x50, 0xad, 0xfc, 0xf7, 0x51, 0xad, 0xfe, 0xff, 0x52, 0xad,
806 0xfe, 0x2b, 0x40, 0xf9, 0xf1, 0x4b, 0x44, 0xa9, 0xee, 0x3f, 0x43, 0xa9, 0xec, 0x37, 0x42, 0xa9,
807 0xea, 0x2f, 0x41, 0xa9, 0xe8, 0x27, 0x40, 0xa9, 0xff, 0x03, 0x0a, 0x91, 0xc0, 0x03, 0x5f, 0xd6,
808 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ];
810pub fn install_trap_handlers(config: &TrapConfig) -> Result<(), TrapError> {
812 MAX_EVENTS.store(
813 u64::try_from(config.max_events).unwrap_or(256),
814 Ordering::SeqCst,
815 );
816 syscall::reset_syscall_state(config.max_syscalls);
817
818 #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
819 {
820 let _ = config;
821 Err(TrapError::UnsupportedArch)
822 }
823
824 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
825 {
826 if HANDLER_INSTALLED.swap(true, Ordering::SeqCst) {
827 reset_trap_state();
828 syscall::reset_syscall_state(config.max_syscalls);
829 return Ok(());
830 }
831
832 disable_core_dumps();
835
836 unsafe {
838 let mut sa: libc::sigaction = std::mem::zeroed();
839 sa.sa_flags = libc::SA_SIGINFO;
840 #[allow(clippy::as_conversions, function_casts_as_integer)]
841 {
842 sa.sa_sigaction = trap_sigaction as *const () as usize;
843 }
844 libc::sigemptyset(std::ptr::addr_of_mut!(sa.sa_mask));
845 if libc::sigaction(libc::SIGTRAP, std::ptr::addr_of!(sa), std::ptr::null_mut()) != 0 {
846 HANDLER_INSTALLED.store(false, Ordering::SeqCst);
847 return Err(TrapError::SignalSetup(io::Error::last_os_error()));
848 }
849
850 let mut fault: libc::sigaction = std::mem::zeroed();
852 fault.sa_flags = libc::SA_SIGINFO;
853 #[allow(clippy::as_conversions, function_casts_as_integer)]
854 {
855 fault.sa_sigaction = guest_fault_sigaction as *const () as usize;
856 }
857 libc::sigemptyset(std::ptr::addr_of_mut!(fault.sa_mask));
858 for sig in [libc::SIGSEGV, libc::SIGBUS] {
859 if libc::sigaction(sig, std::ptr::addr_of!(fault), std::ptr::null_mut()) != 0 {
860 HANDLER_INSTALLED.store(false, Ordering::SeqCst);
861 return Err(TrapError::SignalSetup(io::Error::last_os_error()));
862 }
863 }
864 }
865 reset_trap_state();
866 Ok(())
867 }
868}
869
870#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
873fn disable_core_dumps() {
874 unsafe {
876 let lim = libc::rlimit {
877 rlim_cur: 0,
878 rlim_max: 0,
879 };
880 let _ = libc::setrlimit(libc::RLIMIT_CORE, std::ptr::addr_of!(lim));
881 }
882}
883
884#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
886unsafe extern "C" fn guest_fault_sigaction(
887 signo: libc::c_int,
888 info: *mut libc::siginfo_t,
889 ctx: *mut libc::c_void,
890) {
891 let name = match signo {
892 libc::SIGSEGV => "SIGSEGV",
893 libc::SIGBUS => "SIGBUS",
894 _ => "FAULT",
895 };
896 let mut pc = 0_u64;
897 let mut sp = 0_u64;
898 let mut x0 = 0_u64;
899 let mut x1 = 0_u64;
900 let mut x8 = 0_u64;
901 let mut x16 = 0_u64;
902 let mut lr = 0_u64;
903 if !ctx.is_null() {
904 let uctx = unsafe { &*ctx.cast::<libc::ucontext_t>() };
906 let m = &uctx.uc_mcontext;
907 pc = m.pc;
908 sp = m.sp;
909 x0 = m.regs[0];
910 x1 = m.regs[1];
911 x8 = m.regs[8];
912 x16 = m.regs[16];
913 lr = m.regs[30];
914 }
915 let addr = if info.is_null() {
916 0_u64
917 } else {
918 let p = unsafe { (*info).si_addr() };
920 u64::try_from(p.addr()).unwrap_or(0)
921 };
922 let msg = format!(
923 "error: guest {name} pc={pc:#x} addr={addr:#x} sp={sp:#x} lr={lr:#x} \
924 x0={x0:#x} x1={x1:#x} x8={x8:#x} x16={x16:#x}\n"
925 );
926 drop(io::stderr().write_all(msg.as_bytes()));
927 drop(io::stderr().flush());
928 if TRACE_ON_EXIT.load(Ordering::SeqCst) {
929 dump_events_before_exit();
930 }
931 unsafe { libc::_exit(128_i32.saturating_add(signo)) };
933}
934
935#[must_use]
937pub fn take_trace_events() -> Vec<TrapEvent> {
938 EVENTS
939 .lock()
940 .map(|mut g| std::mem::take(&mut *g))
941 .unwrap_or_default()
942}
943
944#[must_use]
946pub fn exit_requested() -> bool {
947 EXIT_REQUESTED.load(Ordering::SeqCst)
948}
949
950#[must_use]
952pub fn take_exit_code() -> Option<i32> {
953 if EXIT_REQUESTED.load(Ordering::SeqCst) {
954 Some(EXIT_CODE.load(Ordering::SeqCst))
955 } else {
956 None
957 }
958}
959
960pub fn reset_trap_state() {
962 EXIT_REQUESTED.store(false, Ordering::SeqCst);
963 EXIT_CODE.store(0, Ordering::SeqCst);
964 EVENT_COUNT.store(0, Ordering::SeqCst);
965 if let Ok(mut guard) = EVENTS.lock() {
966 guard.clear();
967 }
968}
969
970pub fn set_trace_on_exit(json: bool) {
972 TRACE_ON_EXIT.store(true, Ordering::SeqCst);
973 TRACE_JSON.store(json, Ordering::SeqCst);
974}
975
976pub fn clear_trace_on_exit() {
978 TRACE_ON_EXIT.store(false, Ordering::SeqCst);
979 TRACE_JSON.store(false, Ordering::SeqCst);
980}
981
982pub fn set_expect_code(code: i32) {
984 EXPECT_CODE.store(code, Ordering::SeqCst);
985}
986
987pub fn clear_expect_code() {
989 EXPECT_CODE.store(i32::MIN, Ordering::SeqCst);
990}
991
992pub fn finish_with_exit_code(code: i32) -> ! {
997 EXIT_CODE.store(code, Ordering::SeqCst);
998 EXIT_REQUESTED.store(true, Ordering::SeqCst);
999 if TRACE_ON_EXIT.load(Ordering::SeqCst) {
1000 dump_events_before_exit();
1001 }
1002 let expect = EXPECT_CODE.load(Ordering::SeqCst);
1003 let status = if expect == i32::MIN || expect == code {
1004 code
1005 } else {
1006 let msg = format!("error: guest exit code {code} != expected {expect}\n");
1007 drop(io::stderr().write_all(msg.as_bytes()));
1008 1
1009 };
1010 unsafe { libc::_exit(status) };
1012}
1013
1014#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
1015unsafe extern "C" fn trap_sigaction(
1016 signo: libc::c_int,
1017 _info: *mut libc::siginfo_t,
1018 ctx: *mut libc::c_void,
1019) {
1020 if signo != libc::SIGTRAP || ctx.is_null() {
1021 return;
1022 }
1023
1024 let uctx = unsafe { &mut *ctx.cast::<libc::ucontext_t>() };
1026 let m = &mut uctx.uc_mcontext;
1027 let pc = m.pc;
1028 let x0 = m.regs[0];
1030 let x1 = m.regs[1];
1031 let x2 = m.regs[2];
1032 let x3 = m.regs[3];
1033 let x4 = m.regs[4];
1034 let x5 = m.regs[5];
1035 let x6 = m.regs[6];
1036 let x16 = m.regs[16];
1037
1038 let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
1039 let result = syscall::dispatch(SyscallArgs {
1040 pc,
1041 number: sys_no,
1042 x0,
1043 x1,
1044 x2,
1045 x3,
1046 x4,
1047 x5,
1048 x6,
1049 });
1050
1051 if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
1053 maybe_push_event(pc, sys_no, result.name, x0, x1, x2, result.retval, result.error);
1054 }
1055
1056 if let Some(ret) = result.retval {
1057 m.regs[0] = ret;
1058 }
1059 if result.error {
1061 m.pstate |= PSTATE_C;
1062 } else if result.retval.is_some() {
1063 m.pstate &= !PSTATE_C;
1064 }
1065
1066 match result.outcome {
1067 TrapOutcome::Exit { code } => {
1068 finish_with_exit_code(code);
1069 }
1070 TrapOutcome::Continue => {
1071 m.pc = pc.wrapping_add(4);
1072 }
1073 TrapOutcome::ThreadExit => {
1074 if !crate::thread::redirect_ucontext_to_host_exit(m) {
1078 finish_with_exit_code(0);
1079 }
1080 }
1081 }
1082}
1083
1084fn dump_events_before_exit() {
1085 use std::fmt::Write as _;
1086
1087 let json = TRACE_JSON.load(Ordering::SeqCst);
1088 let events = EVENTS.lock().map(|g| g.clone()).unwrap_or_default();
1089 if json {
1090 let mut out = String::from("{\"events\":[");
1091 for (i, e) in events.iter().enumerate() {
1092 if i > 0 {
1093 out.push(',');
1094 }
1095 let sys = e
1096 .syscall
1097 .map_or_else(|| "null".to_owned(), |s| s.to_string());
1098 let ret = e
1099 .retval
1100 .map_or_else(|| "null".to_owned(), |v| v.to_string());
1101 let _ = write!(
1102 out,
1103 "{{\"pc\":{},\"syscall\":{},\"name\":\"{}\",\"arg0\":{},\"arg1\":{},\"arg2\":{},\"retval\":{},\"error\":{}}}",
1104 e.pc, sys, e.name, e.arg0, e.arg1, e.arg2, ret, e.error
1105 );
1106 }
1107 out.push_str("]}\n");
1108 drop(io::stdout().write_all(out.as_bytes()));
1109 drop(io::stdout().flush());
1110 } else {
1111 let header = format!("trace events: {}\n", events.len());
1112 drop(io::stdout().write_all(header.as_bytes()));
1113 for (i, e) in events.iter().enumerate() {
1114 let ret = e
1115 .retval
1116 .map_or_else(|| "-".to_owned(), |v| format!("{v:#x}"));
1117 let err = if e.error { " err" } else { "" };
1118 let line = format!(
1119 " [{i}] pc={:#x} {} sys={:?} args=({:#x}, {:#x}, {:#x}) ret={ret}{err}\n",
1120 e.pc, e.name, e.syscall, e.arg0, e.arg1, e.arg2
1121 );
1122 drop(io::stdout().write_all(line.as_bytes()));
1123 }
1124 drop(io::stdout().flush());
1125 }
1126}
1127
1128#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
1129#[allow(clippy::too_many_arguments)]
1130fn maybe_push_event(
1131 pc: u64,
1132 sys_no: u32,
1133 name: &'static str,
1134 arg0: u64,
1135 arg1: u64,
1136 arg2: u64,
1137 retval: Option<u64>,
1138 error: bool,
1139) {
1140 let max = MAX_EVENTS.load(Ordering::Relaxed);
1141 if max == 0 {
1143 return;
1144 }
1145 let n = EVENT_COUNT.fetch_add(1, Ordering::Relaxed);
1146 if n >= max {
1147 return;
1148 }
1149 if let Ok(mut guard) = EVENTS.lock()
1150 && u64::try_from(guard.len()).unwrap_or(u64::MAX) < max
1151 {
1152 guard.push(TrapEvent {
1153 pc,
1154 syscall: Some(sys_no),
1155 name: name.to_owned(),
1156 arg0,
1157 arg1,
1158 arg2,
1159 retval,
1160 error,
1161 });
1162 }
1163}
1164
1165#[cfg(test)]
1166mod tests {
1167 use super::*;
1168
1169 #[test]
1170 fn detects_svc_encoding() {
1171 assert!(is_svc(0xD400_0001));
1172 let svc80 = 0xD400_0001 | (0x80_u32 << 5);
1173 assert!(is_svc(svc80));
1174 assert!(!is_svc(0xD503_201F));
1175 assert!(!is_svc(brk_encoding(BRK_TRAP_IMM)));
1176 }
1177
1178 #[test]
1179 fn brk_encoding_roundtrip_imm() {
1180 let enc = brk_encoding(BRK_TRAP_IMM);
1181 let imm = (enc >> 5) & 0xFFFF;
1182 assert_eq!(imm, BRK_TRAP_IMM);
1183 }
1184}