1#![allow(unsafe_code)]
14
15use std::io::{self, Write};
16use std::sync::Mutex;
17use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering};
18
19use thiserror::Error;
20
21use crate::mem::{
22 MapError, MappedRegion, VM_PROT_EXECUTE, VM_PROT_READ, VM_PROT_WRITE, mprotect_darwin,
23 mprotect_rw,
24};
25use crate::syscall;
26#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
27use crate::syscall::SyscallArgs;
28
29pub const BRK_TRAP_IMM: u32 = 0x0F00;
31
32const SVC_MASK: u32 = 0xFFE0_001F;
34const SVC_BASE: u32 = 0xD400_0001;
35
36const fn brk_encoding(imm16: u32) -> u32 {
38 0xD420_0000 | ((imm16 & 0xFFFF) << 5)
39}
40
41#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
43#[allow(
44 clippy::as_conversions,
45 clippy::cast_possible_truncation,
46 clippy::cast_sign_loss,
47 clippy::manual_range_contains
48)]
49fn branch_imm26(from: u64, to: u64) -> Option<u32> {
50 let delta = i64::try_from(to).ok()?.wrapping_sub(i64::try_from(from).ok()?);
51 if delta & 3 != 0 {
52 return None;
53 }
54 let imm = delta >> 2;
55 if imm < -(1_i64 << 25) || imm >= (1_i64 << 25) {
57 return None;
58 }
59 Some((imm as u32) & 0x03FF_FFFF)
61}
62
63#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
65fn bl_encoding(from: u64, to: u64) -> Option<u32> {
66 Some(0x9400_0000 | branch_imm26(from, to)?)
67}
68
69#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
71fn b_encoding(from: u64, to: u64) -> Option<u32> {
72 Some(0x1400_0000 | branch_imm26(from, to)?)
73}
74
75pub const PSTATE_C: u64 = 1 << 29;
79
80#[derive(Debug, Clone, PartialEq, Eq)]
82pub struct TrapEvent {
83 pub pc: u64,
85 pub syscall: Option<u32>,
87 pub name: String,
89 pub arg0: u64,
91 pub arg1: u64,
93 pub arg2: u64,
95 pub retval: Option<u64>,
97 pub error: bool,
99}
100
101#[derive(Debug, Clone, Copy, PartialEq, Eq)]
103pub enum TrapOutcome {
104 Exit {
106 code: i32,
108 },
109 Continue,
111 ThreadExit,
113}
114
115#[derive(Debug, Clone)]
117pub struct TrapConfig {
118 pub max_events: usize,
120 pub max_syscalls: usize,
122}
123
124impl Default for TrapConfig {
125 fn default() -> Self {
126 Self {
127 max_events: 256,
128 max_syscalls: 256,
129 }
130 }
131}
132
133#[derive(Debug, Error)]
135pub enum TrapError {
136 #[error("failed to install trap signal handler: {0}")]
138 SignalSetup(#[source] io::Error),
139
140 #[error(transparent)]
142 Map(#[from] MapError),
143
144 #[error("trap backend requires Linux aarch64 for live execution")]
146 UnsupportedArch,
147}
148
149#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
150static HANDLER_INSTALLED: AtomicBool = AtomicBool::new(false);
151static EXIT_CODE: AtomicI32 = AtomicI32::new(0);
152static EXIT_REQUESTED: AtomicBool = AtomicBool::new(false);
153static EVENT_COUNT: AtomicU64 = AtomicU64::new(0);
154static MAX_EVENTS: AtomicU64 = AtomicU64::new(256);
155static TRACE_ON_EXIT: AtomicBool = AtomicBool::new(false);
156static TRACE_JSON: AtomicBool = AtomicBool::new(false);
157static EXPECT_CODE: AtomicI32 = AtomicI32::new(i32::MIN);
159static EVENTS: Mutex<Vec<TrapEvent>> = Mutex::new(Vec::new());
160
161pub fn patch_svc_to_brk(regions: &mut [MappedRegion]) -> Result<usize, TrapError> {
166 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
167 {
168 patch_svc_linux(regions)
169 }
170 #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
171 {
172 patch_svc_brk_only(regions)
173 }
174}
175
176#[cfg_attr(
177 all(target_arch = "aarch64", target_os = "linux"),
178 allow(dead_code)
179)]
180fn patch_svc_brk_only(regions: &mut [MappedRegion]) -> Result<usize, TrapError> {
181 let mut patched = 0_usize;
182 for region in regions.iter_mut() {
183 if region.prot & VM_PROT_EXECUTE == 0 {
184 continue;
185 }
186 mprotect_rw(region)?;
187 let guest_base = region.guest_addr;
188 let bytes = region.host_bytes_mut();
189 let mut off = 0_usize;
190 while off.saturating_add(4) <= bytes.len() {
191 let end = off.saturating_add(4);
192 let Some(word) = bytes.get(off..end) else {
193 break;
194 };
195 let Ok(word_bytes) = <[u8; 4]>::try_from(word) else {
196 break;
197 };
198 let insn = u32::from_le_bytes(word_bytes);
199 if is_svc(insn) {
200 let brk = brk_encoding(BRK_TRAP_IMM).to_le_bytes();
201 if let Some(slot) = bytes.get_mut(off..end) {
202 slot.copy_from_slice(&brk);
203 patched = patched.saturating_add(1);
204 }
205 }
206 let _ = guest_base;
207 off = end;
208 }
209 let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
210 mprotect_darwin(region, restore)?;
211 region.prot = restore;
212 }
213 Ok(patched)
214}
215
216const fn is_svc(insn: u32) -> bool {
217 (insn & SVC_MASK) == SVC_BASE
218}
219
220#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
233fn patch_svc_linux(regions: &mut [MappedRegion]) -> Result<usize, TrapError> {
234 if !trampoline_enabled() {
235 return patch_svc_brk_only(regions);
236 }
237 let mut patched = 0_usize;
238 for region in regions.iter_mut() {
239 if region.prot & VM_PROT_EXECUTE == 0 {
240 continue;
241 }
242 mprotect_rw(region)?;
243 let guest_base = region.guest_addr;
244 let mut sites = Vec::new();
246 {
247 let bytes = region.host_bytes();
248 let mut off = 0_usize;
249 while off.saturating_add(4) <= bytes.len() {
250 let end = off.saturating_add(4);
251 if let Some(word) = bytes.get(off..end)
252 && let Ok(word_bytes) = <[u8; 4]>::try_from(word)
253 {
254 let insn = u32::from_le_bytes(word_bytes);
255 if is_svc(insn) {
256 sites.push(off);
257 }
258 }
259 off = end;
260 }
261 }
262 if sites.is_empty() {
263 let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
264 mprotect_darwin(region, restore)?;
265 region.prot = restore;
266 continue;
267 }
268
269 patched = patched.saturating_add(patch_region_with_veneers(region, guest_base, &sites));
270 }
271 Ok(patched)
272}
273
274#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
276const VENEER_BYTES: usize = 16;
277
278#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
280const VENEER_PROBE_RADIUS: u64 = 128;
281
282#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
284#[inline]
285fn align_up_to_page(len: usize, page: usize) -> usize {
286 if page <= 1 {
287 return len.max(1);
288 }
289 let mask = page.saturating_sub(1);
290 len.saturating_add(mask) & !mask
291}
292
293#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
295unsafe extern "C" {
296 fn __clear_cache(start: *mut libc::c_void, end: *mut libc::c_void);
297}
298
299#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
301fn trampoline_enabled() -> bool {
302 match std::env::var_os("KAKEHASHI_TRAMPOLINE") {
303 None => false,
304 Some(v) => {
305 v == "1"
306 || v.eq_ignore_ascii_case("true")
307 || v.eq_ignore_ascii_case("yes")
308 || v.eq_ignore_ascii_case("on")
309 }
310 }
311}
312
313#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
315fn clear_icache(ptr: *mut u8, len: usize) {
316 if ptr.is_null() || len == 0 {
317 return;
318 }
319 unsafe {
321 let start = ptr.cast::<libc::c_void>();
322 let end = ptr.wrapping_add(len).cast::<libc::c_void>();
323 __clear_cache(start, end);
324 }
325}
326
327pub fn clear_trampoline_cache() {
329 }
331
332#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
336fn patch_region_with_veneers(
337 region: &mut MappedRegion,
338 guest_base: u64,
339 sites: &[usize],
340) -> usize {
341 let n_sites = sites.len();
342 let Some(hub) = install_veneer_hub(guest_base, region.host_len(), n_sites) else {
343 return patch_sites_brk(region, sites);
344 };
345
346 let mut patched = 0_usize;
347 let slice = unsafe { std::slice::from_raw_parts_mut(hub.ptr, hub.map_len) };
349
350 for (i, &off) in sites.iter().enumerate() {
351 let site_va = guest_base.saturating_add(u64::try_from(off).unwrap_or(0));
352 let site_ret = site_va.saturating_add(4);
353 let veneer_off = hub.veneer0.saturating_add(i.saturating_mul(VENEER_BYTES));
354 let veneer_va = hub.base_va.saturating_add(u64::try_from(veneer_off).unwrap_or(0));
355 let end = off.saturating_add(4);
356
357 let ok = write_veneer(slice, veneer_off, veneer_va, hub.shared_va, site_ret)
358 .and_then(|()| b_encoding(site_va, veneer_va))
359 .and_then(|enc| {
360 let bytes = region.host_bytes_mut();
361 let slot = bytes.get_mut(off..end)?;
362 slot.copy_from_slice(&enc.to_le_bytes());
363 Some(())
364 })
365 .is_some();
366
367 if !ok {
368 let bytes = region.host_bytes_mut();
369 let brk = brk_encoding(BRK_TRAP_IMM).to_le_bytes();
370 if let Some(slot) = bytes.get_mut(off..end) {
371 slot.copy_from_slice(&brk);
372 }
373 }
374 patched = patched.saturating_add(1);
375 }
376
377 let prot_rx = libc::PROT_READ | libc::PROT_EXEC;
379 if !crate::host::mprotect(hub.ptr, hub.map_len, prot_rx) {
380 let _ = crate::host::munmap(hub.ptr, hub.map_len);
381 return patch_sites_brk(region, sites);
382 }
383 clear_icache(hub.ptr, hub.map_len);
384 crate::mem::register_owned(
385 hub.base_va,
386 hub.ptr,
387 hub.map_len,
388 VM_PROT_READ | VM_PROT_EXECUTE,
389 );
390
391 let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
392 drop(mprotect_darwin(region, restore));
393 region.prot = restore;
394 let host = region.host_addr();
395 let len = region.host_len();
396 if let (Ok(base), true) = (usize::try_from(host), len > 0) {
397 let p = std::ptr::with_exposed_provenance_mut::<u8>(base);
398 clear_icache(p, len);
399 }
400 patched
401}
402
403#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
404fn patch_sites_brk(region: &mut MappedRegion, sites: &[usize]) -> usize {
405 let mut patched = 0_usize;
406 let bytes = region.host_bytes_mut();
407 for &off in sites {
408 let end = off.saturating_add(4);
409 let brk = brk_encoding(BRK_TRAP_IMM).to_le_bytes();
410 if let Some(slot) = bytes.get_mut(off..end) {
411 slot.copy_from_slice(&brk);
412 patched = patched.saturating_add(1);
413 }
414 }
415 let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
416 drop(mprotect_darwin(region, restore));
417 region.prot = restore;
418 patched
419}
420
421#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
423struct VeneerHub {
424 ptr: *mut u8,
425 map_len: usize,
426 base_va: u64,
427 shared_va: u64,
428 veneer0: usize,
429}
430
431#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
433fn install_veneer_hub(region_guest: u64, region_len: usize, n_sites: usize) -> Option<VeneerHub> {
434 let region_end = region_guest.saturating_add(u64::try_from(region_len).unwrap_or(0));
435 let site_lo = region_guest;
436 let site_hi = region_end.saturating_sub(4).max(region_guest);
437
438 let page = crate::host::page_size().unwrap_or(4096);
439 let page_u = u64::try_from(page).unwrap_or(4096);
440 let code = trampoline_template();
441 let veneer0 = code.len().saturating_add(15) & !15_usize;
442 let need = veneer0.saturating_add(n_sites.saturating_mul(VENEER_BYTES));
443 let map_len = align_up_to_page(need, page).max(page);
444
445 let flags_fixed = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | crate::host::fixed_map_flag();
446 let flags_free = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
447 let prot_rw = libc::PROT_READ | libc::PROT_WRITE;
448
449 let after0 = region_end.saturating_add(page_u.saturating_sub(1)) & !(page_u.saturating_sub(1));
450 let before0 = region_guest.saturating_sub(page_u) & !(page_u.saturating_sub(1));
451
452 for i in 0..VENEER_PROBE_RADIUS {
453 let mut candidates = [after0.saturating_add(i.saturating_mul(page_u)), 0_u64];
454 if before0 >= i.saturating_mul(page_u) {
455 candidates[1] = before0.saturating_sub(i.saturating_mul(page_u));
456 }
457 for &prefer in &candidates {
458 if prefer == 0 {
459 continue;
460 }
461 if b_encoding(site_lo, prefer).is_none() || b_encoding(site_hi, prefer).is_none() {
463 continue;
464 }
465 if let Some(ptr) =
466 crate::host::mmap(Some(prefer), map_len, prot_rw, flags_fixed, -1, 0)
467 {
468 return init_hub_slice(ptr, map_len, code, veneer0);
469 }
470 }
471 }
472
473 let ptr = crate::host::mmap(None, map_len, prot_rw, flags_free, -1, 0)?;
474 let va = crate::host::ptr_addr_u64(ptr);
475 if b_encoding(site_lo, va).is_none() || b_encoding(site_hi, va).is_none() {
476 let _ = crate::host::munmap(ptr, map_len);
477 return None;
478 }
479 init_hub_slice(ptr, map_len, code, veneer0)
480}
481
482#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
483fn init_hub_slice(ptr: *mut u8, map_len: usize, code: &[u8], veneer0: usize) -> Option<VeneerHub> {
484 let slice = unsafe { std::slice::from_raw_parts_mut(ptr, map_len) };
486 if code.len() > slice.len() || veneer0 > slice.len() {
487 let _ = crate::host::munmap(ptr, map_len);
488 return None;
489 }
490 slice.fill(0);
491 slice.get_mut(..code.len())?.copy_from_slice(code);
492 let reloc_off = trampoline_reloc_offset();
493 if let Some(slot) = slice.get_mut(reloc_off..reloc_off.saturating_add(8)) {
494 slot.copy_from_slice(&trampoline_rust_entry_addr().to_le_bytes());
495 } else {
496 let _ = crate::host::munmap(ptr, map_len);
497 return None;
498 }
499 let base_va = crate::host::ptr_addr_u64(ptr);
500 Some(VeneerHub {
501 ptr,
502 map_len,
503 base_va,
504 shared_va: base_va,
505 veneer0,
506 })
507}
508
509#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
511fn write_veneer(
512 slice: &mut [u8],
513 veneer_off: usize,
514 veneer_va: u64,
515 shared_va: u64,
516 site_ret: u64,
517) -> Option<()> {
518 let str_lr: u32 = 0xf81f_0ffe;
520 let ldr_lr: u32 = 0xf841_07fe;
522 let bl = bl_encoding(veneer_va.saturating_add(4), shared_va)?;
523 let b_back = b_encoding(veneer_va.saturating_add(12), site_ret)?;
524 let words = [str_lr, bl, ldr_lr, b_back];
525 let end = veneer_off.saturating_add(VENEER_BYTES);
526 let slot = slice.get_mut(veneer_off..end)?;
527 for (i, w) in words.iter().enumerate() {
528 let o = i.saturating_mul(4);
529 slot.get_mut(o..o.saturating_add(4))?.copy_from_slice(&w.to_le_bytes());
530 }
531 Some(())
532}
533
534#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
538fn trampoline_template() -> &'static [u8] {
539 TRAMP_BYTES
540}
541
542#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
543fn trampoline_reloc_offset() -> usize {
544 TRAMP_RELOC_OFF
545}
546
547#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
548#[allow(clippy::as_conversions, function_casts_as_integer)]
549fn trampoline_rust_entry_addr() -> u64 {
550 u64::try_from(kh_trampoline_dispatch as usize).unwrap_or(0)
551}
552
553#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
555#[repr(C)]
556pub struct TrampRet {
557 pub retval: u64,
559 pub error: u64,
561}
562
563#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
578#[unsafe(no_mangle)]
579pub unsafe extern "C" fn kh_trampoline_dispatch(
580 x0: u64,
581 x1: u64,
582 x2: u64,
583 x3: u64,
584 x4: u64,
585 x5: u64,
586 x6: u64,
587 x16: u64,
588) -> TrampRet {
589 let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
590 let result = syscall::dispatch(SyscallArgs {
591 pc: 0,
592 number: sys_no,
593 x0,
594 x1,
595 x2,
596 x3,
597 x4,
598 x5,
599 x6,
600 });
601 if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
602 maybe_push_event(0, sys_no, result.name, x0, x1, x2, result.retval, result.error);
603 }
604 match result.outcome {
605 TrapOutcome::Exit { code } => finish_with_exit_code(code),
606 TrapOutcome::ThreadExit => {
607 crate::thread::exit_current_guest_worker();
609 }
610 TrapOutcome::Continue => {}
611 }
612 TrampRet {
613 retval: result.retval.unwrap_or(0),
614 error: u64::from(result.error),
615 }
616}
617
618#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
635std::arch::global_asm!(
636 r#"
637 .text
638 .align 2
639 .global kh_hypercall_entry
640 .type kh_hypercall_entry, %function
641kh_hypercall_entry:
642 // Guest prolog: GPRs + full NEON *before* any bl (NEON must survive host TLS).
643 sub sp, sp, #640
644 stp x29, x30, [sp, #0]
645 stp x0, x1, [sp, #16]
646 stp x2, x3, [sp, #32]
647 stp x4, x5, [sp, #48]
648 stp x6, x7, [sp, #64]
649 mov x29, sp
650 stp q0, q1, [sp, #96]
651 stp q2, q3, [sp, #128]
652 stp q4, q5, [sp, #160]
653 stp q6, q7, [sp, #192]
654 stp q8, q9, [sp, #224]
655 stp q10, q11, [sp, #256]
656 stp q12, q13, [sp, #288]
657 stp q14, q15, [sp, #320]
658 stp q16, q17, [sp, #352]
659 stp q18, q19, [sp, #384]
660 stp q20, q21, [sp, #416]
661 stp q22, q23, [sp, #448]
662 stp q24, q25, [sp, #480]
663 stp q26, q27, [sp, #512]
664 stp q28, q29, [sp, #544]
665 stp q30, q31, [sp, #576]
666 mrs x9, fpcr
667 mrs x10, fpsr
668 // #608 exceeds signed LDP imm7*8 range (±512); use base+add.
669 add x11, sp, #608
670 stp x9, x10, [x11]
671
672 // Host glibc TLS *before* alt-stack lookup/alloc (mmap/mutex need host TPIDR).
673 bl kh_tls_enter_host
674 bl kh_host_alt_sp
675 cbz x0, 1f
676 mov x9, sp // guest prolog frame
677 mov sp, x0 // host alt stack
678 ldr x30, [x9, #8] // reload LR → freestanding thin caller
679 // Host frame: [guest_frame, lr]
680 stp x9, x30, [sp, #-16]!
681 // Reload guest frame ptr / args (enter_host + alt_sp clobbered temps).
682 ldr x9, [sp]
683 ldp x0, x1, [x9, #16]
684 ldp x2, x3, [x9, #32]
685 ldp x4, x5, [x9, #48]
686 ldp x6, x7, [x9, #64]
687 mov x16, x7
688 bl kh_neon_tramp_entry
689 // Preserve TrampRet (x0/x1) across TLS restore.
690 stp x0, x1, [sp, #-16]!
691 bl kh_tls_leave_host
692 ldp x0, x1, [sp], #16
693 ldp x9, x30, [sp], #16
694 mov sp, x9
695 // Restore guest NEON from prolog (pre-bl snapshot).
696 ldp q0, q1, [sp, #96]
697 ldp q2, q3, [sp, #128]
698 ldp q4, q5, [sp, #160]
699 ldp q6, q7, [sp, #192]
700 ldp q8, q9, [sp, #224]
701 ldp q10, q11, [sp, #256]
702 ldp q12, q13, [sp, #288]
703 ldp q14, q15, [sp, #320]
704 ldp q16, q17, [sp, #352]
705 ldp q18, q19, [sp, #384]
706 ldp q20, q21, [sp, #416]
707 ldp q22, q23, [sp, #448]
708 ldp q24, q25, [sp, #480]
709 ldp q26, q27, [sp, #512]
710 ldp q28, q29, [sp, #544]
711 ldp q30, q31, [sp, #576]
712 add x11, sp, #608
713 ldp x9, x10, [x11]
714 msr fpcr, x9
715 msr fpsr, x10
716 ldr x29, [sp, #0]
717 add sp, sp, #640
718 ret
7191:
720 // No alt stack: dispatch on the guest stack (ST / fallback only).
721 // Already on host TLS from the enter above.
722 ldp x0, x1, [sp, #16]
723 ldp x2, x3, [sp, #32]
724 ldp x4, x5, [sp, #48]
725 ldp x6, x7, [sp, #64]
726 mov x16, x7
727 // Preserve real LR across the call (still on guest prolog frame).
728 ldr x30, [sp, #8]
729 bl kh_neon_tramp_entry
730 stp x0, x1, [sp, #-16]!
731 bl kh_tls_leave_host
732 ldp x0, x1, [sp], #16
733 // Restore guest NEON from prolog.
734 ldp q0, q1, [sp, #96]
735 ldp q2, q3, [sp, #128]
736 ldp q4, q5, [sp, #160]
737 ldp q6, q7, [sp, #192]
738 ldp q8, q9, [sp, #224]
739 ldp q10, q11, [sp, #256]
740 ldp q12, q13, [sp, #288]
741 ldp q14, q15, [sp, #320]
742 ldp q16, q17, [sp, #352]
743 ldp q18, q19, [sp, #384]
744 ldp q20, q21, [sp, #416]
745 ldp q22, q23, [sp, #448]
746 ldp q24, q25, [sp, #480]
747 ldp q26, q27, [sp, #512]
748 ldp q28, q29, [sp, #544]
749 ldp q30, q31, [sp, #576]
750 add x11, sp, #608
751 ldp x9, x10, [x11]
752 msr fpcr, x9
753 msr fpsr, x10
754 ldp x29, x30, [sp, #0]
755 add sp, sp, #640
756 ret
757 .size kh_hypercall_entry, .-kh_hypercall_entry
758 "#
759);
760
761#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
763std::arch::global_asm!(
764 r#"
765 .text
766 .align 2
767 .global kh_neon_tramp_entry
768 .type kh_neon_tramp_entry, %function
769kh_neon_tramp_entry:
770 adrp x17, KH_NEON_TRAMP_VA
771 ldr x17, [x17, :lo12:KH_NEON_TRAMP_VA]
772 cbz x17, 1f
773 br x17
7741:
775 mov x7, x16
776 b kh_trampoline_dispatch
777 .size kh_neon_tramp_entry, .-kh_neon_tramp_entry
778
779 .data
780 .align 3
781 .global KH_NEON_TRAMP_VA
782KH_NEON_TRAMP_VA:
783 .quad 0
784 "#
785);
786
787#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
788unsafe extern "C" {
789 fn kh_hypercall_entry();
790 static mut KH_NEON_TRAMP_VA: u64;
791}
792
793#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
795#[must_use]
796pub fn hypercall_entry_addr() -> u64 {
797 ensure_neon_tramp();
798 #[allow(clippy::as_conversions, function_casts_as_integer)]
799 {
800 u64::try_from(kh_hypercall_entry as usize).unwrap_or(0)
801 }
802}
803
804#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
806fn ensure_neon_tramp() {
807 static ONCE: AtomicU64 = AtomicU64::new(0);
808 static LOCK: Mutex<()> = Mutex::new(());
809
810 if ONCE.load(Ordering::Acquire) != 0 {
811 return;
812 }
813 let _g = LOCK
814 .lock()
815 .unwrap_or_else(std::sync::PoisonError::into_inner);
816 if ONCE.load(Ordering::Acquire) != 0 {
817 return;
818 }
819
820 let code = trampoline_template();
821 let page = crate::host::page_size().unwrap_or(4096);
822 let map_len = align_up_to_page(code.len(), page).max(page);
823 let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
824 let prot_rw = libc::PROT_READ | libc::PROT_WRITE;
825 let Some(ptr) = crate::host::mmap(None, map_len, prot_rw, flags, -1, 0) else {
826 tracing::error!("NEON tramp mmap failed");
827 return;
828 };
829 let slice = unsafe { std::slice::from_raw_parts_mut(ptr, map_len) };
831 slice.fill(0);
832 if let Some(dst) = slice.get_mut(..code.len()) {
833 dst.copy_from_slice(code);
834 } else {
835 let _ = crate::host::munmap(ptr, map_len);
836 return;
837 }
838 let reloc_off = trampoline_reloc_offset();
839 if let Some(slot) = slice.get_mut(reloc_off..reloc_off.saturating_add(8)) {
840 slot.copy_from_slice(&trampoline_rust_entry_addr().to_le_bytes());
841 } else {
842 let _ = crate::host::munmap(ptr, map_len);
843 return;
844 }
845 let prot_rx = libc::PROT_READ | libc::PROT_EXEC;
846 if !crate::host::mprotect(ptr, map_len, prot_rx) {
847 tracing::error!("NEON tramp mprotect RX failed");
848 let _ = crate::host::munmap(ptr, map_len);
849 return;
850 }
851 clear_icache(ptr, map_len);
852 let va = crate::host::ptr_addr_u64(ptr);
853 unsafe {
855 KH_NEON_TRAMP_VA = va;
856 }
857 ONCE.store(va, Ordering::Release);
858 tracing::info!(va = format_args!("{va:#x}"), "mapped NEON tramp for hypercall");
859}
860
861#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
862#[must_use]
863pub fn hypercall_entry_addr() -> u64 {
864 0
865}
866
867#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
874const TRAMP_RELOC_OFF: usize = 0x100;
875
876#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
882#[rustfmt::skip]
883const TRAMP_BYTES: &[u8] = &[
884 0x5f, 0x24, 0x03, 0xd5, 0xff, 0x03, 0x0a, 0xd1, 0xe8, 0x27, 0x00, 0xa9, 0xea, 0x2f, 0x01, 0xa9,
885 0xec, 0x37, 0x02, 0xa9, 0xee, 0x3f, 0x03, 0xa9, 0xf1, 0x4b, 0x04, 0xa9, 0xfe, 0x2b, 0x00, 0xf9,
886 0x08, 0x44, 0x3b, 0xd5, 0x29, 0x44, 0x3b, 0xd5, 0xe8, 0x27, 0x06, 0xa9, 0xe0, 0x87, 0x03, 0xad,
887 0xe2, 0x8f, 0x04, 0xad, 0xe4, 0x97, 0x05, 0xad, 0xe6, 0x9f, 0x06, 0xad, 0xe8, 0xa7, 0x07, 0xad,
888 0xea, 0xaf, 0x08, 0xad, 0xec, 0xb7, 0x09, 0xad, 0xee, 0xbf, 0x0a, 0xad, 0xf0, 0xc7, 0x0b, 0xad,
889 0xf2, 0xcf, 0x0c, 0xad, 0xf4, 0xd7, 0x0d, 0xad, 0xf6, 0xdf, 0x0e, 0xad, 0xf8, 0xe7, 0x0f, 0xad,
890 0xfa, 0xef, 0x10, 0xad, 0xfc, 0xf7, 0x11, 0xad, 0xfe, 0xff, 0x12, 0xad, 0xe7, 0x03, 0x10, 0xaa,
891 0x88, 0x04, 0x00, 0x58, 0x00, 0x01, 0x3f, 0xd6, 0x02, 0x42, 0x3b, 0xd5, 0x23, 0x00, 0x80, 0x52,
892 0x63, 0x88, 0x63, 0xd3, 0x42, 0x00, 0x23, 0x8a, 0x41, 0x00, 0x00, 0xb4, 0x42, 0x00, 0x03, 0xaa,
893 0x02, 0x42, 0x1b, 0xd5, 0xe8, 0x27, 0x46, 0xa9, 0x08, 0x44, 0x1b, 0xd5, 0x29, 0x44, 0x1b, 0xd5,
894 0xe0, 0x87, 0x43, 0xad, 0xe2, 0x8f, 0x44, 0xad, 0xe4, 0x97, 0x45, 0xad, 0xe6, 0x9f, 0x46, 0xad,
895 0xe8, 0xa7, 0x47, 0xad, 0xea, 0xaf, 0x48, 0xad, 0xec, 0xb7, 0x49, 0xad, 0xee, 0xbf, 0x4a, 0xad,
896 0xf0, 0xc7, 0x4b, 0xad, 0xf2, 0xcf, 0x4c, 0xad, 0xf4, 0xd7, 0x4d, 0xad, 0xf6, 0xdf, 0x4e, 0xad,
897 0xf8, 0xe7, 0x4f, 0xad, 0xfa, 0xef, 0x50, 0xad, 0xfc, 0xf7, 0x51, 0xad, 0xfe, 0xff, 0x52, 0xad,
898 0xfe, 0x2b, 0x40, 0xf9, 0xf1, 0x4b, 0x44, 0xa9, 0xee, 0x3f, 0x43, 0xa9, 0xec, 0x37, 0x42, 0xa9,
899 0xea, 0x2f, 0x41, 0xa9, 0xe8, 0x27, 0x40, 0xa9, 0xff, 0x03, 0x0a, 0x91, 0xc0, 0x03, 0x5f, 0xd6,
900 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ];
902pub fn install_trap_handlers(config: &TrapConfig) -> Result<(), TrapError> {
904 MAX_EVENTS.store(
905 u64::try_from(config.max_events).unwrap_or(256),
906 Ordering::SeqCst,
907 );
908 syscall::reset_syscall_state(config.max_syscalls);
909
910 #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
911 {
912 let _ = config;
913 Err(TrapError::UnsupportedArch)
914 }
915
916 #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
917 {
918 if HANDLER_INSTALLED.swap(true, Ordering::SeqCst) {
919 reset_trap_state();
920 syscall::reset_syscall_state(config.max_syscalls);
921 return Ok(());
922 }
923
924 disable_core_dumps();
927
928 unsafe {
930 let mut sa: libc::sigaction = std::mem::zeroed();
931 sa.sa_flags = libc::SA_SIGINFO;
932 #[allow(clippy::as_conversions, function_casts_as_integer)]
933 {
934 sa.sa_sigaction = trap_sigaction as *const () as usize;
935 }
936 libc::sigemptyset(std::ptr::addr_of_mut!(sa.sa_mask));
937 if libc::sigaction(libc::SIGTRAP, std::ptr::addr_of!(sa), std::ptr::null_mut()) != 0 {
938 HANDLER_INSTALLED.store(false, Ordering::SeqCst);
939 return Err(TrapError::SignalSetup(io::Error::last_os_error()));
940 }
941
942 let mut fault: libc::sigaction = std::mem::zeroed();
944 fault.sa_flags = libc::SA_SIGINFO;
945 #[allow(clippy::as_conversions, function_casts_as_integer)]
946 {
947 fault.sa_sigaction = guest_fault_sigaction as *const () as usize;
948 }
949 libc::sigemptyset(std::ptr::addr_of_mut!(fault.sa_mask));
950 for sig in [libc::SIGSEGV, libc::SIGBUS] {
951 if libc::sigaction(sig, std::ptr::addr_of!(fault), std::ptr::null_mut()) != 0 {
952 HANDLER_INSTALLED.store(false, Ordering::SeqCst);
953 return Err(TrapError::SignalSetup(io::Error::last_os_error()));
954 }
955 }
956 }
957 reset_trap_state();
958 Ok(())
959 }
960}
961
962#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
965fn disable_core_dumps() {
966 unsafe {
968 let lim = libc::rlimit {
969 rlim_cur: 0,
970 rlim_max: 0,
971 };
972 let _ = libc::setrlimit(libc::RLIMIT_CORE, std::ptr::addr_of!(lim));
973 }
974}
975
976#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
978unsafe extern "C" fn guest_fault_sigaction(
979 signo: libc::c_int,
980 info: *mut libc::siginfo_t,
981 ctx: *mut libc::c_void,
982) {
983 crate::tls::enter_host_tls();
985 let name = match signo {
986 libc::SIGSEGV => "SIGSEGV",
987 libc::SIGBUS => "SIGBUS",
988 _ => "FAULT",
989 };
990 let mut pc = 0_u64;
991 let mut sp = 0_u64;
992 let mut x0 = 0_u64;
993 let mut x1 = 0_u64;
994 let mut x8 = 0_u64;
995 let mut x16 = 0_u64;
996 let mut lr = 0_u64;
997 if !ctx.is_null() {
998 let uctx = unsafe { &*ctx.cast::<libc::ucontext_t>() };
1000 let m = &uctx.uc_mcontext;
1001 pc = m.pc;
1002 sp = m.sp;
1003 x0 = m.regs[0];
1004 x1 = m.regs[1];
1005 x8 = m.regs[8];
1006 x16 = m.regs[16];
1007 lr = m.regs[30];
1008 }
1009 let addr = if info.is_null() {
1010 0_u64
1011 } else {
1012 let p = unsafe { (*info).si_addr() };
1014 u64::try_from(p.addr()).unwrap_or(0)
1015 };
1016 let msg = format!(
1017 "error: guest {name} pc={pc:#x} addr={addr:#x} sp={sp:#x} lr={lr:#x} \
1018 x0={x0:#x} x1={x1:#x} x8={x8:#x} x16={x16:#x}\n"
1019 );
1020 drop(io::stderr().write_all(msg.as_bytes()));
1021 if let Ok(maps) = std::fs::read_to_string("/proc/self/maps") {
1023 for &va in &[pc, addr, lr, x16] {
1024 for line in maps.lines() {
1025 let Some((range, rest)) = line.split_once(' ') else {
1026 continue;
1027 };
1028 let Some((a, b)) = range.split_once('-') else {
1029 continue;
1030 };
1031 let (Ok(lo), Ok(hi)) = (u64::from_str_radix(a, 16), u64::from_str_radix(b, 16))
1032 else {
1033 continue;
1034 };
1035 if va >= lo && va < hi {
1036 let off = va.saturating_sub(lo);
1037 let m = format!(" map {va:#x} = {lo:#x}+{off:#x} {rest}\n");
1038 drop(io::stderr().write_all(m.as_bytes()));
1039 break;
1040 }
1041 }
1042 }
1043 }
1044 drop(io::stderr().flush());
1045 if TRACE_ON_EXIT.load(Ordering::SeqCst) {
1046 dump_events_before_exit();
1047 }
1048 unsafe { libc::_exit(128_i32.saturating_add(signo)) };
1050}
1051
1052#[must_use]
1054pub fn take_trace_events() -> Vec<TrapEvent> {
1055 EVENTS
1056 .lock()
1057 .map(|mut g| std::mem::take(&mut *g))
1058 .unwrap_or_default()
1059}
1060
1061#[must_use]
1063pub fn exit_requested() -> bool {
1064 EXIT_REQUESTED.load(Ordering::SeqCst)
1065}
1066
1067#[must_use]
1069pub fn take_exit_code() -> Option<i32> {
1070 if EXIT_REQUESTED.load(Ordering::SeqCst) {
1071 Some(EXIT_CODE.load(Ordering::SeqCst))
1072 } else {
1073 None
1074 }
1075}
1076
1077pub fn reset_trap_state() {
1079 EXIT_REQUESTED.store(false, Ordering::SeqCst);
1080 EXIT_CODE.store(0, Ordering::SeqCst);
1081 EVENT_COUNT.store(0, Ordering::SeqCst);
1082 if let Ok(mut guard) = EVENTS.lock() {
1083 guard.clear();
1084 }
1085}
1086
1087pub fn set_trace_on_exit(json: bool) {
1089 TRACE_ON_EXIT.store(true, Ordering::SeqCst);
1090 TRACE_JSON.store(json, Ordering::SeqCst);
1091}
1092
1093pub fn clear_trace_on_exit() {
1095 TRACE_ON_EXIT.store(false, Ordering::SeqCst);
1096 TRACE_JSON.store(false, Ordering::SeqCst);
1097}
1098
1099pub fn set_expect_code(code: i32) {
1101 EXPECT_CODE.store(code, Ordering::SeqCst);
1102}
1103
1104pub fn clear_expect_code() {
1106 EXPECT_CODE.store(i32::MIN, Ordering::SeqCst);
1107}
1108
1109pub fn finish_with_exit_code(code: i32) -> ! {
1114 EXIT_CODE.store(code, Ordering::SeqCst);
1115 EXIT_REQUESTED.store(true, Ordering::SeqCst);
1116 if TRACE_ON_EXIT.load(Ordering::SeqCst) {
1117 dump_events_before_exit();
1118 }
1119 let expect = EXPECT_CODE.load(Ordering::SeqCst);
1120 let status = if expect == i32::MIN || expect == code {
1121 code
1122 } else {
1123 let msg = format!("error: guest exit code {code} != expected {expect}\n");
1124 drop(io::stderr().write_all(msg.as_bytes()));
1125 1
1126 };
1127 unsafe { libc::_exit(status) };
1129}
1130
1131#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
1132unsafe extern "C" fn trap_sigaction(
1133 signo: libc::c_int,
1134 _info: *mut libc::siginfo_t,
1135 ctx: *mut libc::c_void,
1136) {
1137 if signo != libc::SIGTRAP || ctx.is_null() {
1138 return;
1139 }
1140
1141 crate::tls::enter_host_tls();
1143
1144 let uctx = unsafe { &mut *ctx.cast::<libc::ucontext_t>() };
1146 let m = &mut uctx.uc_mcontext;
1147 let pc = m.pc;
1148 let x0 = m.regs[0];
1150 let x1 = m.regs[1];
1151 let x2 = m.regs[2];
1152 let x3 = m.regs[3];
1153 let x4 = m.regs[4];
1154 let x5 = m.regs[5];
1155 let x6 = m.regs[6];
1156 let x16 = m.regs[16];
1157
1158 let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
1159 let result = syscall::dispatch(SyscallArgs {
1160 pc,
1161 number: sys_no,
1162 x0,
1163 x1,
1164 x2,
1165 x3,
1166 x4,
1167 x5,
1168 x6,
1169 });
1170
1171 if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
1173 maybe_push_event(pc, sys_no, result.name, x0, x1, x2, result.retval, result.error);
1174 }
1175
1176 if let Some(ret) = result.retval {
1177 m.regs[0] = ret;
1178 }
1179 if result.error {
1181 m.pstate |= PSTATE_C;
1182 } else if result.retval.is_some() {
1183 m.pstate &= !PSTATE_C;
1184 }
1185
1186 match result.outcome {
1187 TrapOutcome::Exit { code } => {
1188 finish_with_exit_code(code);
1189 }
1190 TrapOutcome::Continue => {
1191 m.pc = pc.wrapping_add(4);
1192 crate::tls::leave_host_tls();
1194 }
1195 TrapOutcome::ThreadExit => {
1196 if !crate::thread::redirect_ucontext_to_host_exit(m) {
1201 finish_with_exit_code(0);
1202 }
1203 }
1204 }
1205}
1206
1207fn dump_events_before_exit() {
1208 use std::fmt::Write as _;
1209
1210 let json = TRACE_JSON.load(Ordering::SeqCst);
1211 let events = EVENTS.lock().map(|g| g.clone()).unwrap_or_default();
1212 if json {
1213 let mut out = String::from("{\"events\":[");
1214 for (i, e) in events.iter().enumerate() {
1215 if i > 0 {
1216 out.push(',');
1217 }
1218 let sys = e
1219 .syscall
1220 .map_or_else(|| "null".to_owned(), |s| s.to_string());
1221 let ret = e
1222 .retval
1223 .map_or_else(|| "null".to_owned(), |v| v.to_string());
1224 let _ = write!(
1225 out,
1226 "{{\"pc\":{},\"syscall\":{},\"name\":\"{}\",\"arg0\":{},\"arg1\":{},\"arg2\":{},\"retval\":{},\"error\":{}}}",
1227 e.pc, sys, e.name, e.arg0, e.arg1, e.arg2, ret, e.error
1228 );
1229 }
1230 out.push_str("]}\n");
1231 drop(io::stdout().write_all(out.as_bytes()));
1232 drop(io::stdout().flush());
1233 } else {
1234 let header = format!("trace events: {}\n", events.len());
1235 drop(io::stdout().write_all(header.as_bytes()));
1236 for (i, e) in events.iter().enumerate() {
1237 let ret = e
1238 .retval
1239 .map_or_else(|| "-".to_owned(), |v| format!("{v:#x}"));
1240 let err = if e.error { " err" } else { "" };
1241 let line = format!(
1242 " [{i}] pc={:#x} {} sys={:?} args=({:#x}, {:#x}, {:#x}) ret={ret}{err}\n",
1243 e.pc, e.name, e.syscall, e.arg0, e.arg1, e.arg2
1244 );
1245 drop(io::stdout().write_all(line.as_bytes()));
1246 }
1247 drop(io::stdout().flush());
1248 }
1249}
1250
1251#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
1252#[allow(clippy::too_many_arguments)]
1253fn maybe_push_event(
1254 pc: u64,
1255 sys_no: u32,
1256 name: &'static str,
1257 arg0: u64,
1258 arg1: u64,
1259 arg2: u64,
1260 retval: Option<u64>,
1261 error: bool,
1262) {
1263 let max = MAX_EVENTS.load(Ordering::Relaxed);
1264 if max == 0 {
1266 return;
1267 }
1268 let n = EVENT_COUNT.fetch_add(1, Ordering::Relaxed);
1269 if n >= max {
1270 return;
1271 }
1272 if let Ok(mut guard) = EVENTS.lock()
1273 && u64::try_from(guard.len()).unwrap_or(u64::MAX) < max
1274 {
1275 guard.push(TrapEvent {
1276 pc,
1277 syscall: Some(sys_no),
1278 name: name.to_owned(),
1279 arg0,
1280 arg1,
1281 arg2,
1282 retval,
1283 error,
1284 });
1285 }
1286}
1287
1288#[cfg(test)]
1289mod tests {
1290 use super::*;
1291
1292 #[test]
1293 fn detects_svc_encoding() {
1294 assert!(is_svc(0xD400_0001));
1295 let svc80 = 0xD400_0001 | (0x80_u32 << 5);
1296 assert!(is_svc(svc80));
1297 assert!(!is_svc(0xD503_201F));
1298 assert!(!is_svc(brk_encoding(BRK_TRAP_IMM)));
1299 }
1300
1301 #[test]
1302 fn brk_encoding_roundtrip_imm() {
1303 let enc = brk_encoding(BRK_TRAP_IMM);
1304 let imm = (enc >> 5) & 0xFFFF;
1305 assert_eq!(imm, BRK_TRAP_IMM);
1306 }
1307}