Skip to main content

kh_runtime/
trap.rs

1//! Trap backend: rewrite residual Darwin `svc` and host hypercall entry.
2//!
3//! **Production path:** freestanding libSystem → `kh_hypercall_entry` (alt
4//! stack + NEON tramp + [`kh_trampoline_dispatch`]). No `SIGTRAP` on that path.
5//!
6//! **Residual `svc`:** any leftover Darwin `svc` (fixtures, unpatched stubs)
7//! is rewritten to `brk #IMM` so Linux does not treat it as a host syscall.
8//! The `SIGTRAP` handler translates those sites. Former experimental
9//! `svc`→veneer rewrite (`KAKEHASHI_TRAMPOLINE`) was removed.
10//!
11//! Live handlers require **Linux aarch64**. Patching works on any host.
12#![allow(unsafe_code)]
13
14use std::io::{self, Write};
15use std::sync::Mutex;
16use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering};
17
18use thiserror::Error;
19
20use crate::mem::{
21    MapError, MappedRegion, VM_PROT_EXECUTE, VM_PROT_READ, VM_PROT_WRITE, mprotect_darwin,
22    mprotect_rw,
23};
24use crate::syscall;
25#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
26use crate::syscall::SyscallArgs;
27
28/// Marker immediate for patched traps (`brk #0x0F00`).
29pub const BRK_TRAP_IMM: u32 = 0x0F00;
30
31/// AArch64 `svc #imm16` base encoding: `0xD4000001 | (imm16 << 5)`.
32const SVC_MASK: u32 = 0xFFE0_001F;
33const SVC_BASE: u32 = 0xD400_0001;
34
35/// AArch64 `brk #imm16`: `0xD4200000 | (imm16 << 5)`.
36const fn brk_encoding(imm16: u32) -> u32 {
37    0xD420_0000 | ((imm16 & 0xFFFF) << 5)
38}
39
40/// AArch64 `PSTATE` / CPSR carry flag (bit 29 of NZCV).
41///
42/// Darwin arm64 syscall convention: set on error (errno in `x0`), clear on success.
43pub const PSTATE_C: u64 = 1 << 29;
44
45/// One observed trap / translated syscall event.
46#[derive(Debug, Clone, PartialEq, Eq)]
47pub struct TrapEvent {
48    /// Guest PC at the trap (instruction address).
49    pub pc: u64,
50    /// Darwin BSD syscall number (`x16`) when recognized; otherwise `None`.
51    pub syscall: Option<u32>,
52    /// Short label (`exit`, `write`, `unknown`, …).
53    pub name: String,
54    /// First argument (`x0`) when applicable.
55    pub arg0: u64,
56    /// Second argument (`x1`) when applicable.
57    pub arg1: u64,
58    /// Third argument (`x2`) when applicable.
59    pub arg2: u64,
60    /// Return value written to `x0`, if any.
61    pub retval: Option<u64>,
62    /// Darwin error path: carry set, `retval` is positive errno.
63    pub error: bool,
64}
65
66/// Outcome of handling a trap.
67#[derive(Debug, Clone, Copy, PartialEq, Eq)]
68pub enum TrapOutcome {
69    /// Guest called BSD `exit` (or the host forced process termination).
70    Exit {
71        /// Process exit code.
72        code: i32,
73    },
74    /// Guest should resume after the trap instruction.
75    Continue,
76    /// Guest called `bsdthread_terminate` (or equivalent): end this host thread only.
77    ThreadExit,
78}
79
80/// Configuration for the trap backend.
81#[derive(Debug, Clone)]
82pub struct TrapConfig {
83    /// Maximum number of events to retain for `kh trace`.
84    pub max_events: usize,
85    /// Maximum syscalls before forced exit (safety cap).
86    pub max_syscalls: usize,
87}
88
89impl Default for TrapConfig {
90    fn default() -> Self {
91        Self {
92            max_events: 256,
93            max_syscalls: 256,
94        }
95    }
96}
97
98/// Trap subsystem errors.
99#[derive(Debug, Error)]
100pub enum TrapError {
101    /// Failed to install signal handlers.
102    #[error("failed to install trap signal handler: {0}")]
103    SignalSetup(#[source] io::Error),
104
105    /// Memory protect / patch failure.
106    #[error(transparent)]
107    Map(#[from] MapError),
108
109    /// Architecture / OS is not supported for live traps.
110    #[error("trap backend requires Linux aarch64 for live execution")]
111    UnsupportedArch,
112}
113
114#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
115static HANDLER_INSTALLED: AtomicBool = AtomicBool::new(false);
116static EXIT_CODE: AtomicI32 = AtomicI32::new(0);
117static EXIT_REQUESTED: AtomicBool = AtomicBool::new(false);
118static EVENT_COUNT: AtomicU64 = AtomicU64::new(0);
119static MAX_EVENTS: AtomicU64 = AtomicU64::new(256);
120static TRACE_ON_EXIT: AtomicBool = AtomicBool::new(false);
121static TRACE_JSON: AtomicBool = AtomicBool::new(false);
122/// When set (not `i32::MIN`): if guest exit code differs, process exits 1.
123static EXPECT_CODE: AtomicI32 = AtomicI32::new(i32::MIN);
124static EVENTS: Mutex<Vec<TrapEvent>> = Mutex::new(Vec::new());
125
126/// Patches every AArch64 `svc` in executable regions to `brk #IMM`.
127///
128/// Residual sites only: freestanding libSystem uses hypercall, not `svc`.
129pub fn patch_svc_to_brk(regions: &mut [MappedRegion]) -> Result<usize, TrapError> {
130    let mut patched = 0_usize;
131    for region in regions.iter_mut() {
132        if region.prot & VM_PROT_EXECUTE == 0 {
133            continue;
134        }
135        mprotect_rw(region)?;
136        let bytes = region.host_bytes_mut();
137        let mut off = 0_usize;
138        while off.saturating_add(4) <= bytes.len() {
139            let end = off.saturating_add(4);
140            let Some(word) = bytes.get(off..end) else {
141                break;
142            };
143            let Ok(word_bytes) = <[u8; 4]>::try_from(word) else {
144                break;
145            };
146            let insn = u32::from_le_bytes(word_bytes);
147            if is_svc(insn) {
148                let brk = brk_encoding(BRK_TRAP_IMM).to_le_bytes();
149                if let Some(slot) = bytes.get_mut(off..end) {
150                    slot.copy_from_slice(&brk);
151                    patched = patched.saturating_add(1);
152                }
153            }
154            off = end;
155        }
156        let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
157        mprotect_darwin(region, restore)?;
158        region.prot = restore;
159    }
160    Ok(patched)
161}
162
163const fn is_svc(insn: u32) -> bool {
164    (insn & SVC_MASK) == SVC_BASE
165}
166
167/// Round `len` up to a multiple of `page` (page must be a power of two ≥ 1).
168#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
169#[inline]
170fn align_up_to_page(len: usize, page: usize) -> usize {
171    if page <= 1 {
172        return len.max(1);
173    }
174    let mask = page.saturating_sub(1);
175    len.saturating_add(mask) & !mask
176}
177
178// libgcc / compiler-rt symbol linked on Linux aarch64 toolchains.
179#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
180unsafe extern "C" {
181    fn __clear_cache(start: *mut libc::c_void, end: *mut libc::c_void);
182}
183
184/// Flush D/I caches after writing executable code (aarch64 requirement).
185#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
186fn clear_icache(ptr: *mut u8, len: usize) {
187    if ptr.is_null() || len == 0 {
188        return;
189    }
190    // SAFETY: range is a live mapping the caller owns and just wrote.
191    unsafe {
192        let start = ptr.cast::<libc::c_void>();
193        let end = ptr.wrapping_add(len).cast::<libc::c_void>();
194        __clear_cache(start, end);
195    }
196}
197
198#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
199#[allow(clippy::as_conversions, function_casts_as_integer)]
200fn trampoline_rust_entry_addr() -> u64 {
201    u64::try_from(kh_trampoline_dispatch as usize).unwrap_or(0)
202}
203
204/// Return value of the trampoline dispatch (x0=retval, x1=error flag).
205#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
206#[repr(C)]
207pub struct TrampRet {
208    /// Value for guest `x0`.
209    pub retval: u64,
210    /// Non-zero → set Darwin carry (error path).
211    pub error: u64,
212}
213
214/// Called from the hypercall NEON tramp (`TRAMP_BYTES`) on the host alt stack.
215///
216/// # Safety
217///
218/// Caller must preserve guest SIMD/GPRs around the call (hypercall prolog +
219/// mapped NEON tramp). Arguments are Darwin BSD syscall registers; `x16`
220/// holds the syscall number (or AAPCS 8th arg after the hypercall prologue).
221/// May end the current host worker thread on `bsdthread_terminate`
222/// (`SYS_exit`, not `pthread_exit`), or `_exit` the process on guest `exit`.
223///
224/// Guest entry is [`kh_hypercall_entry`]: alt stack + TLS boundary before this.
225#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
226#[unsafe(no_mangle)]
227pub unsafe extern "C" fn kh_trampoline_dispatch(
228    x0: u64,
229    x1: u64,
230    x2: u64,
231    x3: u64,
232    x4: u64,
233    x5: u64,
234    x6: u64,
235    x16: u64,
236) -> TrampRet {
237    let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
238    let result = syscall::dispatch(SyscallArgs {
239        pc: 0,
240        number: sys_no,
241        x0,
242        x1,
243        x2,
244        x3,
245        x4,
246        x5,
247        x6,
248    });
249    if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
250        maybe_push_event(
251            0,
252            sys_no,
253            result.name,
254            x0,
255            x1,
256            x2,
257            result.retval,
258            result.error,
259        );
260    }
261    match result.outcome {
262        TrapOutcome::Exit { code } => finish_with_exit_code(code),
263        TrapOutcome::ThreadExit => {
264            // End this host worker only (same as bsdthread_terminate).
265            crate::thread::exit_current_guest_worker();
266        }
267        TrapOutcome::Continue => {}
268    }
269    TrampRet {
270        retval: result.retval.unwrap_or(0),
271        error: u64::from(result.error),
272    }
273}
274
275// Freestanding hypercall entry (`kh_hypercall_entry`):
276// Prefer host alt stack (so join/terminate never runs host Rust on a guest
277// stack that `pthread_join` may unmap).
278//
279// **NEON:** Darwin `svc` preserves Q0–Q31. We must too. Any `bl` into Rust/C
280// (TLS enter, alt-stack map) clobbers caller-saved SIMD under AAPCS64, so full
281// Q0–Q31 + FPCR/FPSR are saved on the *guest* prolog frame *before* any `bl`
282// (invariant 13). Dispatch goes through mapped `TRAMP_BYTES` (second full NEON
283// frame on the host/alt stack). A former "light" skip of the second save
284// showed no wall win on UTM 8k and was removed to keep one code path.
285//
286// ABI: x0–x6 + number in x7 → TrampRet in x0/x1.
287// Guest frame layout (640 B):
288//   [0]   x29,x30
289//   [16]  x0,x1  [32] x2,x3  [48] x4,x5  [64] x6,x7
290//   [80]  pad
291//   [96]  q0..q31 (512 B)
292//   [608] fpcr,fpsr
293#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
294std::arch::global_asm!(
295    r#"
296    .text
297    .align 2
298    .global kh_hypercall_entry
299    .type kh_hypercall_entry, %function
300kh_hypercall_entry:
301    // Guest prolog: GPRs + full NEON *before* any bl (NEON must survive host TLS).
302    sub sp, sp, #640
303    stp x29, x30, [sp, #0]
304    stp x0, x1, [sp, #16]
305    stp x2, x3, [sp, #32]
306    stp x4, x5, [sp, #48]
307    stp x6, x7, [sp, #64]
308    mov x29, sp
309    stp q0, q1, [sp, #96]
310    stp q2, q3, [sp, #128]
311    stp q4, q5, [sp, #160]
312    stp q6, q7, [sp, #192]
313    stp q8, q9, [sp, #224]
314    stp q10, q11, [sp, #256]
315    stp q12, q13, [sp, #288]
316    stp q14, q15, [sp, #320]
317    stp q16, q17, [sp, #352]
318    stp q18, q19, [sp, #384]
319    stp q20, q21, [sp, #416]
320    stp q22, q23, [sp, #448]
321    stp q24, q25, [sp, #480]
322    stp q26, q27, [sp, #512]
323    stp q28, q29, [sp, #544]
324    stp q30, q31, [sp, #576]
325    mrs x9, fpcr
326    mrs x10, fpsr
327    // #608 exceeds signed LDP imm7*8 range (±512); use base+add.
328    add x11, sp, #608
329    stp x9, x10, [x11]
330
331    // A1/A2: enter returns alt_top in x0, guest_tpidr in x1 (HyperEnterRet).
332    // Park guest_va at prolog+80 (hole before NEON) so leave needs no gettid.
333    bl kh_tls_enter_host
334    mov x9, sp                 // guest prolog frame
335    str x1, [x9, #80]          // guest_tpidr for leave
336    cbz x0, 1f
337    mov sp, x0                 // host alt stack
338    ldr x30, [x9, #8]          // reload LR → freestanding thin caller
339    // Host frame: [guest_frame, lr]
340    stp x9, x30, [sp, #-16]!
341    // Reload guest frame ptr / args (enter_host clobbered temps).
342    ldr x9, [sp]
343    ldp x0, x1, [x9, #16]
344    ldp x2, x3, [x9, #32]
345    ldp x4, x5, [x9, #48]
346    ldp x6, x7, [x9, #64]
347    mov x16, x7
348    bl kh_neon_tramp_entry
349    // Preserve TrampRet (x0/x1) across TLS restore.
350    stp x0, x1, [sp, #-16]!
351    // guest_va at host_frame.guest_frame + 80; host_frame at sp+16.
352    ldr x9, [sp, #16]
353    ldr x0, [x9, #80]
354    bl kh_tls_leave_host
355    ldp x0, x1, [sp], #16
356    ldp x9, x30, [sp], #16
357    mov sp, x9
358    // Restore guest NEON from prolog (pre-bl snapshot).
359    ldp q0, q1, [sp, #96]
360    ldp q2, q3, [sp, #128]
361    ldp q4, q5, [sp, #160]
362    ldp q6, q7, [sp, #192]
363    ldp q8, q9, [sp, #224]
364    ldp q10, q11, [sp, #256]
365    ldp q12, q13, [sp, #288]
366    ldp q14, q15, [sp, #320]
367    ldp q16, q17, [sp, #352]
368    ldp q18, q19, [sp, #384]
369    ldp q20, q21, [sp, #416]
370    ldp q22, q23, [sp, #448]
371    ldp q24, q25, [sp, #480]
372    ldp q26, q27, [sp, #512]
373    ldp q28, q29, [sp, #544]
374    ldp q30, q31, [sp, #576]
375    add x11, sp, #608
376    ldp x9, x10, [x11]
377    msr fpcr, x9
378    msr fpsr, x10
379    ldr x29, [sp, #0]
380    add sp, sp, #640
381    ret
3821:
383    // No alt stack: dispatch on the guest stack (ST / fallback only).
384    // Already on host TLS from the enter above; guest_va at [sp, #80].
385    ldp x0, x1, [sp, #16]
386    ldp x2, x3, [sp, #32]
387    ldp x4, x5, [sp, #48]
388    ldp x6, x7, [sp, #64]
389    mov x16, x7
390    // Preserve real LR across the call (still on guest prolog frame).
391    ldr x30, [sp, #8]
392    bl kh_neon_tramp_entry
393    stp x0, x1, [sp, #-16]!
394    ldr x0, [sp, #16+80]
395    bl kh_tls_leave_host
396    ldp x0, x1, [sp], #16
397    // Restore guest NEON from prolog.
398    ldp q0, q1, [sp, #96]
399    ldp q2, q3, [sp, #128]
400    ldp q4, q5, [sp, #160]
401    ldp q6, q7, [sp, #192]
402    ldp q8, q9, [sp, #224]
403    ldp q10, q11, [sp, #256]
404    ldp q12, q13, [sp, #288]
405    ldp q14, q15, [sp, #320]
406    ldp q16, q17, [sp, #352]
407    ldp q18, q19, [sp, #384]
408    ldp q20, q21, [sp, #416]
409    ldp q22, q23, [sp, #448]
410    ldp q24, q25, [sp, #480]
411    ldp q26, q27, [sp, #512]
412    ldp q28, q29, [sp, #544]
413    ldp q30, q31, [sp, #576]
414    add x11, sp, #608
415    ldp x9, x10, [x11]
416    msr fpcr, x9
417    msr fpsr, x10
418    ldp x29, x30, [sp, #0]
419    add sp, sp, #640
420    ret
421    .size kh_hypercall_entry, .-kh_hypercall_entry
422    "#
423);
424
425// `kh_neon_tramp_entry` + `KH_NEON_TRAMP_VA`: branch into mapped `TRAMP_BYTES`
426// (full Q* save around Rust dispatch). Fallback if tramp not mapped yet:
427// tail-call `kh_trampoline_dispatch` (still safe: prolog owns guest Q*).
428#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
429std::arch::global_asm!(
430    r#"
431    .text
432    .align 2
433    .global kh_neon_tramp_entry
434    .type kh_neon_tramp_entry, %function
435kh_neon_tramp_entry:
436    adrp x17, KH_NEON_TRAMP_VA
437    ldr x17, [x17, :lo12:KH_NEON_TRAMP_VA]
438    cbz x17, 1f
439    br x17
4401:
441    // No-tramp fallback: number in x16 → AAPCS 8th arg x7; tail call.
442    mov x7, x16
443    b kh_trampoline_dispatch
444    .size kh_neon_tramp_entry, .-kh_neon_tramp_entry
445
446    .data
447    .align 3
448    .global KH_NEON_TRAMP_VA
449KH_NEON_TRAMP_VA:
450    .quad 0
451    "#
452);
453
454#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
455unsafe extern "C" {
456    fn kh_hypercall_entry();
457    static mut KH_NEON_TRAMP_VA: u64;
458}
459
460/// Address of freestanding hypercall entry (alt stack + NEON tramp + dispatch).
461#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
462#[must_use]
463pub fn hypercall_entry_addr() -> u64 {
464    ensure_neon_tramp();
465    #[allow(clippy::as_conversions, function_casts_as_integer)]
466    {
467        u64::try_from(kh_hypercall_entry as usize).unwrap_or(0)
468    }
469}
470
471/// Map TRAMP_BYTES once and publish its VA to `KH_NEON_TRAMP_VA`.
472#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
473fn ensure_neon_tramp() {
474    static ONCE: AtomicU64 = AtomicU64::new(0);
475    static LOCK: Mutex<()> = Mutex::new(());
476
477    if ONCE.load(Ordering::Acquire) != 0 {
478        return;
479    }
480    let _g = LOCK
481        .lock()
482        .unwrap_or_else(std::sync::PoisonError::into_inner);
483    if ONCE.load(Ordering::Acquire) != 0 {
484        return;
485    }
486
487    let code = TRAMP_BYTES;
488    let page = crate::host::page_size().unwrap_or(4096);
489    let map_len = align_up_to_page(code.len(), page).max(page);
490    let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
491    let prot_rw = libc::PROT_READ | libc::PROT_WRITE;
492    let Some(ptr) = crate::host::mmap(None, map_len, prot_rw, flags, -1, 0) else {
493        tracing::error!("NEON tramp mmap failed");
494        return;
495    };
496    // SAFETY: fresh RW mapping.
497    let slice = unsafe { std::slice::from_raw_parts_mut(ptr, map_len) };
498    slice.fill(0);
499    if let Some(dst) = slice.get_mut(..code.len()) {
500        dst.copy_from_slice(code);
501    } else {
502        let _ = crate::host::munmap(ptr, map_len);
503        return;
504    }
505    if let Some(slot) = slice.get_mut(TRAMP_RELOC_OFF..TRAMP_RELOC_OFF.saturating_add(8)) {
506        slot.copy_from_slice(&trampoline_rust_entry_addr().to_le_bytes());
507    } else {
508        let _ = crate::host::munmap(ptr, map_len);
509        return;
510    }
511    let prot_rx = libc::PROT_READ | libc::PROT_EXEC;
512    if !crate::host::mprotect(ptr, map_len, prot_rx) {
513        tracing::error!("NEON tramp mprotect RX failed");
514        let _ = crate::host::munmap(ptr, map_len);
515        return;
516    }
517    clear_icache(ptr, map_len);
518    let va = crate::host::ptr_addr_u64(ptr);
519    // SAFETY: symbol from global_asm; written once under LOCK before guest entry.
520    unsafe {
521        KH_NEON_TRAMP_VA = va;
522    }
523    ONCE.store(va, Ordering::Release);
524    tracing::info!(
525        va = format_args!("{va:#x}"),
526        "mapped NEON tramp for hypercall"
527    );
528}
529
530#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
531#[must_use]
532pub fn hypercall_entry_addr() -> u64 {
533    0
534}
535
536// ── trampoline machine code (verified via rustc/objdump aarch64) ─────────────
537
538/// Offset of the 8-byte absolute `kh_trampoline_dispatch` pointer.
539///
540/// Layout (see `TRAMP_BYTES`): `bti c` + save GPRs/NEON/FPCR + `blr` dispatch
541/// + restore + `ret` + `.quad` reloc @ 0x100.
542#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
543const TRAMP_RELOC_OFF: usize = 0x100;
544
545/// Exact opcodes from assembled trampoline (`as` / objdump on Linux aarch64).
546///
547/// Saves **all Q0–Q31 + FPCR/FPSR** around the Rust call: Darwin `svc` preserves
548/// SIMD state; compression workers (7zz MT) rely on that. A bare GPR-only
549/// trampoline clobbered NEON and faulted under `-mmt>1`.
550#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
551#[rustfmt::skip]
552const TRAMP_BYTES: &[u8] = &[
553    0x5f, 0x24, 0x03, 0xd5, 0xff, 0x03, 0x0a, 0xd1, 0xe8, 0x27, 0x00, 0xa9, 0xea, 0x2f, 0x01, 0xa9,
554    0xec, 0x37, 0x02, 0xa9, 0xee, 0x3f, 0x03, 0xa9, 0xf1, 0x4b, 0x04, 0xa9, 0xfe, 0x2b, 0x00, 0xf9,
555    0x08, 0x44, 0x3b, 0xd5, 0x29, 0x44, 0x3b, 0xd5, 0xe8, 0x27, 0x06, 0xa9, 0xe0, 0x87, 0x03, 0xad,
556    0xe2, 0x8f, 0x04, 0xad, 0xe4, 0x97, 0x05, 0xad, 0xe6, 0x9f, 0x06, 0xad, 0xe8, 0xa7, 0x07, 0xad,
557    0xea, 0xaf, 0x08, 0xad, 0xec, 0xb7, 0x09, 0xad, 0xee, 0xbf, 0x0a, 0xad, 0xf0, 0xc7, 0x0b, 0xad,
558    0xf2, 0xcf, 0x0c, 0xad, 0xf4, 0xd7, 0x0d, 0xad, 0xf6, 0xdf, 0x0e, 0xad, 0xf8, 0xe7, 0x0f, 0xad,
559    0xfa, 0xef, 0x10, 0xad, 0xfc, 0xf7, 0x11, 0xad, 0xfe, 0xff, 0x12, 0xad, 0xe7, 0x03, 0x10, 0xaa,
560    0x88, 0x04, 0x00, 0x58, 0x00, 0x01, 0x3f, 0xd6, 0x02, 0x42, 0x3b, 0xd5, 0x23, 0x00, 0x80, 0x52,
561    0x63, 0x88, 0x63, 0xd3, 0x42, 0x00, 0x23, 0x8a, 0x41, 0x00, 0x00, 0xb4, 0x42, 0x00, 0x03, 0xaa,
562    0x02, 0x42, 0x1b, 0xd5, 0xe8, 0x27, 0x46, 0xa9, 0x08, 0x44, 0x1b, 0xd5, 0x29, 0x44, 0x1b, 0xd5,
563    0xe0, 0x87, 0x43, 0xad, 0xe2, 0x8f, 0x44, 0xad, 0xe4, 0x97, 0x45, 0xad, 0xe6, 0x9f, 0x46, 0xad,
564    0xe8, 0xa7, 0x47, 0xad, 0xea, 0xaf, 0x48, 0xad, 0xec, 0xb7, 0x49, 0xad, 0xee, 0xbf, 0x4a, 0xad,
565    0xf0, 0xc7, 0x4b, 0xad, 0xf2, 0xcf, 0x4c, 0xad, 0xf4, 0xd7, 0x4d, 0xad, 0xf6, 0xdf, 0x4e, 0xad,
566    0xf8, 0xe7, 0x4f, 0xad, 0xfa, 0xef, 0x50, 0xad, 0xfc, 0xf7, 0x51, 0xad, 0xfe, 0xff, 0x52, 0xad,
567    0xfe, 0x2b, 0x40, 0xf9, 0xf1, 0x4b, 0x44, 0xa9, 0xee, 0x3f, 0x43, 0xa9, 0xec, 0x37, 0x42, 0xa9,
568    0xea, 0x2f, 0x41, 0xa9, 0xe8, 0x27, 0x40, 0xa9, 0xff, 0x03, 0x0a, 0x91, 0xc0, 0x03, 0x5f, 0xd6,
569    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // reloc .quad @ 0x100
570];
571/// Installs the `SIGTRAP` handler used by the micro execution backend.
572pub fn install_trap_handlers(config: &TrapConfig) -> Result<(), TrapError> {
573    MAX_EVENTS.store(
574        u64::try_from(config.max_events).unwrap_or(256),
575        Ordering::SeqCst,
576    );
577    syscall::reset_syscall_state(config.max_syscalls);
578
579    #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
580    {
581        let _ = config;
582        Err(TrapError::UnsupportedArch)
583    }
584
585    #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
586    {
587        if HANDLER_INSTALLED.swap(true, Ordering::SeqCst) {
588            reset_trap_state();
589            syscall::reset_syscall_state(config.max_syscalls);
590            return Ok(());
591        }
592
593        // Guest faults used to leave multi‑MB `core` files in the workspace cwd.
594        // Faults go through our handler → `_exit`; disable dumps as belt-and-braces.
595        disable_core_dumps();
596
597        // SAFETY: process-wide SIGTRAP / fault handlers with SA_SIGINFO.
598        unsafe {
599            let mut sa: libc::sigaction = std::mem::zeroed();
600            sa.sa_flags = libc::SA_SIGINFO;
601            #[allow(clippy::as_conversions, function_casts_as_integer)]
602            {
603                sa.sa_sigaction = trap_sigaction as *const () as usize;
604            }
605            libc::sigemptyset(std::ptr::addr_of_mut!(sa.sa_mask));
606            if libc::sigaction(libc::SIGTRAP, std::ptr::addr_of!(sa), std::ptr::null_mut()) != 0 {
607                HANDLER_INSTALLED.store(false, Ordering::SeqCst);
608                return Err(TrapError::SignalSetup(io::Error::last_os_error()));
609            }
610
611            // Guest faults: print PC/addr so `kh run` diagnoses unbound GOT / bad heap.
612            let mut fault: libc::sigaction = std::mem::zeroed();
613            fault.sa_flags = libc::SA_SIGINFO;
614            #[allow(clippy::as_conversions, function_casts_as_integer)]
615            {
616                fault.sa_sigaction = guest_fault_sigaction as *const () as usize;
617            }
618            libc::sigemptyset(std::ptr::addr_of_mut!(fault.sa_mask));
619            for sig in [libc::SIGSEGV, libc::SIGBUS] {
620                if libc::sigaction(sig, std::ptr::addr_of!(fault), std::ptr::null_mut()) != 0 {
621                    HANDLER_INSTALLED.store(false, Ordering::SeqCst);
622                    return Err(TrapError::SignalSetup(io::Error::last_os_error()));
623                }
624            }
625        }
626        reset_trap_state();
627        Ok(())
628    }
629}
630
631/// Sets `RLIMIT_CORE` soft+hard to 0 so guest crashes never litter the repo with
632/// `core` dumps (we already print a guest fault summary and `_exit`).
633#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
634fn disable_core_dumps() {
635    // SAFETY: process-wide rlimit; best-effort, ignore failure.
636    unsafe {
637        let lim = libc::rlimit {
638            rlim_cur: 0,
639            rlim_max: 0,
640        };
641        let _ = libc::setrlimit(libc::RLIMIT_CORE, std::ptr::addr_of!(lim));
642    }
643}
644
645/// Logs guest `SIGSEGV` / `SIGBUS` (PC, fault address, key regs) then exits.
646#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
647unsafe extern "C" fn guest_fault_sigaction(
648    signo: libc::c_int,
649    info: *mut libc::siginfo_t,
650    ctx: *mut libc::c_void,
651) {
652    // May run under guest TPIDR; restore host before formatting / write.
653    crate::tls::enter_host_tls();
654    let name = match signo {
655        libc::SIGSEGV => "SIGSEGV",
656        libc::SIGBUS => "SIGBUS",
657        _ => "FAULT",
658    };
659    let mut pc = 0_u64;
660    let mut sp = 0_u64;
661    let mut x0 = 0_u64;
662    let mut x1 = 0_u64;
663    let mut x8 = 0_u64;
664    let mut x16 = 0_u64;
665    let mut lr = 0_u64;
666    if !ctx.is_null() {
667        // SAFETY: kernel `ucontext_t` for SA_SIGINFO.
668        let uctx = unsafe { &*ctx.cast::<libc::ucontext_t>() };
669        let m = &uctx.uc_mcontext;
670        pc = m.pc;
671        sp = m.sp;
672        x0 = m.regs[0];
673        x1 = m.regs[1];
674        x8 = m.regs[8];
675        x16 = m.regs[16];
676        lr = m.regs[30];
677    }
678    let addr = if info.is_null() {
679        0_u64
680    } else {
681        // SAFETY: kernel-provided siginfo.
682        let p = unsafe { (*info).si_addr() };
683        u64::try_from(p.addr()).unwrap_or(0)
684    };
685    let msg = format!(
686        "error: guest {name} pc={pc:#x} addr={addr:#x} sp={sp:#x} lr={lr:#x} \
687         x0={x0:#x} x1={x1:#x} x8={x8:#x} x16={x16:#x}\n"
688    );
689    drop(io::stderr().write_all(msg.as_bytes()));
690    // Best-effort: which mapping owns PC / fault address (helps MT diagnosis).
691    if let Ok(maps) = std::fs::read_to_string("/proc/self/maps") {
692        for &va in &[pc, addr, lr, x16] {
693            for line in maps.lines() {
694                let Some((range, rest)) = line.split_once(' ') else {
695                    continue;
696                };
697                let Some((a, b)) = range.split_once('-') else {
698                    continue;
699                };
700                let (Ok(lo), Ok(hi)) = (u64::from_str_radix(a, 16), u64::from_str_radix(b, 16))
701                else {
702                    continue;
703                };
704                if va >= lo && va < hi {
705                    let off = va.saturating_sub(lo);
706                    let m = format!("  map {va:#x} = {lo:#x}+{off:#x} {rest}\n");
707                    drop(io::stderr().write_all(m.as_bytes()));
708                    break;
709                }
710            }
711        }
712    }
713    drop(io::stderr().flush());
714    if TRACE_ON_EXIT.load(Ordering::SeqCst) {
715        dump_events_before_exit();
716    }
717    // SAFETY: hard-stop after unrecoverable guest fault.
718    unsafe { libc::_exit(128_i32.saturating_add(signo)) };
719}
720
721/// Drains recorded trap events (for `kh trace`).
722#[must_use]
723pub fn take_trace_events() -> Vec<TrapEvent> {
724    EVENTS
725        .lock()
726        .map(|mut g| std::mem::take(&mut *g))
727        .unwrap_or_default()
728}
729
730/// Whether the guest requested exit.
731#[must_use]
732pub fn exit_requested() -> bool {
733    EXIT_REQUESTED.load(Ordering::SeqCst)
734}
735
736/// Exit code from the last guest `exit` trap.
737#[must_use]
738pub fn take_exit_code() -> Option<i32> {
739    if EXIT_REQUESTED.load(Ordering::SeqCst) {
740        Some(EXIT_CODE.load(Ordering::SeqCst))
741    } else {
742        None
743    }
744}
745
746/// Resets exit / event state before a new run.
747pub fn reset_trap_state() {
748    EXIT_REQUESTED.store(false, Ordering::SeqCst);
749    EXIT_CODE.store(0, Ordering::SeqCst);
750    EVENT_COUNT.store(0, Ordering::SeqCst);
751    if let Ok(mut guard) = EVENTS.lock() {
752        guard.clear();
753    }
754}
755
756/// When set, the exit trap prints recorded events to stdout before `_exit`.
757pub fn set_trace_on_exit(json: bool) {
758    TRACE_ON_EXIT.store(true, Ordering::SeqCst);
759    TRACE_JSON.store(json, Ordering::SeqCst);
760}
761
762/// Clears the trace-on-exit flag (default for plain `kh run`).
763pub fn clear_trace_on_exit() {
764    TRACE_ON_EXIT.store(false, Ordering::SeqCst);
765    TRACE_JSON.store(false, Ordering::SeqCst);
766}
767
768/// Sets the expected guest exit code for the micro gate (`--expect-code`).
769pub fn set_expect_code(code: i32) {
770    EXPECT_CODE.store(code, Ordering::SeqCst);
771}
772
773/// Disables the expect-code gate.
774pub fn clear_expect_code() {
775    EXPECT_CODE.store(i32::MIN, Ordering::SeqCst);
776}
777
778/// Terminates the host process as if the guest called BSD `exit(code)`.
779///
780/// Used after `LC_MAIN` returns (dyld calls `exit` with `main`'s return value).
781/// Honours `--expect-code` and optional trace-on-exit dump.
782pub fn finish_with_exit_code(code: i32) -> ! {
783    EXIT_CODE.store(code, Ordering::SeqCst);
784    EXIT_REQUESTED.store(true, Ordering::SeqCst);
785    if TRACE_ON_EXIT.load(Ordering::SeqCst) {
786        dump_events_before_exit();
787    }
788    // Host TPIDR: opt-in F1 diagnosis (park expected buckets / wake efficiency).
789    crate::syscall::dump_futex_stats_if_enabled();
790    let expect = EXPECT_CODE.load(Ordering::SeqCst);
791    let status = if expect == i32::MIN || expect == code {
792        code
793    } else {
794        let msg = format!("error: guest exit code {code} != expected {expect}\n");
795        drop(io::stderr().write_all(msg.as_bytes()));
796        1
797    };
798    // SAFETY: intentional process termination after guest main return / exit.
799    unsafe { libc::_exit(status) };
800}
801
802#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
803unsafe extern "C" fn trap_sigaction(
804    signo: libc::c_int,
805    _info: *mut libc::siginfo_t,
806    ctx: *mut libc::c_void,
807) {
808    if signo != libc::SIGTRAP || ctx.is_null() {
809        return;
810    }
811
812    // Guest may have TPIDR_EL0 = guest TSD; restore host TLS before Rust/libc.
813    crate::tls::enter_host_tls();
814
815    // SAFETY: kernel provides a valid `ucontext_t` for SA_SIGINFO deliveries.
816    let uctx = unsafe { &mut *ctx.cast::<libc::ucontext_t>() };
817    let m = &mut uctx.uc_mcontext;
818    let pc = m.pc;
819    // Only pull the regs dispatch needs; avoid extra loads on the hot path.
820    let x0 = m.regs[0];
821    let x1 = m.regs[1];
822    let x2 = m.regs[2];
823    let x3 = m.regs[3];
824    let x4 = m.regs[4];
825    let x5 = m.regs[5];
826    let x6 = m.regs[6];
827    let x16 = m.regs[16];
828
829    let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
830    let result = syscall::dispatch(SyscallArgs {
831        pc,
832        number: sys_no,
833        x0,
834        x1,
835        x2,
836        x3,
837        x4,
838        x5,
839        x6,
840    });
841
842    // Zero-cost when max_events == 0 (plain `kh run`).
843    if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
844        maybe_push_event(
845            pc,
846            sys_no,
847            result.name,
848            x0,
849            x1,
850            x2,
851            result.retval,
852            result.error,
853        );
854    }
855
856    if let Some(ret) = result.retval {
857        m.regs[0] = ret;
858    }
859    // Darwin arm64: error → set C; success with write-back → clear C.
860    if result.error {
861        m.pstate |= PSTATE_C;
862    } else if result.retval.is_some() {
863        m.pstate &= !PSTATE_C;
864    }
865
866    match result.outcome {
867        TrapOutcome::Exit { code } => {
868            finish_with_exit_code(code);
869        }
870        TrapOutcome::Continue => {
871            m.pc = pc.wrapping_add(4);
872            // Return to guest with guest TPIDR restored.
873            crate::tls::leave_host_tls();
874        }
875        TrapOutcome::ThreadExit => {
876            // Stay on host TLS; exit trampoline publishes join then pthread_exit.
877            // Redirect ucontext to a host `pthread_exit` trampoline on this
878            // thread's original stack (see `crate::thread`). Main thread
879            // (no host frame) falls back to process exit.
880            if !crate::thread::redirect_ucontext_to_host_exit(m) {
881                finish_with_exit_code(0);
882            }
883        }
884    }
885}
886
887fn dump_events_before_exit() {
888    use std::fmt::Write as _;
889
890    let json = TRACE_JSON.load(Ordering::SeqCst);
891    let events = EVENTS.lock().map(|g| g.clone()).unwrap_or_default();
892    if json {
893        let mut out = String::from("{\"events\":[");
894        for (i, e) in events.iter().enumerate() {
895            if i > 0 {
896                out.push(',');
897            }
898            let sys = e
899                .syscall
900                .map_or_else(|| "null".to_owned(), |s| s.to_string());
901            let ret = e
902                .retval
903                .map_or_else(|| "null".to_owned(), |v| v.to_string());
904            let _ = write!(
905                out,
906                "{{\"pc\":{},\"syscall\":{},\"name\":\"{}\",\"arg0\":{},\"arg1\":{},\"arg2\":{},\"retval\":{},\"error\":{}}}",
907                e.pc, sys, e.name, e.arg0, e.arg1, e.arg2, ret, e.error
908            );
909        }
910        out.push_str("]}\n");
911        drop(io::stdout().write_all(out.as_bytes()));
912        drop(io::stdout().flush());
913    } else {
914        let header = format!("trace events: {}\n", events.len());
915        drop(io::stdout().write_all(header.as_bytes()));
916        for (i, e) in events.iter().enumerate() {
917            let ret = e
918                .retval
919                .map_or_else(|| "-".to_owned(), |v| format!("{v:#x}"));
920            let err = if e.error { " err" } else { "" };
921            let line = format!(
922                "  [{i}] pc={:#x} {} sys={:?} args=({:#x}, {:#x}, {:#x}) ret={ret}{err}\n",
923                e.pc, e.name, e.syscall, e.arg0, e.arg1, e.arg2
924            );
925            drop(io::stdout().write_all(line.as_bytes()));
926        }
927        drop(io::stdout().flush());
928    }
929}
930
931#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
932#[allow(clippy::too_many_arguments)]
933fn maybe_push_event(
934    pc: u64,
935    sys_no: u32,
936    name: &'static str,
937    arg0: u64,
938    arg1: u64,
939    arg2: u64,
940    retval: Option<u64>,
941    error: bool,
942) {
943    let max = MAX_EVENTS.load(Ordering::Relaxed);
944    // `kh run` sets max_events=0: single load, no String/Mutex.
945    if max == 0 {
946        return;
947    }
948    let n = EVENT_COUNT.fetch_add(1, Ordering::Relaxed);
949    if n >= max {
950        return;
951    }
952    if let Ok(mut guard) = EVENTS.lock()
953        && u64::try_from(guard.len()).unwrap_or(u64::MAX) < max
954    {
955        guard.push(TrapEvent {
956            pc,
957            syscall: Some(sys_no),
958            name: name.to_owned(),
959            arg0,
960            arg1,
961            arg2,
962            retval,
963            error,
964        });
965    }
966}
967
968#[cfg(test)]
969mod tests {
970    use super::*;
971
972    #[test]
973    fn detects_svc_encoding() {
974        assert!(is_svc(0xD400_0001));
975        let svc80 = 0xD400_0001 | (0x80_u32 << 5);
976        assert!(is_svc(svc80));
977        assert!(!is_svc(0xD503_201F));
978        assert!(!is_svc(brk_encoding(BRK_TRAP_IMM)));
979    }
980
981    #[test]
982    fn brk_encoding_roundtrip_imm() {
983        let enc = brk_encoding(BRK_TRAP_IMM);
984        let imm = (enc >> 5) & 0xFFFF;
985        assert_eq!(imm, BRK_TRAP_IMM);
986    }
987}