Skip to main content

kh_runtime/
trap.rs

1//! Trap backend: rewrite Darwin `svc` for Linux aarch64 translation.
2//!
3//! Linux aarch64 treats every `svc` as a Linux syscall (number in `x8`), which
4//! is incompatible with Darwin (number in `x16`, `svc #0x80`).
5//!
6//! Prefer **userspace trampoline** so the common path never hits `SIGTRAP`:
7//! each `svc` becomes `b veneer` (no link — **preserves `x30`/LR**), and the
8//! per-site veneer saves LR, `bl`s the shared dispatcher, restores LR, then
9//! `b`s back to the instruction after the original `svc`. Fall back to
10//! `brk #IMM` + `SIGTRAP` when a branch is out of range.
11//!
12//! Live handlers require **Linux aarch64**. Patching works on any host.
13#![allow(unsafe_code)]
14
15use std::io::{self, Write};
16use std::sync::Mutex;
17use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering};
18
19use thiserror::Error;
20
21use crate::mem::{
22    MapError, MappedRegion, VM_PROT_EXECUTE, VM_PROT_READ, VM_PROT_WRITE, mprotect_darwin,
23    mprotect_rw,
24};
25use crate::syscall;
26#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
27use crate::syscall::SyscallArgs;
28
29/// Marker immediate for patched traps (`brk #0x0F00`).
30pub const BRK_TRAP_IMM: u32 = 0x0F00;
31
32/// AArch64 `svc #imm16` base encoding: `0xD4000001 | (imm16 << 5)`.
33const SVC_MASK: u32 = 0xFFE0_001F;
34const SVC_BASE: u32 = 0xD400_0001;
35
36/// AArch64 `brk #imm16`: `0xD4200000 | (imm16 << 5)`.
37const fn brk_encoding(imm16: u32) -> u32 {
38    0xD420_0000 | ((imm16 & 0xFFFF) << 5)
39}
40
41/// AArch64 PC-relative branch imm26 (±128 MiB), or `None` if out of range.
42#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
43#[allow(
44    clippy::as_conversions,
45    clippy::cast_possible_truncation,
46    clippy::cast_sign_loss,
47    clippy::manual_range_contains
48)]
49fn branch_imm26(from: u64, to: u64) -> Option<u32> {
50    let delta = i64::try_from(to).ok()?.wrapping_sub(i64::try_from(from).ok()?);
51    if delta & 3 != 0 {
52        return None;
53    }
54    let imm = delta >> 2;
55    // Imm26 signed range for B/BL.
56    if imm < -(1_i64 << 25) || imm >= (1_i64 << 25) {
57        return None;
58    }
59    // Truncate to 26-bit two's complement field (intentional).
60    Some((imm as u32) & 0x03FF_FFFF)
61}
62
63/// AArch64 `BL` (imm26) from `from` to `to`.
64#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
65fn bl_encoding(from: u64, to: u64) -> Option<u32> {
66    Some(0x9400_0000 | branch_imm26(from, to)?)
67}
68
69/// AArch64 `B` (imm26, **no link** — preserves `x30`) from `from` to `to`.
70#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
71fn b_encoding(from: u64, to: u64) -> Option<u32> {
72    Some(0x1400_0000 | branch_imm26(from, to)?)
73}
74
75/// AArch64 `PSTATE` / CPSR carry flag (bit 29 of NZCV).
76///
77/// Darwin arm64 syscall convention: set on error (errno in `x0`), clear on success.
78pub const PSTATE_C: u64 = 1 << 29;
79
80/// One observed trap / translated syscall event.
81#[derive(Debug, Clone, PartialEq, Eq)]
82pub struct TrapEvent {
83    /// Guest PC at the trap (instruction address).
84    pub pc: u64,
85    /// Darwin BSD syscall number (`x16`) when recognized; otherwise `None`.
86    pub syscall: Option<u32>,
87    /// Short label (`exit`, `write`, `unknown`, …).
88    pub name: String,
89    /// First argument (`x0`) when applicable.
90    pub arg0: u64,
91    /// Second argument (`x1`) when applicable.
92    pub arg1: u64,
93    /// Third argument (`x2`) when applicable.
94    pub arg2: u64,
95    /// Return value written to `x0`, if any.
96    pub retval: Option<u64>,
97    /// Darwin error path: carry set, `retval` is positive errno.
98    pub error: bool,
99}
100
101/// Outcome of handling a trap.
102#[derive(Debug, Clone, Copy, PartialEq, Eq)]
103pub enum TrapOutcome {
104    /// Guest called BSD `exit` (or the host forced process termination).
105    Exit {
106        /// Process exit code.
107        code: i32,
108    },
109    /// Guest should resume after the trap instruction.
110    Continue,
111    /// Guest called `bsdthread_terminate` (or equivalent): end this host thread only.
112    ThreadExit,
113}
114
115/// Configuration for the trap backend.
116#[derive(Debug, Clone)]
117pub struct TrapConfig {
118    /// Maximum number of events to retain for `kh trace`.
119    pub max_events: usize,
120    /// Maximum syscalls before forced exit (safety cap).
121    pub max_syscalls: usize,
122}
123
124impl Default for TrapConfig {
125    fn default() -> Self {
126        Self {
127            max_events: 256,
128            max_syscalls: 256,
129        }
130    }
131}
132
133/// Trap subsystem errors.
134#[derive(Debug, Error)]
135pub enum TrapError {
136    /// Failed to install signal handlers.
137    #[error("failed to install trap signal handler: {0}")]
138    SignalSetup(#[source] io::Error),
139
140    /// Memory protect / patch failure.
141    #[error(transparent)]
142    Map(#[from] MapError),
143
144    /// Architecture / OS is not supported for live traps.
145    #[error("trap backend requires Linux aarch64 for live execution")]
146    UnsupportedArch,
147}
148
149#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
150static HANDLER_INSTALLED: AtomicBool = AtomicBool::new(false);
151static EXIT_CODE: AtomicI32 = AtomicI32::new(0);
152static EXIT_REQUESTED: AtomicBool = AtomicBool::new(false);
153static EVENT_COUNT: AtomicU64 = AtomicU64::new(0);
154static MAX_EVENTS: AtomicU64 = AtomicU64::new(256);
155static TRACE_ON_EXIT: AtomicBool = AtomicBool::new(false);
156static TRACE_JSON: AtomicBool = AtomicBool::new(false);
157/// When set (not `i32::MIN`): if guest exit code differs, process exits 1.
158static EXPECT_CODE: AtomicI32 = AtomicI32::new(i32::MIN);
159static EVENTS: Mutex<Vec<TrapEvent>> = Mutex::new(Vec::new());
160
161/// Patches every AArch64 `svc` in executable regions.
162///
163/// On Linux aarch64: prefers `bl` into a guest-mapped trampoline (no signal).
164/// Falls back to `brk #IMM` for out-of-range sites.
165pub fn patch_svc_to_brk(regions: &mut [MappedRegion]) -> Result<usize, TrapError> {
166    #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
167    {
168        patch_svc_linux(regions)
169    }
170    #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
171    {
172        patch_svc_brk_only(regions)
173    }
174}
175
176#[cfg_attr(
177    all(target_arch = "aarch64", target_os = "linux"),
178    allow(dead_code)
179)]
180fn patch_svc_brk_only(regions: &mut [MappedRegion]) -> Result<usize, TrapError> {
181    let mut patched = 0_usize;
182    for region in regions.iter_mut() {
183        if region.prot & VM_PROT_EXECUTE == 0 {
184            continue;
185        }
186        mprotect_rw(region)?;
187        let guest_base = region.guest_addr;
188        let bytes = region.host_bytes_mut();
189        let mut off = 0_usize;
190        while off.saturating_add(4) <= bytes.len() {
191            let end = off.saturating_add(4);
192            let Some(word) = bytes.get(off..end) else {
193                break;
194            };
195            let Ok(word_bytes) = <[u8; 4]>::try_from(word) else {
196                break;
197            };
198            let insn = u32::from_le_bytes(word_bytes);
199            if is_svc(insn) {
200                let brk = brk_encoding(BRK_TRAP_IMM).to_le_bytes();
201                if let Some(slot) = bytes.get_mut(off..end) {
202                    slot.copy_from_slice(&brk);
203                    patched = patched.saturating_add(1);
204                }
205            }
206            let _ = guest_base;
207            off = end;
208        }
209        let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
210        mprotect_darwin(region, restore)?;
211        region.prot = restore;
212    }
213    Ok(patched)
214}
215
216const fn is_svc(insn: u32) -> bool {
217    (insn & SVC_MASK) == SVC_BASE
218}
219
220/// Linux aarch64: optional userspace trampoline (`svc`→`b veneer`).
221///
222/// Enable with `KAKEHASHI_TRAMPOLINE=1`. Default is classic `brk` + `SIGTRAP`
223/// (safe fallback when freestanding hypercall is off). Prefer hypercall for
224/// multi-thread compression; the veneer path remains opt-in.
225///
226/// **Why a veneer (not a direct `bl`)?** Darwin `svc` preserves `x30` (LR). A
227/// direct `bl trampoline` overwrites LR with the resume PC; the libSystem stub
228/// then `ret`s to itself and spins. The veneer saves/restores the real LR.
229///
230/// The shared dispatcher also saves **Q0–Q31 + FPCR/FPSR** (Darwin `svc`
231/// preserves SIMD; Rust would otherwise clobber NEON used by 7zz workers).
232#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
233fn patch_svc_linux(regions: &mut [MappedRegion]) -> Result<usize, TrapError> {
234    if !trampoline_enabled() {
235        return patch_svc_brk_only(regions);
236    }
237    let mut patched = 0_usize;
238    for region in regions.iter_mut() {
239        if region.prot & VM_PROT_EXECUTE == 0 {
240            continue;
241        }
242        mprotect_rw(region)?;
243        let guest_base = region.guest_addr;
244        // Collect svc offsets first.
245        let mut sites = Vec::new();
246        {
247            let bytes = region.host_bytes();
248            let mut off = 0_usize;
249            while off.saturating_add(4) <= bytes.len() {
250                let end = off.saturating_add(4);
251                if let Some(word) = bytes.get(off..end)
252                    && let Ok(word_bytes) = <[u8; 4]>::try_from(word)
253                {
254                    let insn = u32::from_le_bytes(word_bytes);
255                    if is_svc(insn) {
256                        sites.push(off);
257                    }
258                }
259                off = end;
260            }
261        }
262        if sites.is_empty() {
263            let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
264            mprotect_darwin(region, restore)?;
265            region.prot = restore;
266            continue;
267        }
268
269        patched = patched.saturating_add(patch_region_with_veneers(region, guest_base, &sites));
270    }
271    Ok(patched)
272}
273
274/// Per-site veneer size: `str lr; bl shared; ldr lr; b site+4`.
275#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
276const VENEER_BYTES: usize = 16;
277
278/// How many page slots to probe above/below an RX region for a veneer hub.
279#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
280const VENEER_PROBE_RADIUS: u64 = 128;
281
282/// Round `len` up to a multiple of `page` (page must be a power of two ≥ 1).
283#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
284#[inline]
285fn align_up_to_page(len: usize, page: usize) -> usize {
286    if page <= 1 {
287        return len.max(1);
288    }
289    let mask = page.saturating_sub(1);
290    len.saturating_add(mask) & !mask
291}
292
293// libgcc / compiler-rt symbol linked on Linux aarch64 toolchains.
294#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
295unsafe extern "C" {
296    fn __clear_cache(start: *mut libc::c_void, end: *mut libc::c_void);
297}
298
299/// Opt-in trampoline: `KAKEHASHI_TRAMPOLINE=1|true|yes|on`.
300#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
301fn trampoline_enabled() -> bool {
302    match std::env::var_os("KAKEHASHI_TRAMPOLINE") {
303        None => false,
304        Some(v) => {
305            v == "1"
306                || v.eq_ignore_ascii_case("true")
307                || v.eq_ignore_ascii_case("yes")
308                || v.eq_ignore_ascii_case("on")
309        }
310    }
311}
312
313/// Flush D/I caches after writing executable code (aarch64 requirement).
314#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
315fn clear_icache(ptr: *mut u8, len: usize) {
316    if ptr.is_null() || len == 0 {
317        return;
318    }
319    // SAFETY: range is a live mapping the caller owns and just wrote.
320    unsafe {
321        let start = ptr.cast::<libc::c_void>();
322        let end = ptr.wrapping_add(len).cast::<libc::c_void>();
323        __clear_cache(start, end);
324    }
325}
326
327/// Drop trampoline bookkeeping (owned pages are munmap'd with the address space).
328pub fn clear_trampoline_cache() {
329    // Veneer hubs live only as `register_owned` mappings; nothing else to clear.
330}
331
332/// Patch one RX region: map shared dispatcher + per-site veneers near it.
333///
334/// Returns the number of sites rewritten (`b veneer` or fallback `brk`).
335#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
336fn patch_region_with_veneers(
337    region: &mut MappedRegion,
338    guest_base: u64,
339    sites: &[usize],
340) -> usize {
341    let n_sites = sites.len();
342    let Some(hub) = install_veneer_hub(guest_base, region.host_len(), n_sites) else {
343        return patch_sites_brk(region, sites);
344    };
345
346    let mut patched = 0_usize;
347    // SAFETY: hub mapping is RW until we mprotect RX below; length is hub.map_len.
348    let slice = unsafe { std::slice::from_raw_parts_mut(hub.ptr, hub.map_len) };
349
350    for (i, &off) in sites.iter().enumerate() {
351        let site_va = guest_base.saturating_add(u64::try_from(off).unwrap_or(0));
352        let site_ret = site_va.saturating_add(4);
353        let veneer_off = hub.veneer0.saturating_add(i.saturating_mul(VENEER_BYTES));
354        let veneer_va = hub.base_va.saturating_add(u64::try_from(veneer_off).unwrap_or(0));
355        let end = off.saturating_add(4);
356
357        let ok = write_veneer(slice, veneer_off, veneer_va, hub.shared_va, site_ret)
358            .and_then(|()| b_encoding(site_va, veneer_va))
359            .and_then(|enc| {
360                let bytes = region.host_bytes_mut();
361                let slot = bytes.get_mut(off..end)?;
362                slot.copy_from_slice(&enc.to_le_bytes());
363                Some(())
364            })
365            .is_some();
366
367        if !ok {
368            let bytes = region.host_bytes_mut();
369            let brk = brk_encoding(BRK_TRAP_IMM).to_le_bytes();
370            if let Some(slot) = bytes.get_mut(off..end) {
371                slot.copy_from_slice(&brk);
372            }
373        }
374        patched = patched.saturating_add(1);
375    }
376
377    // W^X on hub + I-cache; register owned RX.
378    let prot_rx = libc::PROT_READ | libc::PROT_EXEC;
379    if !crate::host::mprotect(hub.ptr, hub.map_len, prot_rx) {
380        let _ = crate::host::munmap(hub.ptr, hub.map_len);
381        return patch_sites_brk(region, sites);
382    }
383    clear_icache(hub.ptr, hub.map_len);
384    crate::mem::register_owned(
385        hub.base_va,
386        hub.ptr,
387        hub.map_len,
388        VM_PROT_READ | VM_PROT_EXECUTE,
389    );
390
391    let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
392    drop(mprotect_darwin(region, restore));
393    region.prot = restore;
394    let host = region.host_addr();
395    let len = region.host_len();
396    if let (Ok(base), true) = (usize::try_from(host), len > 0) {
397        let p = std::ptr::with_exposed_provenance_mut::<u8>(base);
398        clear_icache(p, len);
399    }
400    patched
401}
402
403#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
404fn patch_sites_brk(region: &mut MappedRegion, sites: &[usize]) -> usize {
405    let mut patched = 0_usize;
406    let bytes = region.host_bytes_mut();
407    for &off in sites {
408        let end = off.saturating_add(4);
409        let brk = brk_encoding(BRK_TRAP_IMM).to_le_bytes();
410        if let Some(slot) = bytes.get_mut(off..end) {
411            slot.copy_from_slice(&brk);
412            patched = patched.saturating_add(1);
413        }
414    }
415    let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
416    drop(mprotect_darwin(region, restore));
417    region.prot = restore;
418    patched
419}
420
421/// Shared dispatcher + veneer pool mapping.
422#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
423struct VeneerHub {
424    ptr: *mut u8,
425    map_len: usize,
426    base_va: u64,
427    shared_va: u64,
428    veneer0: usize,
429}
430
431/// Map RW hub near the RX region (enough room for `n_sites` veneers).
432#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
433fn install_veneer_hub(region_guest: u64, region_len: usize, n_sites: usize) -> Option<VeneerHub> {
434    let region_end = region_guest.saturating_add(u64::try_from(region_len).unwrap_or(0));
435    let site_lo = region_guest;
436    let site_hi = region_end.saturating_sub(4).max(region_guest);
437
438    let page = crate::host::page_size().unwrap_or(4096);
439    let page_u = u64::try_from(page).unwrap_or(4096);
440    let code = trampoline_template();
441    let veneer0 = code.len().saturating_add(15) & !15_usize;
442    let need = veneer0.saturating_add(n_sites.saturating_mul(VENEER_BYTES));
443    let map_len = align_up_to_page(need, page).max(page);
444
445    let flags_fixed = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | crate::host::fixed_map_flag();
446    let flags_free = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
447    let prot_rw = libc::PROT_READ | libc::PROT_WRITE;
448
449    let after0 = region_end.saturating_add(page_u.saturating_sub(1)) & !(page_u.saturating_sub(1));
450    let before0 = region_guest.saturating_sub(page_u) & !(page_u.saturating_sub(1));
451
452    for i in 0..VENEER_PROBE_RADIUS {
453        let mut candidates = [after0.saturating_add(i.saturating_mul(page_u)), 0_u64];
454        if before0 >= i.saturating_mul(page_u) {
455            candidates[1] = before0.saturating_sub(i.saturating_mul(page_u));
456        }
457        for &prefer in &candidates {
458            if prefer == 0 {
459                continue;
460            }
461            // Sites must `b` to veneers in this hub; veneers `bl` shared at base.
462            if b_encoding(site_lo, prefer).is_none() || b_encoding(site_hi, prefer).is_none() {
463                continue;
464            }
465            if let Some(ptr) =
466                crate::host::mmap(Some(prefer), map_len, prot_rw, flags_fixed, -1, 0)
467            {
468                return init_hub_slice(ptr, map_len, code, veneer0);
469            }
470        }
471    }
472
473    let ptr = crate::host::mmap(None, map_len, prot_rw, flags_free, -1, 0)?;
474    let va = crate::host::ptr_addr_u64(ptr);
475    if b_encoding(site_lo, va).is_none() || b_encoding(site_hi, va).is_none() {
476        let _ = crate::host::munmap(ptr, map_len);
477        return None;
478    }
479    init_hub_slice(ptr, map_len, code, veneer0)
480}
481
482#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
483fn init_hub_slice(ptr: *mut u8, map_len: usize, code: &[u8], veneer0: usize) -> Option<VeneerHub> {
484    // SAFETY: fresh RW mapping of `map_len`.
485    let slice = unsafe { std::slice::from_raw_parts_mut(ptr, map_len) };
486    if code.len() > slice.len() || veneer0 > slice.len() {
487        let _ = crate::host::munmap(ptr, map_len);
488        return None;
489    }
490    slice.fill(0);
491    slice.get_mut(..code.len())?.copy_from_slice(code);
492    let reloc_off = trampoline_reloc_offset();
493    if let Some(slot) = slice.get_mut(reloc_off..reloc_off.saturating_add(8)) {
494        slot.copy_from_slice(&trampoline_rust_entry_addr().to_le_bytes());
495    } else {
496        let _ = crate::host::munmap(ptr, map_len);
497        return None;
498    }
499    let base_va = crate::host::ptr_addr_u64(ptr);
500    Some(VeneerHub {
501        ptr,
502        map_len,
503        base_va,
504        shared_va: base_va,
505        veneer0,
506    })
507}
508
509/// Emit one veneer: save LR → bl shared → restore LR → b resume.
510#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
511fn write_veneer(
512    slice: &mut [u8],
513    veneer_off: usize,
514    veneer_va: u64,
515    shared_va: u64,
516    site_ret: u64,
517) -> Option<()> {
518    // str x30, [sp, #-16]!
519    let str_lr: u32 = 0xf81f_0ffe;
520    // ldr x30, [sp], #16
521    let ldr_lr: u32 = 0xf841_07fe;
522    let bl = bl_encoding(veneer_va.saturating_add(4), shared_va)?;
523    let b_back = b_encoding(veneer_va.saturating_add(12), site_ret)?;
524    let words = [str_lr, bl, ldr_lr, b_back];
525    let end = veneer_off.saturating_add(VENEER_BYTES);
526    let slot = slice.get_mut(veneer_off..end)?;
527    for (i, w) in words.iter().enumerate() {
528        let o = i.saturating_mul(4);
529        slot.get_mut(o..o.saturating_add(4))?.copy_from_slice(&w.to_le_bytes());
530    }
531    Some(())
532}
533
534/// Verified aarch64 trampoline (no host-relative BL).
535///
536/// Layout: code bytes + 8-byte absolute pointer (`kh_trampoline_dispatch`).
537#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
538fn trampoline_template() -> &'static [u8] {
539    TRAMP_BYTES
540}
541
542#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
543fn trampoline_reloc_offset() -> usize {
544    TRAMP_RELOC_OFF
545}
546
547#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
548#[allow(clippy::as_conversions, function_casts_as_integer)]
549fn trampoline_rust_entry_addr() -> u64 {
550    u64::try_from(kh_trampoline_dispatch as usize).unwrap_or(0)
551}
552
553/// Return value of the trampoline dispatch (x0=retval, x1=error flag).
554#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
555#[repr(C)]
556pub struct TrampRet {
557    /// Value for guest `x0`.
558    pub retval: u64,
559    /// Non-zero → set Darwin carry (error path).
560    pub error: u64,
561}
562
563/// Called from the guest-mapped trampoline (AAPCS64).
564///
565/// # Safety
566///
567/// Caller must preserve guest SIMD/GPRs around the call (veneer trampoline or
568/// freestanding NEON shim). Arguments are Darwin BSD syscall registers;
569/// `x16` holds the syscall number (or AAPCS 8th arg after the hypercall
570/// prologue). May `pthread_exit` the current host thread on
571/// `bsdthread_terminate`, or `_exit` the process on guest `exit`.
572///
573/// Prefer [`kh_hypercall_entry`] for freestanding libSystem: that path switches
574/// onto a host alt stack first so multi-thread guests never run host Rust on
575/// guest worker stacks.
576#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
577#[unsafe(no_mangle)]
578pub unsafe extern "C" fn kh_trampoline_dispatch(
579    x0: u64,
580    x1: u64,
581    x2: u64,
582    x3: u64,
583    x4: u64,
584    x5: u64,
585    x6: u64,
586    x16: u64,
587) -> TrampRet {
588    let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
589    let result = syscall::dispatch(SyscallArgs {
590        pc: 0,
591        number: sys_no,
592        x0,
593        x1,
594        x2,
595        x3,
596        x4,
597        x5,
598        x6,
599    });
600    if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
601        maybe_push_event(0, sys_no, result.name, x0, x1, x2, result.retval, result.error);
602    }
603    match result.outcome {
604        TrapOutcome::Exit { code } => finish_with_exit_code(code),
605        TrapOutcome::ThreadExit => {
606            // End this host worker only (same as bsdthread_terminate).
607            crate::thread::exit_current_guest_worker();
608        }
609        TrapOutcome::Continue => {}
610    }
611    TrampRet {
612        retval: result.retval.unwrap_or(0),
613        error: u64::from(result.error),
614    }
615}
616
617// Freestanding hypercall entry (`kh_hypercall_entry`):
618// Prefer host alt stack (so join/terminate never runs host Rust on a guest
619// stack that `pthread_join` may unmap). NEON preserve is TRAMP_BYTES (same as
620// the svc veneer). ABI: x0–x6 + number in x7 → TrampRet in x0/x1.
621#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
622std::arch::global_asm!(
623    r#"
624    .text
625    .align 2
626    .global kh_hypercall_entry
627    .type kh_hypercall_entry, %function
628kh_hypercall_entry:
629    // Guest prolog: args + real LR (before any `bl` clobbers x30).
630    sub sp, sp, #96
631    stp x29, x30, [sp, #0]
632    stp x0, x1, [sp, #16]
633    stp x2, x3, [sp, #32]
634    stp x4, x5, [sp, #48]
635    stp x6, x7, [sp, #64]
636    mov x29, sp
637
638    bl kh_host_alt_sp
639    cbz x0, 1f
640    mov x9, sp                 // guest prolog frame
641    mov sp, x0                 // host alt stack
642    ldr x30, [x9, #8]          // reload LR → freestanding thin caller
643    stp x9, x30, [sp, #-16]!
644    ldp x0, x1, [x9, #16]
645    ldp x2, x3, [x9, #32]
646    ldp x4, x5, [x9, #48]
647    ldp x6, x7, [x9, #64]
648    mov x16, x7
649    bl kh_neon_tramp_entry
650    ldp x9, x30, [sp], #16
651    mov sp, x9
652    ldr x29, [sp, #0]
653    add sp, sp, #96
654    ret
6551:
656    // No alt stack: NEON tramp on the guest stack (ST / fallback).
657    ldp x0, x1, [sp, #16]
658    ldp x2, x3, [sp, #32]
659    ldp x4, x5, [sp, #48]
660    ldp x6, x7, [sp, #64]
661    ldp x29, x30, [sp, #0]
662    add sp, sp, #96
663    mov x16, x7
664    b kh_neon_tramp_entry
665    .size kh_hypercall_entry, .-kh_hypercall_entry
666    "#
667);
668
669// `kh_neon_tramp_entry` + `KH_NEON_TRAMP_VA` (filled by `ensure_neon_tramp`).
670#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
671std::arch::global_asm!(
672    r#"
673    .text
674    .align 2
675    .global kh_neon_tramp_entry
676    .type kh_neon_tramp_entry, %function
677kh_neon_tramp_entry:
678    adrp x17, KH_NEON_TRAMP_VA
679    ldr x17, [x17, :lo12:KH_NEON_TRAMP_VA]
680    cbz x17, 1f
681    br x17
6821:
683    mov x7, x16
684    b kh_trampoline_dispatch
685    .size kh_neon_tramp_entry, .-kh_neon_tramp_entry
686
687    .data
688    .align 3
689    .global KH_NEON_TRAMP_VA
690KH_NEON_TRAMP_VA:
691    .quad 0
692    "#
693);
694
695#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
696unsafe extern "C" {
697    fn kh_hypercall_entry();
698    static mut KH_NEON_TRAMP_VA: u64;
699}
700
701/// Address of freestanding hypercall entry (alt stack + NEON tramp + dispatch).
702#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
703#[must_use]
704pub fn hypercall_entry_addr() -> u64 {
705    ensure_neon_tramp();
706    #[allow(clippy::as_conversions, function_casts_as_integer)]
707    {
708        u64::try_from(kh_hypercall_entry as usize).unwrap_or(0)
709    }
710}
711
712/// Map TRAMP_BYTES once and publish its VA to `KH_NEON_TRAMP_VA`.
713#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
714fn ensure_neon_tramp() {
715    static ONCE: AtomicU64 = AtomicU64::new(0);
716    static LOCK: Mutex<()> = Mutex::new(());
717
718    if ONCE.load(Ordering::Acquire) != 0 {
719        return;
720    }
721    let _g = LOCK
722        .lock()
723        .unwrap_or_else(std::sync::PoisonError::into_inner);
724    if ONCE.load(Ordering::Acquire) != 0 {
725        return;
726    }
727
728    let code = trampoline_template();
729    let page = crate::host::page_size().unwrap_or(4096);
730    let map_len = align_up_to_page(code.len(), page).max(page);
731    let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
732    let prot_rw = libc::PROT_READ | libc::PROT_WRITE;
733    let Some(ptr) = crate::host::mmap(None, map_len, prot_rw, flags, -1, 0) else {
734        tracing::error!("NEON tramp mmap failed");
735        return;
736    };
737    // SAFETY: fresh RW mapping.
738    let slice = unsafe { std::slice::from_raw_parts_mut(ptr, map_len) };
739    slice.fill(0);
740    if let Some(dst) = slice.get_mut(..code.len()) {
741        dst.copy_from_slice(code);
742    } else {
743        let _ = crate::host::munmap(ptr, map_len);
744        return;
745    }
746    let reloc_off = trampoline_reloc_offset();
747    if let Some(slot) = slice.get_mut(reloc_off..reloc_off.saturating_add(8)) {
748        slot.copy_from_slice(&trampoline_rust_entry_addr().to_le_bytes());
749    } else {
750        let _ = crate::host::munmap(ptr, map_len);
751        return;
752    }
753    let prot_rx = libc::PROT_READ | libc::PROT_EXEC;
754    if !crate::host::mprotect(ptr, map_len, prot_rx) {
755        tracing::error!("NEON tramp mprotect RX failed");
756        let _ = crate::host::munmap(ptr, map_len);
757        return;
758    }
759    clear_icache(ptr, map_len);
760    let va = crate::host::ptr_addr_u64(ptr);
761    // SAFETY: symbol from global_asm; written once under LOCK before guest entry.
762    unsafe {
763        KH_NEON_TRAMP_VA = va;
764    }
765    ONCE.store(va, Ordering::Release);
766    tracing::info!(va = format_args!("{va:#x}"), "mapped NEON tramp for hypercall");
767}
768
769#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
770#[must_use]
771pub fn hypercall_entry_addr() -> u64 {
772    0
773}
774
775// ── trampoline machine code (verified via rustc/objdump aarch64) ─────────────
776
777/// Offset of the 8-byte absolute `kh_trampoline_dispatch` pointer.
778///
779/// Layout (see `TRAMP_BYTES`): `bti c` + save GPRs/NEON/FPCR + `blr` dispatch
780/// + restore + `ret` + `.quad` reloc @ 0x100.
781#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
782const TRAMP_RELOC_OFF: usize = 0x100;
783
784/// Exact opcodes from assembled trampoline (`as` / objdump on Linux aarch64).
785///
786/// Saves **all Q0–Q31 + FPCR/FPSR** around the Rust call: Darwin `svc` preserves
787/// SIMD state; compression workers (7zz MT) rely on that. A bare GPR-only
788/// trampoline clobbered NEON and faulted under `-mmt>1`.
789#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
790#[rustfmt::skip]
791const TRAMP_BYTES: &[u8] = &[
792    0x5f, 0x24, 0x03, 0xd5, 0xff, 0x03, 0x0a, 0xd1, 0xe8, 0x27, 0x00, 0xa9, 0xea, 0x2f, 0x01, 0xa9,
793    0xec, 0x37, 0x02, 0xa9, 0xee, 0x3f, 0x03, 0xa9, 0xf1, 0x4b, 0x04, 0xa9, 0xfe, 0x2b, 0x00, 0xf9,
794    0x08, 0x44, 0x3b, 0xd5, 0x29, 0x44, 0x3b, 0xd5, 0xe8, 0x27, 0x06, 0xa9, 0xe0, 0x87, 0x03, 0xad,
795    0xe2, 0x8f, 0x04, 0xad, 0xe4, 0x97, 0x05, 0xad, 0xe6, 0x9f, 0x06, 0xad, 0xe8, 0xa7, 0x07, 0xad,
796    0xea, 0xaf, 0x08, 0xad, 0xec, 0xb7, 0x09, 0xad, 0xee, 0xbf, 0x0a, 0xad, 0xf0, 0xc7, 0x0b, 0xad,
797    0xf2, 0xcf, 0x0c, 0xad, 0xf4, 0xd7, 0x0d, 0xad, 0xf6, 0xdf, 0x0e, 0xad, 0xf8, 0xe7, 0x0f, 0xad,
798    0xfa, 0xef, 0x10, 0xad, 0xfc, 0xf7, 0x11, 0xad, 0xfe, 0xff, 0x12, 0xad, 0xe7, 0x03, 0x10, 0xaa,
799    0x88, 0x04, 0x00, 0x58, 0x00, 0x01, 0x3f, 0xd6, 0x02, 0x42, 0x3b, 0xd5, 0x23, 0x00, 0x80, 0x52,
800    0x63, 0x88, 0x63, 0xd3, 0x42, 0x00, 0x23, 0x8a, 0x41, 0x00, 0x00, 0xb4, 0x42, 0x00, 0x03, 0xaa,
801    0x02, 0x42, 0x1b, 0xd5, 0xe8, 0x27, 0x46, 0xa9, 0x08, 0x44, 0x1b, 0xd5, 0x29, 0x44, 0x1b, 0xd5,
802    0xe0, 0x87, 0x43, 0xad, 0xe2, 0x8f, 0x44, 0xad, 0xe4, 0x97, 0x45, 0xad, 0xe6, 0x9f, 0x46, 0xad,
803    0xe8, 0xa7, 0x47, 0xad, 0xea, 0xaf, 0x48, 0xad, 0xec, 0xb7, 0x49, 0xad, 0xee, 0xbf, 0x4a, 0xad,
804    0xf0, 0xc7, 0x4b, 0xad, 0xf2, 0xcf, 0x4c, 0xad, 0xf4, 0xd7, 0x4d, 0xad, 0xf6, 0xdf, 0x4e, 0xad,
805    0xf8, 0xe7, 0x4f, 0xad, 0xfa, 0xef, 0x50, 0xad, 0xfc, 0xf7, 0x51, 0xad, 0xfe, 0xff, 0x52, 0xad,
806    0xfe, 0x2b, 0x40, 0xf9, 0xf1, 0x4b, 0x44, 0xa9, 0xee, 0x3f, 0x43, 0xa9, 0xec, 0x37, 0x42, 0xa9,
807    0xea, 0x2f, 0x41, 0xa9, 0xe8, 0x27, 0x40, 0xa9, 0xff, 0x03, 0x0a, 0x91, 0xc0, 0x03, 0x5f, 0xd6,
808    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // reloc .quad @ 0x100
809];
810/// Installs the `SIGTRAP` handler used by the micro execution backend.
811pub fn install_trap_handlers(config: &TrapConfig) -> Result<(), TrapError> {
812    MAX_EVENTS.store(
813        u64::try_from(config.max_events).unwrap_or(256),
814        Ordering::SeqCst,
815    );
816    syscall::reset_syscall_state(config.max_syscalls);
817
818    #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
819    {
820        let _ = config;
821        Err(TrapError::UnsupportedArch)
822    }
823
824    #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
825    {
826        if HANDLER_INSTALLED.swap(true, Ordering::SeqCst) {
827            reset_trap_state();
828            syscall::reset_syscall_state(config.max_syscalls);
829            return Ok(());
830        }
831
832        // Guest faults used to leave multi‑MB `core` files in the workspace cwd.
833        // Faults go through our handler → `_exit`; disable dumps as belt-and-braces.
834        disable_core_dumps();
835
836        // SAFETY: process-wide SIGTRAP / fault handlers with SA_SIGINFO.
837        unsafe {
838            let mut sa: libc::sigaction = std::mem::zeroed();
839            sa.sa_flags = libc::SA_SIGINFO;
840            #[allow(clippy::as_conversions, function_casts_as_integer)]
841            {
842                sa.sa_sigaction = trap_sigaction as *const () as usize;
843            }
844            libc::sigemptyset(std::ptr::addr_of_mut!(sa.sa_mask));
845            if libc::sigaction(libc::SIGTRAP, std::ptr::addr_of!(sa), std::ptr::null_mut()) != 0 {
846                HANDLER_INSTALLED.store(false, Ordering::SeqCst);
847                return Err(TrapError::SignalSetup(io::Error::last_os_error()));
848            }
849
850            // Guest faults: print PC/addr so `kh run` diagnoses unbound GOT / bad heap.
851            let mut fault: libc::sigaction = std::mem::zeroed();
852            fault.sa_flags = libc::SA_SIGINFO;
853            #[allow(clippy::as_conversions, function_casts_as_integer)]
854            {
855                fault.sa_sigaction = guest_fault_sigaction as *const () as usize;
856            }
857            libc::sigemptyset(std::ptr::addr_of_mut!(fault.sa_mask));
858            for sig in [libc::SIGSEGV, libc::SIGBUS] {
859                if libc::sigaction(sig, std::ptr::addr_of!(fault), std::ptr::null_mut()) != 0 {
860                    HANDLER_INSTALLED.store(false, Ordering::SeqCst);
861                    return Err(TrapError::SignalSetup(io::Error::last_os_error()));
862                }
863            }
864        }
865        reset_trap_state();
866        Ok(())
867    }
868}
869
870/// Sets `RLIMIT_CORE` soft+hard to 0 so guest crashes never litter the repo with
871/// `core` dumps (we already print a guest fault summary and `_exit`).
872#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
873fn disable_core_dumps() {
874    // SAFETY: process-wide rlimit; best-effort, ignore failure.
875    unsafe {
876        let lim = libc::rlimit {
877            rlim_cur: 0,
878            rlim_max: 0,
879        };
880        let _ = libc::setrlimit(libc::RLIMIT_CORE, std::ptr::addr_of!(lim));
881    }
882}
883
884/// Logs guest `SIGSEGV` / `SIGBUS` (PC, fault address, key regs) then exits.
885#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
886unsafe extern "C" fn guest_fault_sigaction(
887    signo: libc::c_int,
888    info: *mut libc::siginfo_t,
889    ctx: *mut libc::c_void,
890) {
891    let name = match signo {
892        libc::SIGSEGV => "SIGSEGV",
893        libc::SIGBUS => "SIGBUS",
894        _ => "FAULT",
895    };
896    let mut pc = 0_u64;
897    let mut sp = 0_u64;
898    let mut x0 = 0_u64;
899    let mut x1 = 0_u64;
900    let mut x8 = 0_u64;
901    let mut x16 = 0_u64;
902    let mut lr = 0_u64;
903    if !ctx.is_null() {
904        // SAFETY: kernel `ucontext_t` for SA_SIGINFO.
905        let uctx = unsafe { &*ctx.cast::<libc::ucontext_t>() };
906        let m = &uctx.uc_mcontext;
907        pc = m.pc;
908        sp = m.sp;
909        x0 = m.regs[0];
910        x1 = m.regs[1];
911        x8 = m.regs[8];
912        x16 = m.regs[16];
913        lr = m.regs[30];
914    }
915    let addr = if info.is_null() {
916        0_u64
917    } else {
918        // SAFETY: kernel-provided siginfo.
919        let p = unsafe { (*info).si_addr() };
920        u64::try_from(p.addr()).unwrap_or(0)
921    };
922    let msg = format!(
923        "error: guest {name} pc={pc:#x} addr={addr:#x} sp={sp:#x} lr={lr:#x} \
924         x0={x0:#x} x1={x1:#x} x8={x8:#x} x16={x16:#x}\n"
925    );
926    drop(io::stderr().write_all(msg.as_bytes()));
927    drop(io::stderr().flush());
928    if TRACE_ON_EXIT.load(Ordering::SeqCst) {
929        dump_events_before_exit();
930    }
931    // SAFETY: hard-stop after unrecoverable guest fault.
932    unsafe { libc::_exit(128_i32.saturating_add(signo)) };
933}
934
935/// Drains recorded trap events (for `kh trace`).
936#[must_use]
937pub fn take_trace_events() -> Vec<TrapEvent> {
938    EVENTS
939        .lock()
940        .map(|mut g| std::mem::take(&mut *g))
941        .unwrap_or_default()
942}
943
944/// Whether the guest requested exit.
945#[must_use]
946pub fn exit_requested() -> bool {
947    EXIT_REQUESTED.load(Ordering::SeqCst)
948}
949
950/// Exit code from the last guest `exit` trap.
951#[must_use]
952pub fn take_exit_code() -> Option<i32> {
953    if EXIT_REQUESTED.load(Ordering::SeqCst) {
954        Some(EXIT_CODE.load(Ordering::SeqCst))
955    } else {
956        None
957    }
958}
959
960/// Resets exit / event state before a new run.
961pub fn reset_trap_state() {
962    EXIT_REQUESTED.store(false, Ordering::SeqCst);
963    EXIT_CODE.store(0, Ordering::SeqCst);
964    EVENT_COUNT.store(0, Ordering::SeqCst);
965    if let Ok(mut guard) = EVENTS.lock() {
966        guard.clear();
967    }
968}
969
970/// When set, the exit trap prints recorded events to stdout before `_exit`.
971pub fn set_trace_on_exit(json: bool) {
972    TRACE_ON_EXIT.store(true, Ordering::SeqCst);
973    TRACE_JSON.store(json, Ordering::SeqCst);
974}
975
976/// Clears the trace-on-exit flag (default for plain `kh run`).
977pub fn clear_trace_on_exit() {
978    TRACE_ON_EXIT.store(false, Ordering::SeqCst);
979    TRACE_JSON.store(false, Ordering::SeqCst);
980}
981
982/// Sets the expected guest exit code for the micro gate (`--expect-code`).
983pub fn set_expect_code(code: i32) {
984    EXPECT_CODE.store(code, Ordering::SeqCst);
985}
986
987/// Disables the expect-code gate.
988pub fn clear_expect_code() {
989    EXPECT_CODE.store(i32::MIN, Ordering::SeqCst);
990}
991
992/// Terminates the host process as if the guest called BSD `exit(code)`.
993///
994/// Used after `LC_MAIN` returns (dyld calls `exit` with `main`'s return value).
995/// Honours `--expect-code` and optional trace-on-exit dump.
996pub fn finish_with_exit_code(code: i32) -> ! {
997    EXIT_CODE.store(code, Ordering::SeqCst);
998    EXIT_REQUESTED.store(true, Ordering::SeqCst);
999    if TRACE_ON_EXIT.load(Ordering::SeqCst) {
1000        dump_events_before_exit();
1001    }
1002    let expect = EXPECT_CODE.load(Ordering::SeqCst);
1003    let status = if expect == i32::MIN || expect == code {
1004        code
1005    } else {
1006        let msg = format!("error: guest exit code {code} != expected {expect}\n");
1007        drop(io::stderr().write_all(msg.as_bytes()));
1008        1
1009    };
1010    // SAFETY: intentional process termination after guest main return / exit.
1011    unsafe { libc::_exit(status) };
1012}
1013
1014#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
1015unsafe extern "C" fn trap_sigaction(
1016    signo: libc::c_int,
1017    _info: *mut libc::siginfo_t,
1018    ctx: *mut libc::c_void,
1019) {
1020    if signo != libc::SIGTRAP || ctx.is_null() {
1021        return;
1022    }
1023
1024    // SAFETY: kernel provides a valid `ucontext_t` for SA_SIGINFO deliveries.
1025    let uctx = unsafe { &mut *ctx.cast::<libc::ucontext_t>() };
1026    let m = &mut uctx.uc_mcontext;
1027    let pc = m.pc;
1028    // Only pull the regs dispatch needs; avoid extra loads on the hot path.
1029    let x0 = m.regs[0];
1030    let x1 = m.regs[1];
1031    let x2 = m.regs[2];
1032    let x3 = m.regs[3];
1033    let x4 = m.regs[4];
1034    let x5 = m.regs[5];
1035    let x6 = m.regs[6];
1036    let x16 = m.regs[16];
1037
1038    let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
1039    let result = syscall::dispatch(SyscallArgs {
1040        pc,
1041        number: sys_no,
1042        x0,
1043        x1,
1044        x2,
1045        x3,
1046        x4,
1047        x5,
1048        x6,
1049    });
1050
1051    // Zero-cost when max_events == 0 (plain `kh run`).
1052    if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
1053        maybe_push_event(pc, sys_no, result.name, x0, x1, x2, result.retval, result.error);
1054    }
1055
1056    if let Some(ret) = result.retval {
1057        m.regs[0] = ret;
1058    }
1059    // Darwin arm64: error → set C; success with write-back → clear C.
1060    if result.error {
1061        m.pstate |= PSTATE_C;
1062    } else if result.retval.is_some() {
1063        m.pstate &= !PSTATE_C;
1064    }
1065
1066    match result.outcome {
1067        TrapOutcome::Exit { code } => {
1068            finish_with_exit_code(code);
1069        }
1070        TrapOutcome::Continue => {
1071            m.pc = pc.wrapping_add(4);
1072        }
1073        TrapOutcome::ThreadExit => {
1074            // Redirect ucontext to a host `pthread_exit` trampoline on this
1075            // thread's original stack (see `crate::thread`). Main thread
1076            // (no host frame) falls back to process exit.
1077            if !crate::thread::redirect_ucontext_to_host_exit(m) {
1078                finish_with_exit_code(0);
1079            }
1080        }
1081    }
1082}
1083
1084fn dump_events_before_exit() {
1085    use std::fmt::Write as _;
1086
1087    let json = TRACE_JSON.load(Ordering::SeqCst);
1088    let events = EVENTS.lock().map(|g| g.clone()).unwrap_or_default();
1089    if json {
1090        let mut out = String::from("{\"events\":[");
1091        for (i, e) in events.iter().enumerate() {
1092            if i > 0 {
1093                out.push(',');
1094            }
1095            let sys = e
1096                .syscall
1097                .map_or_else(|| "null".to_owned(), |s| s.to_string());
1098            let ret = e
1099                .retval
1100                .map_or_else(|| "null".to_owned(), |v| v.to_string());
1101            let _ = write!(
1102                out,
1103                "{{\"pc\":{},\"syscall\":{},\"name\":\"{}\",\"arg0\":{},\"arg1\":{},\"arg2\":{},\"retval\":{},\"error\":{}}}",
1104                e.pc, sys, e.name, e.arg0, e.arg1, e.arg2, ret, e.error
1105            );
1106        }
1107        out.push_str("]}\n");
1108        drop(io::stdout().write_all(out.as_bytes()));
1109        drop(io::stdout().flush());
1110    } else {
1111        let header = format!("trace events: {}\n", events.len());
1112        drop(io::stdout().write_all(header.as_bytes()));
1113        for (i, e) in events.iter().enumerate() {
1114            let ret = e
1115                .retval
1116                .map_or_else(|| "-".to_owned(), |v| format!("{v:#x}"));
1117            let err = if e.error { " err" } else { "" };
1118            let line = format!(
1119                "  [{i}] pc={:#x} {} sys={:?} args=({:#x}, {:#x}, {:#x}) ret={ret}{err}\n",
1120                e.pc, e.name, e.syscall, e.arg0, e.arg1, e.arg2
1121            );
1122            drop(io::stdout().write_all(line.as_bytes()));
1123        }
1124        drop(io::stdout().flush());
1125    }
1126}
1127
1128#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
1129#[allow(clippy::too_many_arguments)]
1130fn maybe_push_event(
1131    pc: u64,
1132    sys_no: u32,
1133    name: &'static str,
1134    arg0: u64,
1135    arg1: u64,
1136    arg2: u64,
1137    retval: Option<u64>,
1138    error: bool,
1139) {
1140    let max = MAX_EVENTS.load(Ordering::Relaxed);
1141    // `kh run` sets max_events=0: single load, no String/Mutex.
1142    if max == 0 {
1143        return;
1144    }
1145    let n = EVENT_COUNT.fetch_add(1, Ordering::Relaxed);
1146    if n >= max {
1147        return;
1148    }
1149    if let Ok(mut guard) = EVENTS.lock()
1150        && u64::try_from(guard.len()).unwrap_or(u64::MAX) < max
1151    {
1152        guard.push(TrapEvent {
1153            pc,
1154            syscall: Some(sys_no),
1155            name: name.to_owned(),
1156            arg0,
1157            arg1,
1158            arg2,
1159            retval,
1160            error,
1161        });
1162    }
1163}
1164
1165#[cfg(test)]
1166mod tests {
1167    use super::*;
1168
1169    #[test]
1170    fn detects_svc_encoding() {
1171        assert!(is_svc(0xD400_0001));
1172        let svc80 = 0xD400_0001 | (0x80_u32 << 5);
1173        assert!(is_svc(svc80));
1174        assert!(!is_svc(0xD503_201F));
1175        assert!(!is_svc(brk_encoding(BRK_TRAP_IMM)));
1176    }
1177
1178    #[test]
1179    fn brk_encoding_roundtrip_imm() {
1180        let enc = brk_encoding(BRK_TRAP_IMM);
1181        let imm = (enc >> 5) & 0xFFFF;
1182        assert_eq!(imm, BRK_TRAP_IMM);
1183    }
1184}