Skip to main content

kh_runtime/
trap.rs

1//! Trap backend: rewrite residual Darwin `svc` and host hypercall entry.
2//!
3//! **Production path:** freestanding libSystem → `kh_hypercall_entry` (alt
4//! stack + NEON tramp + [`kh_trampoline_dispatch`]). No `SIGTRAP` on that path.
5//!
6//! **Residual `svc`:** leftover Darwin `svc` in fixtures / unpatched third-party
7//! code is rewritten to `brk #IMM` so Linux does not treat it as a host
8//! syscall. The `SIGTRAP` handler translates those sites. This is **not** a
9//! second production path — freestanding libSystem is always hypercall-wired.
10//! Former experimental `svc`→veneer (`KAKEHASHI_TRAMPOLINE`) was removed.
11//!
12//! Live handlers require **Linux aarch64**. Patching works on any host.
13#![allow(unsafe_code)]
14
15use std::io::{self, Write};
16use std::sync::Mutex;
17use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering};
18
19use thiserror::Error;
20
21use crate::mem::{
22    MapError, MappedRegion, VM_PROT_EXECUTE, VM_PROT_READ, VM_PROT_WRITE, mprotect_darwin,
23    mprotect_rw,
24};
25use crate::syscall;
26#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
27use crate::syscall::SyscallArgs;
28
29/// Marker immediate for patched traps (`brk #0x0F00`).
30pub const BRK_TRAP_IMM: u32 = 0x0F00;
31
32/// AArch64 `svc #imm16` base encoding: `0xD4000001 | (imm16 << 5)`.
33const SVC_MASK: u32 = 0xFFE0_001F;
34const SVC_BASE: u32 = 0xD400_0001;
35
36/// AArch64 `brk #imm16`: `0xD4200000 | (imm16 << 5)`.
37const fn brk_encoding(imm16: u32) -> u32 {
38    0xD420_0000 | ((imm16 & 0xFFFF) << 5)
39}
40
41/// AArch64 `PSTATE` / CPSR carry flag (bit 29 of NZCV).
42///
43/// Darwin arm64 syscall convention: set on error (errno in `x0`), clear on success.
44pub const PSTATE_C: u64 = 1 << 29;
45
46/// One observed trap / translated syscall event.
47#[derive(Debug, Clone, PartialEq, Eq)]
48pub struct TrapEvent {
49    /// Guest PC at the trap (instruction address).
50    pub pc: u64,
51    /// Darwin BSD syscall number (`x16`) when recognized; otherwise `None`.
52    pub syscall: Option<u32>,
53    /// Short label (`exit`, `write`, `unknown`, …).
54    pub name: String,
55    /// First argument (`x0`) when applicable.
56    pub arg0: u64,
57    /// Second argument (`x1`) when applicable.
58    pub arg1: u64,
59    /// Third argument (`x2`) when applicable.
60    pub arg2: u64,
61    /// Return value written to `x0`, if any.
62    pub retval: Option<u64>,
63    /// Darwin error path: carry set, `retval` is positive errno.
64    pub error: bool,
65}
66
67/// Outcome of handling a trap.
68#[derive(Debug, Clone, Copy, PartialEq, Eq)]
69pub enum TrapOutcome {
70    /// Guest called BSD `exit` (or the host forced process termination).
71    Exit {
72        /// Process exit code.
73        code: i32,
74    },
75    /// Guest should resume after the trap instruction.
76    Continue,
77    /// Guest called `bsdthread_terminate` (or equivalent): end this host thread only.
78    ThreadExit,
79}
80
81/// Configuration for the trap backend.
82#[derive(Debug, Clone)]
83pub struct TrapConfig {
84    /// Maximum number of events to retain for `kh trace`.
85    pub max_events: usize,
86    /// Maximum syscalls before forced exit (safety cap).
87    pub max_syscalls: usize,
88}
89
90impl Default for TrapConfig {
91    fn default() -> Self {
92        Self {
93            max_events: 256,
94            max_syscalls: 256,
95        }
96    }
97}
98
99/// Trap subsystem errors.
100#[derive(Debug, Error)]
101pub enum TrapError {
102    /// Failed to install signal handlers.
103    #[error("failed to install trap signal handler: {0}")]
104    SignalSetup(#[source] io::Error),
105
106    /// Memory protect / patch failure.
107    #[error(transparent)]
108    Map(#[from] MapError),
109
110    /// Architecture / OS is not supported for live traps.
111    #[error("trap backend requires Linux aarch64 for live execution")]
112    UnsupportedArch,
113}
114
115#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
116static HANDLER_INSTALLED: AtomicBool = AtomicBool::new(false);
117static EXIT_CODE: AtomicI32 = AtomicI32::new(0);
118static EXIT_REQUESTED: AtomicBool = AtomicBool::new(false);
119static EVENT_COUNT: AtomicU64 = AtomicU64::new(0);
120static MAX_EVENTS: AtomicU64 = AtomicU64::new(256);
121static TRACE_ON_EXIT: AtomicBool = AtomicBool::new(false);
122static TRACE_JSON: AtomicBool = AtomicBool::new(false);
123/// When set (not `i32::MIN`): if guest exit code differs, process exits 1.
124static EXPECT_CODE: AtomicI32 = AtomicI32::new(i32::MIN);
125static EVENTS: Mutex<Vec<TrapEvent>> = Mutex::new(Vec::new());
126
127/// Patches every AArch64 `svc` in executable regions to `brk #IMM`.
128///
129/// Residual sites only: freestanding libSystem uses hypercall, not `svc`.
130/// Applies to **any** guest image that embeds Darwin `svc` (fixtures, rare
131/// apps); product tools that only call through freestanding typically have
132/// **zero** hits — we still scan, but skip `mprotect` RW/RX when none found.
133pub fn patch_svc_to_brk(regions: &mut [MappedRegion]) -> Result<usize, TrapError> {
134    let mut patched = 0_usize;
135    for region in regions.iter_mut() {
136        if region.prot & VM_PROT_EXECUTE == 0 {
137            continue;
138        }
139        // Only walk image content; BSS tail cannot hold code from disk.
140        let file_len = usize::try_from(region.file_bytes)
141            .unwrap_or(0)
142            .min(region.host_len());
143        if file_len < 4 {
144            continue;
145        }
146
147        // Prefer instruction-section ranges from the Mach-O plan (any guest).
148        // Full-TEXT walks hit false-positive `svc` encodings in `__const` /
149        // `__cstring` and force-fault non-code pages of large CLT tools.
150        let ranges = resolve_svc_scan_ranges(region, file_len);
151        if ranges.is_empty() {
152            continue;
153        }
154
155        // Pass 1: read-only scan of instruction ranges only.
156        let hits = {
157            let bytes = region.host_bytes();
158            let mut hits = Vec::new();
159            for &(start, len) in &ranges {
160                let end = start.saturating_add(len).min(file_len);
161                if end.saturating_sub(start) < 4 {
162                    continue;
163                }
164                let Some(scan) = bytes.get(start..end) else {
165                    continue;
166                };
167                hits.extend(find_svc_offsets(scan).into_iter().map(|o| start.saturating_add(o)));
168            }
169            hits
170        };
171        if hits.is_empty() {
172            continue;
173        }
174
175        // Pass 2: mprotect RW only when there is something to rewrite.
176        mprotect_rw(region)?;
177        let bytes = region.host_bytes_mut();
178        let brk = brk_encoding(BRK_TRAP_IMM).to_le_bytes();
179        for off in &hits {
180            let end = off.saturating_add(4);
181            if let Some(slot) = bytes.get_mut(*off..end) {
182                slot.copy_from_slice(&brk);
183                patched = patched.saturating_add(1);
184            }
185        }
186        let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
187        mprotect_darwin(region, restore)?;
188        region.prot = restore;
189        // Flush only patched sites (not whole TEXT — pure file maps stay lazy
190        // until first demand fault / these CoW writes).
191        let base = region.host_bytes().as_ptr();
192        for off in hits {
193            // SAFETY: `off` is within file_bytes of this live mapping.
194            let ptr = unsafe { base.add(off).cast_mut() };
195            clear_icache(ptr, 4);
196        }
197    }
198    Ok(patched)
199}
200
201/// Instruction ranges to scan, or whole `file_len` when the plan had none.
202fn resolve_svc_scan_ranges(region: &MappedRegion, file_len: usize) -> Vec<(usize, usize)> {
203    if region.svc_scan_ranges.is_empty() {
204        return vec![(0, file_len)];
205    }
206    let mut out = Vec::with_capacity(region.svc_scan_ranges.len());
207    for &(off, len) in &region.svc_scan_ranges {
208        let start = usize::try_from(off).unwrap_or(usize::MAX);
209        let len = usize::try_from(len).unwrap_or(0);
210        if start >= file_len || len == 0 {
211            continue;
212        }
213        let len = len.min(file_len.saturating_sub(start));
214        if len >= 4 {
215            out.push((start, len));
216        }
217    }
218    out
219}
220
221const fn is_svc(insn: u32) -> bool {
222    (insn & SVC_MASK) == SVC_BASE
223}
224
225/// Scan little-endian AArch64 insn stream for `svc` encodings.
226///
227/// Unrolled word loads; large regions (≥8 MiB) fan out across worker threads
228/// so CLT-sized TEXT does not serialize page-fault + scan on one core.
229#[allow(clippy::integer_division)] // insn stream: 4-byte AArch64 words
230fn find_svc_offsets(scan: &[u8]) -> Vec<usize> {
231    const PARALLEL_MIN: usize = 8 << 20; // 8 MiB
232    if scan.len() >= PARALLEL_MIN {
233        return find_svc_offsets_parallel(scan);
234    }
235    find_svc_offsets_range(scan, 0)
236}
237
238/// Parallel `svc` scan over word-aligned chunks (any guest with large RX).
239#[allow(clippy::integer_division)]
240fn find_svc_offsets_parallel(scan: &[u8]) -> Vec<usize> {
241    let nthreads = std::thread::available_parallelism()
242        .map_or(1, std::num::NonZeroUsize::get)
243        .clamp(1, 8);
244    if nthreads <= 1 {
245        return find_svc_offsets_range(scan, 0);
246    }
247    let total = scan.len() & !3_usize;
248    if total < 4 {
249        return Vec::new();
250    }
251    let chunk = (total
252        .checked_div(nthreads)
253        .unwrap_or(total)
254        & !3_usize)
255        .max(4);
256    let mut hits = Vec::new();
257    std::thread::scope(|scope| {
258        let mut handles = Vec::with_capacity(nthreads);
259        let mut start = 0_usize;
260        while start < total {
261            let end = (start.saturating_add(chunk)).min(total);
262            let Some(part) = scan.get(start..end) else {
263                break;
264            };
265            let base_off = start;
266            handles.push(scope.spawn(move || find_svc_offsets_range(part, base_off)));
267            start = end;
268        }
269        for h in handles {
270            if let Ok(mut part) = h.join() {
271                hits.append(&mut part);
272            }
273        }
274    });
275    hits.sort_unstable();
276    hits
277}
278
279/// Scan `scan` for `svc`; returned offsets are relative to the full image
280/// when `base_off` is the absolute start of this slice.
281#[allow(clippy::integer_division)]
282fn find_svc_offsets_range(scan: &[u8], base_off: usize) -> Vec<usize> {
283    let mut hits = Vec::new();
284    let words = scan.len() / 4;
285    if words == 0 {
286        return hits;
287    }
288    // SAFETY: scan is a live host mapping; unaligned u32 load is fine on
289    // aarch64/x86_64.
290    let base = scan.as_ptr();
291    let mut i = 0_usize;
292    while i.saturating_add(4) <= words {
293        // SAFETY: i..i+3 < words.
294        let w0 = unsafe { base.add(i.saturating_mul(4)).cast::<u32>().read_unaligned() };
295        let w1 = unsafe {
296            base.add(i.saturating_add(1).saturating_mul(4))
297                .cast::<u32>()
298                .read_unaligned()
299        };
300        let w2 = unsafe {
301            base.add(i.saturating_add(2).saturating_mul(4))
302                .cast::<u32>()
303                .read_unaligned()
304        };
305        let w3 = unsafe {
306            base.add(i.saturating_add(3).saturating_mul(4))
307                .cast::<u32>()
308                .read_unaligned()
309        };
310        if is_svc(u32::from_le(w0)) {
311            hits.push(base_off.saturating_add(i.saturating_mul(4)));
312        }
313        if is_svc(u32::from_le(w1)) {
314            hits.push(
315                base_off.saturating_add(i.saturating_add(1).saturating_mul(4)),
316            );
317        }
318        if is_svc(u32::from_le(w2)) {
319            hits.push(
320                base_off.saturating_add(i.saturating_add(2).saturating_mul(4)),
321            );
322        }
323        if is_svc(u32::from_le(w3)) {
324            hits.push(
325                base_off.saturating_add(i.saturating_add(3).saturating_mul(4)),
326            );
327        }
328        i = i.saturating_add(4);
329    }
330    while i < words {
331        let w = unsafe { base.add(i.saturating_mul(4)).cast::<u32>().read_unaligned() };
332        if is_svc(u32::from_le(w)) {
333            hits.push(base_off.saturating_add(i.saturating_mul(4)));
334        }
335        i = i.saturating_add(1);
336    }
337    hits
338}
339
340/// Round `len` up to a multiple of `page` (page must be a power of two ≥ 1).
341#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
342#[inline]
343fn align_up_to_page(len: usize, page: usize) -> usize {
344    if page <= 1 {
345        return len.max(1);
346    }
347    let mask = page.saturating_sub(1);
348    len.saturating_add(mask) & !mask
349}
350
351// libgcc / compiler-rt symbol linked on Linux aarch64 toolchains.
352#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
353unsafe extern "C" {
354    fn __clear_cache(start: *mut libc::c_void, end: *mut libc::c_void);
355}
356
357/// Flush D/I caches after writing executable code (aarch64 requirement).
358#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
359fn clear_icache(ptr: *mut u8, len: usize) {
360    if ptr.is_null() || len == 0 {
361        return;
362    }
363    // SAFETY: range is a live mapping the caller owns and just wrote.
364    unsafe {
365        let start = ptr.cast::<libc::c_void>();
366        let end = ptr.wrapping_add(len).cast::<libc::c_void>();
367        __clear_cache(start, end);
368    }
369}
370
371#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
372fn clear_icache(_ptr: *mut u8, _len: usize) {}
373
374#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
375#[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
376fn trampoline_rust_entry_addr() -> u64 {
377    u64::try_from(kh_trampoline_dispatch as usize).unwrap_or(0)
378}
379
380/// Return value of the trampoline dispatch (x0=retval, x1=error flag).
381#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
382#[repr(C)]
383pub struct TrampRet {
384    /// Value for guest `x0`.
385    pub retval: u64,
386    /// Non-zero → set Darwin carry (error path).
387    pub error: u64,
388}
389
390/// Called from the hypercall NEON tramp (`TRAMP_BYTES`) on the host alt stack.
391///
392/// # Safety
393///
394/// Caller must preserve guest SIMD/GPRs around the call (hypercall prolog +
395/// mapped NEON tramp). Arguments are Darwin BSD syscall registers; `x16`
396/// holds the syscall number (or AAPCS 8th arg after the hypercall prologue).
397/// May end the current host worker thread on `bsdthread_terminate`
398/// (`SYS_exit`, not `pthread_exit`), or `_exit` the process on guest `exit`.
399///
400/// Guest entry is [`kh_hypercall_entry`]: alt stack + TLS boundary before this.
401#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
402#[unsafe(no_mangle)]
403pub unsafe extern "C" fn kh_trampoline_dispatch(
404    x0: u64,
405    x1: u64,
406    x2: u64,
407    x3: u64,
408    x4: u64,
409    x5: u64,
410    x6: u64,
411    x16: u64,
412) -> TrampRet {
413    let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
414    let result = syscall::dispatch(SyscallArgs {
415        pc: 0,
416        number: sys_no,
417        x0,
418        x1,
419        x2,
420        x3,
421        x4,
422        x5,
423        x6,
424    });
425    if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
426        maybe_push_event(
427            0,
428            sys_no,
429            result.name,
430            x0,
431            x1,
432            x2,
433            result.retval,
434            result.error,
435        );
436    }
437    match result.outcome {
438        TrapOutcome::Exit { code } => finish_with_exit_code(code),
439        TrapOutcome::ThreadExit => {
440            // End this host worker only (same as bsdthread_terminate).
441            crate::thread::exit_current_guest_worker();
442        }
443        TrapOutcome::Continue => {}
444    }
445    TrampRet {
446        retval: result.retval.unwrap_or(0),
447        error: u64::from(result.error),
448    }
449}
450
451// Freestanding hypercall entry (`kh_hypercall_entry`):
452// Prefer host alt stack (so join/terminate never runs host Rust on a guest
453// stack that `pthread_join` may unmap).
454//
455// **NEON:** Darwin `svc` preserves Q0–Q31. We must too. Any `bl` into Rust/C
456// (TLS enter, alt-stack map) clobbers caller-saved SIMD under AAPCS64, so full
457// Q0–Q31 + FPCR/FPSR are saved on the *guest* prolog frame *before* any `bl`
458// (invariant 13). Dispatch goes through mapped `TRAMP_BYTES` (second full NEON
459// frame on the host/alt stack). A former "light" skip of the second save
460// showed no wall win on UTM 8k and was removed to keep one code path.
461//
462// ABI: x0–x6 + number in x7 → TrampRet in x0/x1.
463// Guest frame layout (640 B):
464//   [0]   x29,x30
465//   [16]  x0,x1  [32] x2,x3  [48] x4,x5  [64] x6,x7
466//   [80]  pad
467//   [96]  q0..q31 (512 B)
468//   [608] fpcr,fpsr
469#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
470std::arch::global_asm!(
471    r#"
472    .text
473    .align 2
474    .global kh_hypercall_entry
475    .type kh_hypercall_entry, %function
476kh_hypercall_entry:
477    // Guest prolog: GPRs + full NEON *before* any bl (NEON must survive host TLS).
478    sub sp, sp, #640
479    stp x29, x30, [sp, #0]
480    stp x0, x1, [sp, #16]
481    stp x2, x3, [sp, #32]
482    stp x4, x5, [sp, #48]
483    stp x6, x7, [sp, #64]
484    mov x29, sp
485    stp q0, q1, [sp, #96]
486    stp q2, q3, [sp, #128]
487    stp q4, q5, [sp, #160]
488    stp q6, q7, [sp, #192]
489    stp q8, q9, [sp, #224]
490    stp q10, q11, [sp, #256]
491    stp q12, q13, [sp, #288]
492    stp q14, q15, [sp, #320]
493    stp q16, q17, [sp, #352]
494    stp q18, q19, [sp, #384]
495    stp q20, q21, [sp, #416]
496    stp q22, q23, [sp, #448]
497    stp q24, q25, [sp, #480]
498    stp q26, q27, [sp, #512]
499    stp q28, q29, [sp, #544]
500    stp q30, q31, [sp, #576]
501    mrs x9, fpcr
502    mrs x10, fpsr
503    // #608 exceeds signed LDP imm7*8 range (±512); use base+add.
504    add x11, sp, #608
505    stp x9, x10, [x11]
506
507    // A1/A2: enter returns alt_top in x0, guest_tpidr in x1 (HyperEnterRet).
508    // Park guest_va at prolog+80 (hole before NEON) so leave needs no gettid.
509    bl kh_tls_enter_host
510    mov x9, sp                 // guest prolog frame
511    str x1, [x9, #80]          // guest_tpidr for leave
512    cbz x0, 1f
513    mov sp, x0                 // host alt stack
514    ldr x30, [x9, #8]          // reload LR → freestanding thin caller
515    // Host frame: [guest_frame, lr]
516    stp x9, x30, [sp, #-16]!
517    // Reload guest frame ptr / args (enter_host clobbered temps).
518    ldr x9, [sp]
519    ldp x0, x1, [x9, #16]
520    ldp x2, x3, [x9, #32]
521    ldp x4, x5, [x9, #48]
522    ldp x6, x7, [x9, #64]
523    mov x16, x7
524    bl kh_neon_tramp_entry
525    // Preserve TrampRet (x0/x1) across TLS restore.
526    stp x0, x1, [sp, #-16]!
527    // guest_va at host_frame.guest_frame + 80; host_frame at sp+16.
528    ldr x9, [sp, #16]
529    ldr x0, [x9, #80]
530    bl kh_tls_leave_host
531    ldp x0, x1, [sp], #16
532    ldp x9, x30, [sp], #16
533    mov sp, x9
534    // Restore guest NEON from prolog (pre-bl snapshot).
535    ldp q0, q1, [sp, #96]
536    ldp q2, q3, [sp, #128]
537    ldp q4, q5, [sp, #160]
538    ldp q6, q7, [sp, #192]
539    ldp q8, q9, [sp, #224]
540    ldp q10, q11, [sp, #256]
541    ldp q12, q13, [sp, #288]
542    ldp q14, q15, [sp, #320]
543    ldp q16, q17, [sp, #352]
544    ldp q18, q19, [sp, #384]
545    ldp q20, q21, [sp, #416]
546    ldp q22, q23, [sp, #448]
547    ldp q24, q25, [sp, #480]
548    ldp q26, q27, [sp, #512]
549    ldp q28, q29, [sp, #544]
550    ldp q30, q31, [sp, #576]
551    add x11, sp, #608
552    ldp x9, x10, [x11]
553    msr fpcr, x9
554    msr fpsr, x10
555    ldr x29, [sp, #0]
556    add sp, sp, #640
557    ret
5581:
559    // No alt stack: dispatch on the guest stack (ST / fallback only).
560    // Already on host TLS from the enter above; guest_va at [sp, #80].
561    ldp x0, x1, [sp, #16]
562    ldp x2, x3, [sp, #32]
563    ldp x4, x5, [sp, #48]
564    ldp x6, x7, [sp, #64]
565    mov x16, x7
566    // Preserve real LR across the call (still on guest prolog frame).
567    ldr x30, [sp, #8]
568    bl kh_neon_tramp_entry
569    stp x0, x1, [sp, #-16]!
570    ldr x0, [sp, #16+80]
571    bl kh_tls_leave_host
572    ldp x0, x1, [sp], #16
573    // Restore guest NEON from prolog.
574    ldp q0, q1, [sp, #96]
575    ldp q2, q3, [sp, #128]
576    ldp q4, q5, [sp, #160]
577    ldp q6, q7, [sp, #192]
578    ldp q8, q9, [sp, #224]
579    ldp q10, q11, [sp, #256]
580    ldp q12, q13, [sp, #288]
581    ldp q14, q15, [sp, #320]
582    ldp q16, q17, [sp, #352]
583    ldp q18, q19, [sp, #384]
584    ldp q20, q21, [sp, #416]
585    ldp q22, q23, [sp, #448]
586    ldp q24, q25, [sp, #480]
587    ldp q26, q27, [sp, #512]
588    ldp q28, q29, [sp, #544]
589    ldp q30, q31, [sp, #576]
590    add x11, sp, #608
591    ldp x9, x10, [x11]
592    msr fpcr, x9
593    msr fpsr, x10
594    ldp x29, x30, [sp, #0]
595    add sp, sp, #640
596    ret
597    .size kh_hypercall_entry, .-kh_hypercall_entry
598    "#
599);
600
601// `kh_neon_tramp_entry` + `KH_NEON_TRAMP_VA`: branch into mapped `TRAMP_BYTES`
602// (full Q* save around Rust dispatch). Fallback if tramp not mapped yet:
603// tail-call `kh_trampoline_dispatch` (still safe: prolog owns guest Q*).
604#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
605std::arch::global_asm!(
606    r#"
607    .text
608    .align 2
609    .global kh_neon_tramp_entry
610    .type kh_neon_tramp_entry, %function
611kh_neon_tramp_entry:
612    adrp x17, KH_NEON_TRAMP_VA
613    ldr x17, [x17, :lo12:KH_NEON_TRAMP_VA]
614    cbz x17, 1f
615    br x17
6161:
617    // No-tramp fallback: number in x16 → AAPCS 8th arg x7; tail call.
618    mov x7, x16
619    b kh_trampoline_dispatch
620    .size kh_neon_tramp_entry, .-kh_neon_tramp_entry
621
622    .data
623    .align 3
624    .global KH_NEON_TRAMP_VA
625KH_NEON_TRAMP_VA:
626    .quad 0
627    "#
628);
629
630#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
631unsafe extern "C" {
632    fn kh_hypercall_entry();
633    static mut KH_NEON_TRAMP_VA: u64;
634}
635
636/// Address of freestanding hypercall entry (alt stack + NEON tramp + dispatch).
637#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
638#[must_use]
639pub fn hypercall_entry_addr() -> u64 {
640    ensure_neon_tramp();
641    #[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
642    {
643        u64::try_from(kh_hypercall_entry as usize).unwrap_or(0)
644    }
645}
646
647/// Map TRAMP_BYTES once and publish its VA to `KH_NEON_TRAMP_VA`.
648#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
649fn ensure_neon_tramp() {
650    static ONCE: AtomicU64 = AtomicU64::new(0);
651    static LOCK: Mutex<()> = Mutex::new(());
652
653    if ONCE.load(Ordering::Acquire) != 0 {
654        return;
655    }
656    let _g = LOCK
657        .lock()
658        .unwrap_or_else(std::sync::PoisonError::into_inner);
659    if ONCE.load(Ordering::Acquire) != 0 {
660        return;
661    }
662
663    let code = TRAMP_BYTES;
664    let page = crate::host::page_size().unwrap_or(4096);
665    let map_len = align_up_to_page(code.len(), page).max(page);
666    let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
667    let prot_rw = libc::PROT_READ | libc::PROT_WRITE;
668    let Some(ptr) = crate::host::mmap(None, map_len, prot_rw, flags, -1, 0) else {
669        tracing::error!("NEON tramp mmap failed");
670        return;
671    };
672    // SAFETY: fresh RW mapping.
673    let slice = unsafe { std::slice::from_raw_parts_mut(ptr, map_len) };
674    slice.fill(0);
675    if let Some(dst) = slice.get_mut(..code.len()) {
676        dst.copy_from_slice(code);
677    } else {
678        let _ = crate::host::munmap(ptr, map_len);
679        return;
680    }
681    if let Some(slot) = slice.get_mut(TRAMP_RELOC_OFF..TRAMP_RELOC_OFF.saturating_add(8)) {
682        slot.copy_from_slice(&trampoline_rust_entry_addr().to_le_bytes());
683    } else {
684        let _ = crate::host::munmap(ptr, map_len);
685        return;
686    }
687    let prot_rx = libc::PROT_READ | libc::PROT_EXEC;
688    if !crate::host::mprotect(ptr, map_len, prot_rx) {
689        tracing::error!("NEON tramp mprotect RX failed");
690        let _ = crate::host::munmap(ptr, map_len);
691        return;
692    }
693    clear_icache(ptr, map_len);
694    let va = crate::host::ptr_addr_u64(ptr);
695    // SAFETY: symbol from global_asm; written once under LOCK before guest entry.
696    unsafe {
697        KH_NEON_TRAMP_VA = va;
698    }
699    ONCE.store(va, Ordering::Release);
700    tracing::info!(
701        va = format_args!("{va:#x}"),
702        "mapped NEON tramp for hypercall"
703    );
704}
705
706#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
707#[must_use]
708pub fn hypercall_entry_addr() -> u64 {
709    0
710}
711
712// ── trampoline machine code (verified via rustc/objdump aarch64) ─────────────
713
714/// Offset of the 8-byte absolute `kh_trampoline_dispatch` pointer.
715///
716/// Layout (see `TRAMP_BYTES`): `bti c` + save GPRs/NEON/FPCR + `blr` dispatch
717/// + restore + `ret` + `.quad` reloc @ 0x100.
718#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
719const TRAMP_RELOC_OFF: usize = 0x100;
720
721/// Exact opcodes from assembled trampoline (`as` / objdump on Linux aarch64).
722///
723/// Saves **all Q0–Q31 + FPCR/FPSR** around the Rust call: Darwin `svc` preserves
724/// SIMD state; compression workers (7zz MT) rely on that. A bare GPR-only
725/// trampoline clobbered NEON and faulted under `-mmt>1`.
726#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
727#[rustfmt::skip]
728const TRAMP_BYTES: &[u8] = &[
729    0x5f, 0x24, 0x03, 0xd5, 0xff, 0x03, 0x0a, 0xd1, 0xe8, 0x27, 0x00, 0xa9, 0xea, 0x2f, 0x01, 0xa9,
730    0xec, 0x37, 0x02, 0xa9, 0xee, 0x3f, 0x03, 0xa9, 0xf1, 0x4b, 0x04, 0xa9, 0xfe, 0x2b, 0x00, 0xf9,
731    0x08, 0x44, 0x3b, 0xd5, 0x29, 0x44, 0x3b, 0xd5, 0xe8, 0x27, 0x06, 0xa9, 0xe0, 0x87, 0x03, 0xad,
732    0xe2, 0x8f, 0x04, 0xad, 0xe4, 0x97, 0x05, 0xad, 0xe6, 0x9f, 0x06, 0xad, 0xe8, 0xa7, 0x07, 0xad,
733    0xea, 0xaf, 0x08, 0xad, 0xec, 0xb7, 0x09, 0xad, 0xee, 0xbf, 0x0a, 0xad, 0xf0, 0xc7, 0x0b, 0xad,
734    0xf2, 0xcf, 0x0c, 0xad, 0xf4, 0xd7, 0x0d, 0xad, 0xf6, 0xdf, 0x0e, 0xad, 0xf8, 0xe7, 0x0f, 0xad,
735    0xfa, 0xef, 0x10, 0xad, 0xfc, 0xf7, 0x11, 0xad, 0xfe, 0xff, 0x12, 0xad, 0xe7, 0x03, 0x10, 0xaa,
736    0x88, 0x04, 0x00, 0x58, 0x00, 0x01, 0x3f, 0xd6, 0x02, 0x42, 0x3b, 0xd5, 0x23, 0x00, 0x80, 0x52,
737    0x63, 0x88, 0x63, 0xd3, 0x42, 0x00, 0x23, 0x8a, 0x41, 0x00, 0x00, 0xb4, 0x42, 0x00, 0x03, 0xaa,
738    0x02, 0x42, 0x1b, 0xd5, 0xe8, 0x27, 0x46, 0xa9, 0x08, 0x44, 0x1b, 0xd5, 0x29, 0x44, 0x1b, 0xd5,
739    0xe0, 0x87, 0x43, 0xad, 0xe2, 0x8f, 0x44, 0xad, 0xe4, 0x97, 0x45, 0xad, 0xe6, 0x9f, 0x46, 0xad,
740    0xe8, 0xa7, 0x47, 0xad, 0xea, 0xaf, 0x48, 0xad, 0xec, 0xb7, 0x49, 0xad, 0xee, 0xbf, 0x4a, 0xad,
741    0xf0, 0xc7, 0x4b, 0xad, 0xf2, 0xcf, 0x4c, 0xad, 0xf4, 0xd7, 0x4d, 0xad, 0xf6, 0xdf, 0x4e, 0xad,
742    0xf8, 0xe7, 0x4f, 0xad, 0xfa, 0xef, 0x50, 0xad, 0xfc, 0xf7, 0x51, 0xad, 0xfe, 0xff, 0x52, 0xad,
743    0xfe, 0x2b, 0x40, 0xf9, 0xf1, 0x4b, 0x44, 0xa9, 0xee, 0x3f, 0x43, 0xa9, 0xec, 0x37, 0x42, 0xa9,
744    0xea, 0x2f, 0x41, 0xa9, 0xe8, 0x27, 0x40, 0xa9, 0xff, 0x03, 0x0a, 0x91, 0xc0, 0x03, 0x5f, 0xd6,
745    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // reloc .quad @ 0x100
746];
747/// Installs the `SIGTRAP` handler used by the micro execution backend.
748pub fn install_trap_handlers(config: &TrapConfig) -> Result<(), TrapError> {
749    MAX_EVENTS.store(
750        u64::try_from(config.max_events).unwrap_or(256),
751        Ordering::SeqCst,
752    );
753    syscall::reset_syscall_state(config.max_syscalls);
754
755    #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
756    {
757        let _ = config;
758        Err(TrapError::UnsupportedArch)
759    }
760
761    #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
762    {
763        if HANDLER_INSTALLED.swap(true, Ordering::SeqCst) {
764            reset_trap_state();
765            syscall::reset_syscall_state(config.max_syscalls);
766            return Ok(());
767        }
768
769        // Guest faults used to leave multi‑MB `core` files in the workspace cwd.
770        // Faults go through our handler → `_exit`; disable dumps as belt-and-braces.
771        disable_core_dumps();
772
773        // SAFETY: process-wide SIGTRAP / fault handlers with SA_SIGINFO.
774        unsafe {
775            let mut sa: libc::sigaction = std::mem::zeroed();
776            sa.sa_flags = libc::SA_SIGINFO;
777            #[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
778            {
779                sa.sa_sigaction = trap_sigaction as *const () as usize;
780            }
781            libc::sigemptyset(std::ptr::addr_of_mut!(sa.sa_mask));
782            if libc::sigaction(libc::SIGTRAP, std::ptr::addr_of!(sa), std::ptr::null_mut()) != 0 {
783                HANDLER_INSTALLED.store(false, Ordering::SeqCst);
784                return Err(TrapError::SignalSetup(io::Error::last_os_error()));
785            }
786
787            // Guest faults: print PC/addr so `kh run` diagnoses unbound GOT / bad heap.
788            let mut fault: libc::sigaction = std::mem::zeroed();
789            fault.sa_flags = libc::SA_SIGINFO;
790            #[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
791            {
792                fault.sa_sigaction = guest_fault_sigaction as *const () as usize;
793            }
794            libc::sigemptyset(std::ptr::addr_of_mut!(fault.sa_mask));
795            for sig in [libc::SIGSEGV, libc::SIGBUS] {
796                if libc::sigaction(sig, std::ptr::addr_of!(fault), std::ptr::null_mut()) != 0 {
797                    HANDLER_INSTALLED.store(false, Ordering::SeqCst);
798                    return Err(TrapError::SignalSetup(io::Error::last_os_error()));
799                }
800            }
801        }
802        reset_trap_state();
803        Ok(())
804    }
805}
806
807/// Sets `RLIMIT_CORE` soft+hard to 0 so guest crashes never litter the repo with
808/// `core` dumps (we already print a guest fault summary and `_exit`).
809#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
810fn disable_core_dumps() {
811    // SAFETY: process-wide rlimit; best-effort, ignore failure.
812    unsafe {
813        let lim = libc::rlimit {
814            rlim_cur: 0,
815            rlim_max: 0,
816        };
817        let _ = libc::setrlimit(libc::RLIMIT_CORE, std::ptr::addr_of!(lim));
818    }
819}
820
821/// Logs guest `SIGSEGV` / `SIGBUS` (PC, fault address, key regs) then exits.
822#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
823unsafe extern "C" fn guest_fault_sigaction(
824    signo: libc::c_int,
825    info: *mut libc::siginfo_t,
826    ctx: *mut libc::c_void,
827) {
828    // May run under guest TPIDR; restore host before formatting / write.
829    crate::tls::enter_host_tls();
830    let name = match signo {
831        libc::SIGSEGV => "SIGSEGV",
832        libc::SIGBUS => "SIGBUS",
833        _ => "FAULT",
834    };
835    let mut pc = 0_u64;
836    let mut sp = 0_u64;
837    let mut x0 = 0_u64;
838    let mut x1 = 0_u64;
839    let mut x2 = 0_u64;
840    let mut x8 = 0_u64;
841    let mut x16 = 0_u64;
842    let mut x19 = 0_u64;
843    let mut x3 = 0_u64;
844    let mut x4 = 0_u64;
845    let mut x5 = 0_u64;
846    let mut x6 = 0_u64;
847    let mut x7 = 0_u64;
848    let mut x9 = 0_u64;
849    let mut x20 = 0_u64;
850    let mut x21 = 0_u64;
851    let mut x22 = 0_u64;
852    let mut x23 = 0_u64;
853    let mut x24 = 0_u64;
854    let mut x25 = 0_u64;
855    let mut x26 = 0_u64;
856    let mut x27 = 0_u64;
857    let mut x28 = 0_u64;
858    let mut x29 = 0_u64;
859    let mut lr = 0_u64;
860    if !ctx.is_null() {
861        // SAFETY: kernel `ucontext_t` for SA_SIGINFO.
862        let uctx = unsafe { &*ctx.cast::<libc::ucontext_t>() };
863        let m = &uctx.uc_mcontext;
864        pc = m.pc;
865        sp = m.sp;
866        x0 = m.regs[0];
867        x1 = m.regs[1];
868        x2 = m.regs[2];
869        x3 = m.regs[3];
870        x4 = m.regs[4];
871        x5 = m.regs[5];
872        x6 = m.regs[6];
873        x7 = m.regs[7];
874        x8 = m.regs[8];
875        x9 = m.regs[9];
876        x16 = m.regs[16];
877        x19 = m.regs[19];
878        x20 = m.regs[20];
879        x21 = m.regs[21];
880        x22 = m.regs[22];
881        x23 = m.regs[23];
882        x24 = m.regs[24];
883        x25 = m.regs[25];
884        x26 = m.regs[26];
885        x27 = m.regs[27];
886        x28 = m.regs[28];
887        x29 = m.regs[29];
888        lr = m.regs[30];
889    }
890    let addr = if info.is_null() {
891        0_u64
892    } else {
893        // SAFETY: kernel-provided siginfo.
894        let p = unsafe { (*info).si_addr() };
895        u64::try_from(p.addr()).unwrap_or(0)
896    };
897    let msg = format!(
898        "error: guest {name} pc={pc:#x} addr={addr:#x} sp={sp:#x} lr={lr:#x} \
899         x0={x0:#x} x1={x1:#x} x2={x2:#x} x3={x3:#x} x4={x4:#x} x5={x5:#x} \
900         x6={x6:#x} x7={x7:#x} x8={x8:#x} x9={x9:#x} x16={x16:#x} \
901         x19={x19:#x} x20={x20:#x} x21={x21:#x} x22={x22:#x} x23={x23:#x} \
902         x24={x24:#x} x25={x25:#x} x26={x26:#x} x27={x27:#x} x28={x28:#x} x29={x29:#x}\n"
903    );
904    drop(io::stderr().write_all(msg.as_bytes()));
905    // Best-effort: dump AArch64 insn at PC via /proc/self/mem (no raw guest deref).
906    if pc != 0 {
907        if let Ok(mut f) = std::fs::File::open("/proc/self/mem") {
908            use std::io::{Read, Seek, SeekFrom};
909            let mut insn = [0_u8; 4];
910            if f.seek(SeekFrom::Start(pc)).is_ok() && f.read_exact(&mut insn).is_ok() {
911                let w = u32::from_le_bytes(insn);
912                let m = format!("  insn@{pc:#x} = {w:#010x} bytes={insn:02x?}\n");
913                drop(io::stderr().write_all(m.as_bytes()));
914            }
915        }
916    }
917    // Best-effort: which mapping owns PC / fault address / key regs.
918    if let Ok(maps) = std::fs::read_to_string("/proc/self/maps") {
919        for &va in &[pc, addr, lr, x0, x1, x2, x16, x19, x25] {
920            if va == 0 {
921                continue;
922            }
923            for line in maps.lines() {
924                let Some((range, rest)) = line.split_once(' ') else {
925                    continue;
926                };
927                let Some((a, b)) = range.split_once('-') else {
928                    continue;
929                };
930                let (Ok(lo), Ok(hi)) = (u64::from_str_radix(a, 16), u64::from_str_radix(b, 16))
931                else {
932                    continue;
933                };
934                if va >= lo && va < hi {
935                    let off = va.saturating_sub(lo);
936                    let m = format!("  map {va:#x} = {lo:#x}+{off:#x} {rest}\n");
937                    drop(io::stderr().write_all(m.as_bytes()));
938                    break;
939                }
940            }
941        }
942    }
943    drop(io::stderr().flush());
944    if TRACE_ON_EXIT.load(Ordering::SeqCst) {
945        dump_events_before_exit();
946    }
947    // SAFETY: hard-stop after unrecoverable guest fault.
948    unsafe { libc::_exit(128_i32.saturating_add(signo)) };
949}
950
951/// Drains recorded trap events (for `kh trace`).
952#[must_use]
953pub fn take_trace_events() -> Vec<TrapEvent> {
954    EVENTS
955        .lock()
956        .map(|mut g| std::mem::take(&mut *g))
957        .unwrap_or_default()
958}
959
960/// Whether the guest requested exit.
961#[must_use]
962pub fn exit_requested() -> bool {
963    EXIT_REQUESTED.load(Ordering::SeqCst)
964}
965
966/// Exit code from the last guest `exit` trap.
967#[must_use]
968pub fn take_exit_code() -> Option<i32> {
969    if EXIT_REQUESTED.load(Ordering::SeqCst) {
970        Some(EXIT_CODE.load(Ordering::SeqCst))
971    } else {
972        None
973    }
974}
975
976/// Resets exit / event state before a new run.
977pub fn reset_trap_state() {
978    EXIT_REQUESTED.store(false, Ordering::SeqCst);
979    EXIT_CODE.store(0, Ordering::SeqCst);
980    EVENT_COUNT.store(0, Ordering::SeqCst);
981    if let Ok(mut guard) = EVENTS.lock() {
982        guard.clear();
983    }
984}
985
986/// When set, the exit trap prints recorded events to stdout before `_exit`.
987pub fn set_trace_on_exit(json: bool) {
988    TRACE_ON_EXIT.store(true, Ordering::SeqCst);
989    TRACE_JSON.store(json, Ordering::SeqCst);
990}
991
992/// Clears the trace-on-exit flag (default for plain `kh run`).
993pub fn clear_trace_on_exit() {
994    TRACE_ON_EXIT.store(false, Ordering::SeqCst);
995    TRACE_JSON.store(false, Ordering::SeqCst);
996}
997
998/// Sets the expected guest exit code for the micro gate (`--expect-code`).
999pub fn set_expect_code(code: i32) {
1000    EXPECT_CODE.store(code, Ordering::SeqCst);
1001}
1002
1003/// Disables the expect-code gate.
1004pub fn clear_expect_code() {
1005    EXPECT_CODE.store(i32::MIN, Ordering::SeqCst);
1006}
1007
1008/// Terminates the host process as if the guest called BSD `exit(code)`.
1009///
1010/// Used after `LC_MAIN` returns (dyld calls `exit` with `main`'s return value).
1011/// Honours `--expect-code` and optional trace-on-exit dump.
1012pub fn finish_with_exit_code(code: i32) -> ! {
1013    EXIT_CODE.store(code, Ordering::SeqCst);
1014    EXIT_REQUESTED.store(true, Ordering::SeqCst);
1015    if TRACE_ON_EXIT.load(Ordering::SeqCst) {
1016        dump_events_before_exit();
1017    }
1018    // Host TPIDR: opt-in diagnosis (futex classes; boundary crossing counts).
1019    crate::syscall::dump_futex_stats_if_enabled();
1020    crate::syscall::dump_boundary_stats_if_enabled();
1021    let expect = EXPECT_CODE.load(Ordering::SeqCst);
1022    let status = if expect == i32::MIN || expect == code {
1023        code
1024    } else {
1025        let msg = format!("error: guest exit code {code} != expected {expect}\n");
1026        drop(io::stderr().write_all(msg.as_bytes()));
1027        1
1028    };
1029    // SAFETY: intentional process termination after guest main return / exit.
1030    unsafe { libc::_exit(status) };
1031}
1032
1033#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
1034unsafe extern "C" fn trap_sigaction(
1035    signo: libc::c_int,
1036    _info: *mut libc::siginfo_t,
1037    ctx: *mut libc::c_void,
1038) {
1039    if signo != libc::SIGTRAP || ctx.is_null() {
1040        return;
1041    }
1042
1043    // Guest may have TPIDR_EL0 = guest TSD; restore host TLS before Rust/libc.
1044    crate::tls::enter_host_tls();
1045
1046    // SAFETY: kernel provides a valid `ucontext_t` for SA_SIGINFO deliveries.
1047    let uctx = unsafe { &mut *ctx.cast::<libc::ucontext_t>() };
1048    let m = &mut uctx.uc_mcontext;
1049    let pc = m.pc;
1050    // Only pull the regs dispatch needs; avoid extra loads on the hot path.
1051    let x0 = m.regs[0];
1052    let x1 = m.regs[1];
1053    let x2 = m.regs[2];
1054    let x3 = m.regs[3];
1055    let x4 = m.regs[4];
1056    let x5 = m.regs[5];
1057    let x6 = m.regs[6];
1058    let x16 = m.regs[16];
1059
1060    let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
1061    let result = syscall::dispatch(SyscallArgs {
1062        pc,
1063        number: sys_no,
1064        x0,
1065        x1,
1066        x2,
1067        x3,
1068        x4,
1069        x5,
1070        x6,
1071    });
1072
1073    // Zero-cost when max_events == 0 (plain `kh run`).
1074    if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
1075        maybe_push_event(
1076            pc,
1077            sys_no,
1078            result.name,
1079            x0,
1080            x1,
1081            x2,
1082            result.retval,
1083            result.error,
1084        );
1085    }
1086
1087    if let Some(ret) = result.retval {
1088        m.regs[0] = ret;
1089    }
1090    // Darwin arm64: error → set C; success with write-back → clear C.
1091    if result.error {
1092        m.pstate |= PSTATE_C;
1093    } else if result.retval.is_some() {
1094        m.pstate &= !PSTATE_C;
1095    }
1096
1097    match result.outcome {
1098        TrapOutcome::Exit { code } => {
1099            finish_with_exit_code(code);
1100        }
1101        TrapOutcome::Continue => {
1102            m.pc = pc.wrapping_add(4);
1103            // Return to guest with guest TPIDR restored.
1104            crate::tls::leave_host_tls();
1105        }
1106        TrapOutcome::ThreadExit => {
1107            // Stay on host TLS; exit trampoline publishes join then pthread_exit.
1108            // Redirect ucontext to a host `pthread_exit` trampoline on this
1109            // thread's original stack (see `crate::thread`). Main thread
1110            // (no host frame) falls back to process exit.
1111            if !crate::thread::redirect_ucontext_to_host_exit(m) {
1112                finish_with_exit_code(0);
1113            }
1114        }
1115    }
1116}
1117
1118fn dump_events_before_exit() {
1119    use std::fmt::Write as _;
1120
1121    let json = TRACE_JSON.load(Ordering::SeqCst);
1122    let events = EVENTS.lock().map(|g| g.clone()).unwrap_or_default();
1123    if json {
1124        let mut out = String::from("{\"events\":[");
1125        for (i, e) in events.iter().enumerate() {
1126            if i > 0 {
1127                out.push(',');
1128            }
1129            let sys = e
1130                .syscall
1131                .map_or_else(|| "null".to_owned(), |s| s.to_string());
1132            let ret = e
1133                .retval
1134                .map_or_else(|| "null".to_owned(), |v| v.to_string());
1135            let _ = write!(
1136                out,
1137                "{{\"pc\":{},\"syscall\":{},\"name\":\"{}\",\"arg0\":{},\"arg1\":{},\"arg2\":{},\"retval\":{},\"error\":{}}}",
1138                e.pc, sys, e.name, e.arg0, e.arg1, e.arg2, ret, e.error
1139            );
1140        }
1141        out.push_str("]}\n");
1142        drop(io::stdout().write_all(out.as_bytes()));
1143        drop(io::stdout().flush());
1144    } else {
1145        let header = format!("trace events: {}\n", events.len());
1146        drop(io::stdout().write_all(header.as_bytes()));
1147        for (i, e) in events.iter().enumerate() {
1148            let ret = e
1149                .retval
1150                .map_or_else(|| "-".to_owned(), |v| format!("{v:#x}"));
1151            let err = if e.error { " err" } else { "" };
1152            let line = format!(
1153                "  [{i}] pc={:#x} {} sys={:?} args=({:#x}, {:#x}, {:#x}) ret={ret}{err}\n",
1154                e.pc, e.name, e.syscall, e.arg0, e.arg1, e.arg2
1155            );
1156            drop(io::stdout().write_all(line.as_bytes()));
1157        }
1158        drop(io::stdout().flush());
1159    }
1160}
1161
1162#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
1163#[allow(clippy::too_many_arguments)]
1164fn maybe_push_event(
1165    pc: u64,
1166    sys_no: u32,
1167    name: &'static str,
1168    arg0: u64,
1169    arg1: u64,
1170    arg2: u64,
1171    retval: Option<u64>,
1172    error: bool,
1173) {
1174    let max = MAX_EVENTS.load(Ordering::Relaxed);
1175    // `kh run` sets max_events=0: single load, no String/Mutex.
1176    if max == 0 {
1177        return;
1178    }
1179    let n = EVENT_COUNT.fetch_add(1, Ordering::Relaxed);
1180    if n >= max {
1181        return;
1182    }
1183    if let Ok(mut guard) = EVENTS.lock()
1184        && u64::try_from(guard.len()).unwrap_or(u64::MAX) < max
1185    {
1186        guard.push(TrapEvent {
1187            pc,
1188            syscall: Some(sys_no),
1189            name: name.to_owned(),
1190            arg0,
1191            arg1,
1192            arg2,
1193            retval,
1194            error,
1195        });
1196    }
1197}
1198
1199#[cfg(test)]
1200mod tests {
1201    use super::*;
1202
1203    #[test]
1204    fn detects_svc_encoding() {
1205        assert!(is_svc(0xD400_0001));
1206        let svc80 = 0xD400_0001 | (0x80_u32 << 5);
1207        assert!(is_svc(svc80));
1208        assert!(!is_svc(0xD503_201F));
1209        assert!(!is_svc(brk_encoding(BRK_TRAP_IMM)));
1210    }
1211
1212    #[test]
1213    fn brk_encoding_roundtrip_imm() {
1214        let enc = brk_encoding(BRK_TRAP_IMM);
1215        let imm = (enc >> 5) & 0xFFFF;
1216        assert_eq!(imm, BRK_TRAP_IMM);
1217    }
1218}