Skip to main content

kh_runtime/cpu/
trap.rs

1//! Trap backend: residual Darwin `svc` → `brk`, and freestanding hypercall entry.
2//!
3//! Production: freestanding libSystem → `kh_hypercall_entry` (alt stack + NEON
4//! tramp + [`kh_trampoline_dispatch`]). Residual unpatched `svc` is rewritten to
5//! `brk #IMM` and handled via `SIGTRAP` (not a second product path).
6//! Live handlers require **Linux aarch64**; patching works on any host.
7#![allow(unsafe_code)]
8
9use std::io::{self, Write};
10use std::sync::Mutex;
11use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering};
12
13use thiserror::Error;
14
15use crate::mem::{
16    MapError, MappedRegion, VM_PROT_EXECUTE, VM_PROT_READ, VM_PROT_WRITE, mprotect_darwin,
17    mprotect_rw,
18};
19use crate::syscall;
20#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
21use crate::syscall::SyscallArgs;
22
23/// Marker immediate for patched traps (`brk #0x0F00`).
24pub const BRK_TRAP_IMM: u32 = 0x0F00;
25
26/// AArch64 `svc #imm16` base encoding: `0xD4000001 | (imm16 << 5)`.
27const SVC_MASK: u32 = 0xFFE0_001F;
28const SVC_BASE: u32 = 0xD400_0001;
29
30/// AArch64 `brk #imm16`: `0xD4200000 | (imm16 << 5)`.
31const fn brk_encoding(imm16: u32) -> u32 {
32    0xD420_0000 | ((imm16 & 0xFFFF) << 5)
33}
34
35/// AArch64 `PSTATE` / CPSR carry flag (bit 29 of NZCV).
36///
37/// Darwin arm64 syscall convention: set on error (errno in `x0`), clear on success.
38pub const PSTATE_C: u64 = 1 << 29;
39
40/// One observed trap / translated syscall event.
41#[derive(Debug, Clone, PartialEq, Eq)]
42pub struct TrapEvent {
43    /// Guest PC at the trap (instruction address).
44    pub pc: u64,
45    /// Darwin BSD syscall number (`x16`) when recognized; otherwise `None`.
46    pub syscall: Option<u32>,
47    /// Short label (`exit`, `write`, `unknown`, …).
48    pub name: String,
49    /// First argument (`x0`) when applicable.
50    pub arg0: u64,
51    /// Second argument (`x1`) when applicable.
52    pub arg1: u64,
53    /// Third argument (`x2`) when applicable.
54    pub arg2: u64,
55    /// Return value written to `x0`, if any.
56    pub retval: Option<u64>,
57    /// Darwin error path: carry set, `retval` is positive errno.
58    pub error: bool,
59}
60
61/// Outcome of handling a trap.
62#[derive(Debug, Clone, Copy, PartialEq, Eq)]
63pub enum TrapOutcome {
64    /// Guest called BSD `exit` (or the host forced process termination).
65    Exit {
66        /// Process exit code.
67        code: i32,
68    },
69    /// Guest should resume after the trap instruction.
70    Continue,
71    /// Guest called `bsdthread_terminate` (or equivalent): end this host thread only.
72    ThreadExit,
73}
74
75/// Configuration for the trap backend.
76#[derive(Debug, Clone)]
77pub struct TrapConfig {
78    /// Maximum number of events to retain for `kh trace`.
79    pub max_events: usize,
80    /// Maximum syscalls before forced exit (safety cap).
81    pub max_syscalls: usize,
82}
83
84impl Default for TrapConfig {
85    fn default() -> Self {
86        Self {
87            max_events: 256,
88            max_syscalls: 256,
89        }
90    }
91}
92
93/// Trap subsystem errors.
94#[derive(Debug, Error)]
95pub enum TrapError {
96    /// Failed to install signal handlers.
97    #[error("failed to install trap signal handler: {0}")]
98    SignalSetup(#[source] io::Error),
99
100    /// Memory protect / patch failure.
101    #[error(transparent)]
102    Map(#[from] MapError),
103
104    /// Architecture / OS is not supported for live traps.
105    #[error("trap backend requires Linux aarch64 for live execution")]
106    UnsupportedArch,
107}
108
109#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
110static HANDLER_INSTALLED: AtomicBool = AtomicBool::new(false);
111static EXIT_CODE: AtomicI32 = AtomicI32::new(0);
112static EXIT_REQUESTED: AtomicBool = AtomicBool::new(false);
113static EVENT_COUNT: AtomicU64 = AtomicU64::new(0);
114static MAX_EVENTS: AtomicU64 = AtomicU64::new(256);
115static TRACE_ON_EXIT: AtomicBool = AtomicBool::new(false);
116static TRACE_JSON: AtomicBool = AtomicBool::new(false);
117/// When set (not `i32::MIN`): if guest exit code differs, process exits 1.
118static EXPECT_CODE: AtomicI32 = AtomicI32::new(i32::MIN);
119static EVENTS: Mutex<Vec<TrapEvent>> = Mutex::new(Vec::new());
120
121/// Patches every AArch64 `svc` in executable regions to `brk #IMM`.
122///
123/// Residual sites only: freestanding libSystem uses hypercall, not `svc`.
124/// Applies to **any** guest image that embeds Darwin `svc` (fixtures, rare
125/// apps); product tools that only call through freestanding typically have
126/// **zero** hits — we still scan, but skip `mprotect` RW/RX when none found.
127pub fn patch_svc_to_brk(regions: &mut [MappedRegion]) -> Result<usize, TrapError> {
128    let mut patched = 0_usize;
129    for region in regions.iter_mut() {
130        if region.prot & VM_PROT_EXECUTE == 0 {
131            continue;
132        }
133        // Only walk image content; BSS tail cannot hold code from disk.
134        let file_len = usize::try_from(region.file_bytes)
135            .unwrap_or(0)
136            .min(region.host_len());
137        if file_len < 4 {
138            continue;
139        }
140
141        // Prefer instruction-section ranges (full-TEXT false-positives in data).
142        let ranges = resolve_svc_scan_ranges(region, file_len);
143        if ranges.is_empty() {
144            continue;
145        }
146
147        // Pass 1: read-only scan of instruction ranges only.
148        let hits = {
149            let bytes = region.host_bytes();
150            let mut hits = Vec::new();
151            for &(start, len) in &ranges {
152                let end = start.saturating_add(len).min(file_len);
153                if end.saturating_sub(start) < 4 {
154                    continue;
155                }
156                let Some(scan) = bytes.get(start..end) else {
157                    continue;
158                };
159                hits.extend(find_svc_offsets(scan).into_iter().map(|o| start.saturating_add(o)));
160            }
161            hits
162        };
163        if hits.is_empty() {
164            continue;
165        }
166
167        // Pass 2: mprotect RW only when there is something to rewrite.
168        mprotect_rw(region)?;
169        let bytes = region.host_bytes_mut();
170        let brk = brk_encoding(BRK_TRAP_IMM).to_le_bytes();
171        for off in &hits {
172            let end = off.saturating_add(4);
173            if let Some(slot) = bytes.get_mut(*off..end) {
174                slot.copy_from_slice(&brk);
175                patched = patched.saturating_add(1);
176            }
177        }
178        let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
179        mprotect_darwin(region, restore)?;
180        region.prot = restore;
181        // Flush only patched sites (pure file maps stay demand-paged).
182        let base = region.host_bytes().as_ptr();
183        for off in hits {
184            // SAFETY: `off` is within file_bytes of this live mapping.
185            let ptr = unsafe { base.add(off).cast_mut() };
186            clear_icache(ptr, 4);
187        }
188    }
189    Ok(patched)
190}
191
192/// Instruction ranges to scan, or whole `file_len` when the plan had none.
193fn resolve_svc_scan_ranges(region: &MappedRegion, file_len: usize) -> Vec<(usize, usize)> {
194    if region.svc_scan_ranges.is_empty() {
195        return vec![(0, file_len)];
196    }
197    let mut out = Vec::with_capacity(region.svc_scan_ranges.len());
198    for &(off, len) in &region.svc_scan_ranges {
199        let start = usize::try_from(off).unwrap_or(usize::MAX);
200        let len = usize::try_from(len).unwrap_or(0);
201        if start >= file_len || len == 0 {
202            continue;
203        }
204        let len = len.min(file_len.saturating_sub(start));
205        if len >= 4 {
206            out.push((start, len));
207        }
208    }
209    out
210}
211
212const fn is_svc(insn: u32) -> bool {
213    (insn & SVC_MASK) == SVC_BASE
214}
215
216/// Scan little-endian AArch64 insn stream for `svc` encodings.
217///
218/// Unrolled word loads; large regions (≥8 MiB) fan out across worker threads
219/// so CLT-sized TEXT does not serialize page-fault + scan on one core.
220#[allow(clippy::integer_division)] // insn stream: 4-byte AArch64 words
221fn find_svc_offsets(scan: &[u8]) -> Vec<usize> {
222    const PARALLEL_MIN: usize = 8 << 20; // 8 MiB
223    if scan.len() >= PARALLEL_MIN {
224        return find_svc_offsets_parallel(scan);
225    }
226    find_svc_offsets_range(scan, 0)
227}
228
229/// Parallel `svc` scan over word-aligned chunks (any guest with large RX).
230#[allow(clippy::integer_division)]
231fn find_svc_offsets_parallel(scan: &[u8]) -> Vec<usize> {
232    let nthreads = std::thread::available_parallelism()
233        .map_or(1, std::num::NonZeroUsize::get)
234        .clamp(1, 8);
235    if nthreads <= 1 {
236        return find_svc_offsets_range(scan, 0);
237    }
238    let total = scan.len() & !3_usize;
239    if total < 4 {
240        return Vec::new();
241    }
242    let chunk = (total
243        .checked_div(nthreads)
244        .unwrap_or(total)
245        & !3_usize)
246        .max(4);
247    let mut hits = Vec::new();
248    std::thread::scope(|scope| {
249        let mut handles = Vec::with_capacity(nthreads);
250        let mut start = 0_usize;
251        while start < total {
252            let end = (start.saturating_add(chunk)).min(total);
253            let Some(part) = scan.get(start..end) else {
254                break;
255            };
256            let base_off = start;
257            handles.push(scope.spawn(move || find_svc_offsets_range(part, base_off)));
258            start = end;
259        }
260        for h in handles {
261            if let Ok(mut part) = h.join() {
262                hits.append(&mut part);
263            }
264        }
265    });
266    hits.sort_unstable();
267    hits
268}
269
270/// Scan `scan` for `svc`; returned offsets are relative to the full image
271/// when `base_off` is the absolute start of this slice.
272#[allow(clippy::integer_division)]
273fn find_svc_offsets_range(scan: &[u8], base_off: usize) -> Vec<usize> {
274    let mut hits = Vec::new();
275    let words = scan.len() / 4;
276    if words == 0 {
277        return hits;
278    }
279    // SAFETY: scan is a live host mapping; unaligned u32 load is fine on
280    // aarch64/x86_64.
281    let base = scan.as_ptr();
282    let mut i = 0_usize;
283    while i.saturating_add(4) <= words {
284        // SAFETY: i..i+3 < words.
285        let w0 = unsafe { base.add(i.saturating_mul(4)).cast::<u32>().read_unaligned() };
286        let w1 = unsafe {
287            base.add(i.saturating_add(1).saturating_mul(4))
288                .cast::<u32>()
289                .read_unaligned()
290        };
291        let w2 = unsafe {
292            base.add(i.saturating_add(2).saturating_mul(4))
293                .cast::<u32>()
294                .read_unaligned()
295        };
296        let w3 = unsafe {
297            base.add(i.saturating_add(3).saturating_mul(4))
298                .cast::<u32>()
299                .read_unaligned()
300        };
301        if is_svc(u32::from_le(w0)) {
302            hits.push(base_off.saturating_add(i.saturating_mul(4)));
303        }
304        if is_svc(u32::from_le(w1)) {
305            hits.push(
306                base_off.saturating_add(i.saturating_add(1).saturating_mul(4)),
307            );
308        }
309        if is_svc(u32::from_le(w2)) {
310            hits.push(
311                base_off.saturating_add(i.saturating_add(2).saturating_mul(4)),
312            );
313        }
314        if is_svc(u32::from_le(w3)) {
315            hits.push(
316                base_off.saturating_add(i.saturating_add(3).saturating_mul(4)),
317            );
318        }
319        i = i.saturating_add(4);
320    }
321    while i < words {
322        let w = unsafe { base.add(i.saturating_mul(4)).cast::<u32>().read_unaligned() };
323        if is_svc(u32::from_le(w)) {
324            hits.push(base_off.saturating_add(i.saturating_mul(4)));
325        }
326        i = i.saturating_add(1);
327    }
328    hits
329}
330
331/// Round `len` up to a multiple of `page` (page must be a power of two ≥ 1).
332#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
333#[inline]
334fn align_up_to_page(len: usize, page: usize) -> usize {
335    if page <= 1 {
336        return len.max(1);
337    }
338    let mask = page.saturating_sub(1);
339    len.saturating_add(mask) & !mask
340}
341
342// libgcc / compiler-rt symbol linked on Linux aarch64 toolchains.
343#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
344unsafe extern "C" {
345    fn __clear_cache(start: *mut libc::c_void, end: *mut libc::c_void);
346}
347
348/// Flush D/I caches after writing executable code (aarch64 requirement).
349#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
350fn clear_icache(ptr: *mut u8, len: usize) {
351    if ptr.is_null() || len == 0 {
352        return;
353    }
354    // SAFETY: range is a live mapping the caller owns and just wrote.
355    unsafe {
356        let start = ptr.cast::<libc::c_void>();
357        let end = ptr.wrapping_add(len).cast::<libc::c_void>();
358        __clear_cache(start, end);
359    }
360}
361
362#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
363fn clear_icache(_ptr: *mut u8, _len: usize) {}
364
365#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
366#[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
367fn trampoline_rust_entry_addr() -> u64 {
368    u64::try_from(kh_trampoline_dispatch as usize).unwrap_or(0)
369}
370
371/// Return value of the trampoline dispatch (x0=retval, x1=error flag).
372#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
373#[repr(C)]
374pub struct TrampRet {
375    /// Value for guest `x0`.
376    pub retval: u64,
377    /// Non-zero → set Darwin carry (error path).
378    pub error: u64,
379}
380
381/// Called from the hypercall NEON tramp (`TRAMP_BYTES`) on the host alt stack.
382///
383/// # Safety
384///
385/// Caller must preserve guest SIMD/GPRs around the call (hypercall prolog +
386/// mapped NEON tramp). Arguments are Darwin BSD syscall registers; `x16`
387/// holds the syscall number (or AAPCS 8th arg after the hypercall prologue).
388/// May end the current host worker thread on `bsdthread_terminate`
389/// (`SYS_exit`, not `pthread_exit`), or `_exit` the process on guest `exit`.
390///
391/// Guest entry is [`kh_hypercall_entry`]: alt stack + TLS boundary before this.
392#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
393#[unsafe(no_mangle)]
394pub unsafe extern "C" fn kh_trampoline_dispatch(
395    x0: u64,
396    x1: u64,
397    x2: u64,
398    x3: u64,
399    x4: u64,
400    x5: u64,
401    x6: u64,
402    x16: u64,
403) -> TrampRet {
404    let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
405    let result = syscall::dispatch(SyscallArgs {
406        pc: 0,
407        number: sys_no,
408        x0,
409        x1,
410        x2,
411        x3,
412        x4,
413        x5,
414        x6,
415    });
416    if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
417        maybe_push_event(
418            0,
419            sys_no,
420            result.name,
421            x0,
422            x1,
423            x2,
424            result.retval,
425            result.error,
426        );
427    }
428    match result.outcome {
429        TrapOutcome::Exit { code } => finish_with_exit_code(code),
430        TrapOutcome::ThreadExit => {
431            // End this host worker only (same as bsdthread_terminate).
432            crate::thread::exit_current_guest_worker();
433        }
434        TrapOutcome::Continue => {}
435    }
436    TrampRet {
437        retval: result.retval.unwrap_or(0),
438        error: u64::from(result.error),
439    }
440}
441
442// Freestanding hypercall entry (`kh_hypercall_entry`): host alt stack preferred
443// (join must not run Rust on a guest stack that can be unmapped).
444//
445// NEON: Darwin `svc` preserves Q0–Q31 — full Q* + FPCR/FPSR saved on the guest
446// prolog frame *before* any `bl` (AAPCS64 clobbers). Dispatch via `TRAMP_BYTES`
447// (second NEON frame on host/alt stack). ABI: x0–x6 + # in x7 → TrampRet x0/x1.
448// Guest frame (640 B): [0] x29,x30; [16..] x0–x7; [80] pad; [96] q0–q31; [608] fpcr,fpsr.
449#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
450std::arch::global_asm!(
451    r#"
452    .text
453    .align 2
454    .global kh_hypercall_entry
455    .type kh_hypercall_entry, %function
456kh_hypercall_entry:
457    // Guest prolog: GPRs + full NEON *before* any bl (NEON must survive host TLS).
458    sub sp, sp, #640
459    stp x29, x30, [sp, #0]
460    stp x0, x1, [sp, #16]
461    stp x2, x3, [sp, #32]
462    stp x4, x5, [sp, #48]
463    stp x6, x7, [sp, #64]
464    mov x29, sp
465    stp q0, q1, [sp, #96]
466    stp q2, q3, [sp, #128]
467    stp q4, q5, [sp, #160]
468    stp q6, q7, [sp, #192]
469    stp q8, q9, [sp, #224]
470    stp q10, q11, [sp, #256]
471    stp q12, q13, [sp, #288]
472    stp q14, q15, [sp, #320]
473    stp q16, q17, [sp, #352]
474    stp q18, q19, [sp, #384]
475    stp q20, q21, [sp, #416]
476    stp q22, q23, [sp, #448]
477    stp q24, q25, [sp, #480]
478    stp q26, q27, [sp, #512]
479    stp q28, q29, [sp, #544]
480    stp q30, q31, [sp, #576]
481    mrs x9, fpcr
482    mrs x10, fpsr
483    // #608 exceeds signed LDP imm7*8 range (±512); use base+add.
484    add x11, sp, #608
485    stp x9, x10, [x11]
486
487    // A1/A2: enter returns alt_top in x0, guest_tpidr in x1 (HyperEnterRet).
488    // Park guest_va at prolog+80 (hole before NEON) so leave needs no gettid.
489    bl kh_tls_enter_host
490    mov x9, sp                 // guest prolog frame
491    str x1, [x9, #80]          // guest_tpidr for leave
492    cbz x0, 1f
493    mov sp, x0                 // host alt stack
494    ldr x30, [x9, #8]          // reload LR → freestanding thin caller
495    // Host frame: [guest_frame, lr]
496    stp x9, x30, [sp, #-16]!
497    // Reload guest frame ptr / args (enter_host clobbered temps).
498    ldr x9, [sp]
499    ldp x0, x1, [x9, #16]
500    ldp x2, x3, [x9, #32]
501    ldp x4, x5, [x9, #48]
502    ldp x6, x7, [x9, #64]
503    mov x16, x7
504    bl kh_neon_tramp_entry
505    // Preserve TrampRet (x0/x1) across TLS restore.
506    stp x0, x1, [sp, #-16]!
507    // guest_va at host_frame.guest_frame + 80; host_frame at sp+16.
508    ldr x9, [sp, #16]
509    ldr x0, [x9, #80]
510    bl kh_tls_leave_host
511    ldp x0, x1, [sp], #16
512    ldp x9, x30, [sp], #16
513    mov sp, x9
514    // Restore guest NEON from prolog (pre-bl snapshot).
515    ldp q0, q1, [sp, #96]
516    ldp q2, q3, [sp, #128]
517    ldp q4, q5, [sp, #160]
518    ldp q6, q7, [sp, #192]
519    ldp q8, q9, [sp, #224]
520    ldp q10, q11, [sp, #256]
521    ldp q12, q13, [sp, #288]
522    ldp q14, q15, [sp, #320]
523    ldp q16, q17, [sp, #352]
524    ldp q18, q19, [sp, #384]
525    ldp q20, q21, [sp, #416]
526    ldp q22, q23, [sp, #448]
527    ldp q24, q25, [sp, #480]
528    ldp q26, q27, [sp, #512]
529    ldp q28, q29, [sp, #544]
530    ldp q30, q31, [sp, #576]
531    add x11, sp, #608
532    ldp x9, x10, [x11]
533    msr fpcr, x9
534    msr fpsr, x10
535    ldr x29, [sp, #0]
536    add sp, sp, #640
537    ret
5381:
539    // No alt stack: dispatch on the guest stack (ST / fallback only).
540    // Already on host TLS from the enter above; guest_va at [sp, #80].
541    ldp x0, x1, [sp, #16]
542    ldp x2, x3, [sp, #32]
543    ldp x4, x5, [sp, #48]
544    ldp x6, x7, [sp, #64]
545    mov x16, x7
546    // Preserve real LR across the call (still on guest prolog frame).
547    ldr x30, [sp, #8]
548    bl kh_neon_tramp_entry
549    stp x0, x1, [sp, #-16]!
550    ldr x0, [sp, #16+80]
551    bl kh_tls_leave_host
552    ldp x0, x1, [sp], #16
553    // Restore guest NEON from prolog.
554    ldp q0, q1, [sp, #96]
555    ldp q2, q3, [sp, #128]
556    ldp q4, q5, [sp, #160]
557    ldp q6, q7, [sp, #192]
558    ldp q8, q9, [sp, #224]
559    ldp q10, q11, [sp, #256]
560    ldp q12, q13, [sp, #288]
561    ldp q14, q15, [sp, #320]
562    ldp q16, q17, [sp, #352]
563    ldp q18, q19, [sp, #384]
564    ldp q20, q21, [sp, #416]
565    ldp q22, q23, [sp, #448]
566    ldp q24, q25, [sp, #480]
567    ldp q26, q27, [sp, #512]
568    ldp q28, q29, [sp, #544]
569    ldp q30, q31, [sp, #576]
570    add x11, sp, #608
571    ldp x9, x10, [x11]
572    msr fpcr, x9
573    msr fpsr, x10
574    ldp x29, x30, [sp, #0]
575    add sp, sp, #640
576    ret
577    .size kh_hypercall_entry, .-kh_hypercall_entry
578    "#
579);
580
581// `kh_neon_tramp_entry` + `KH_NEON_TRAMP_VA`: branch into mapped `TRAMP_BYTES`
582// (full Q* save around Rust dispatch). Fallback if tramp not mapped yet:
583// tail-call `kh_trampoline_dispatch` (still safe: prolog owns guest Q*).
584#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
585std::arch::global_asm!(
586    r#"
587    .text
588    .align 2
589    .global kh_neon_tramp_entry
590    .type kh_neon_tramp_entry, %function
591kh_neon_tramp_entry:
592    adrp x17, KH_NEON_TRAMP_VA
593    ldr x17, [x17, :lo12:KH_NEON_TRAMP_VA]
594    cbz x17, 1f
595    br x17
5961:
597    // No-tramp fallback: number in x16 → AAPCS 8th arg x7; tail call.
598    mov x7, x16
599    b kh_trampoline_dispatch
600    .size kh_neon_tramp_entry, .-kh_neon_tramp_entry
601
602    .data
603    .align 3
604    .global KH_NEON_TRAMP_VA
605KH_NEON_TRAMP_VA:
606    .quad 0
607    "#
608);
609
610#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
611unsafe extern "C" {
612    fn kh_hypercall_entry();
613    static mut KH_NEON_TRAMP_VA: u64;
614}
615
616/// Address of freestanding hypercall entry (alt stack + NEON tramp + dispatch).
617#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
618#[must_use]
619pub fn hypercall_entry_addr() -> u64 {
620    ensure_neon_tramp();
621    #[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
622    {
623        u64::try_from(kh_hypercall_entry as usize).unwrap_or(0)
624    }
625}
626
627/// Map TRAMP_BYTES once and publish its VA to `KH_NEON_TRAMP_VA`.
628#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
629fn ensure_neon_tramp() {
630    static ONCE: AtomicU64 = AtomicU64::new(0);
631    static LOCK: Mutex<()> = Mutex::new(());
632
633    if ONCE.load(Ordering::Acquire) != 0 {
634        return;
635    }
636    let _g = LOCK
637        .lock()
638        .unwrap_or_else(std::sync::PoisonError::into_inner);
639    if ONCE.load(Ordering::Acquire) != 0 {
640        return;
641    }
642
643    let code = TRAMP_BYTES;
644    let page = crate::host::page_size().unwrap_or(4096);
645    let map_len = align_up_to_page(code.len(), page).max(page);
646    let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
647    let prot_rw = libc::PROT_READ | libc::PROT_WRITE;
648    let Some(ptr) = crate::host::mmap(None, map_len, prot_rw, flags, -1, 0) else {
649        tracing::error!("NEON tramp mmap failed");
650        return;
651    };
652    // SAFETY: fresh RW mapping.
653    let slice = unsafe { std::slice::from_raw_parts_mut(ptr, map_len) };
654    slice.fill(0);
655    if let Some(dst) = slice.get_mut(..code.len()) {
656        dst.copy_from_slice(code);
657    } else {
658        let _ = crate::host::munmap(ptr, map_len);
659        return;
660    }
661    if let Some(slot) = slice.get_mut(TRAMP_RELOC_OFF..TRAMP_RELOC_OFF.saturating_add(8)) {
662        slot.copy_from_slice(&trampoline_rust_entry_addr().to_le_bytes());
663    } else {
664        let _ = crate::host::munmap(ptr, map_len);
665        return;
666    }
667    let prot_rx = libc::PROT_READ | libc::PROT_EXEC;
668    if !crate::host::mprotect(ptr, map_len, prot_rx) {
669        tracing::error!("NEON tramp mprotect RX failed");
670        let _ = crate::host::munmap(ptr, map_len);
671        return;
672    }
673    clear_icache(ptr, map_len);
674    let va = crate::host::ptr_addr_u64(ptr);
675    // SAFETY: symbol from global_asm; written once under LOCK before guest entry.
676    unsafe {
677        KH_NEON_TRAMP_VA = va;
678    }
679    ONCE.store(va, Ordering::Release);
680    tracing::info!(
681        va = format_args!("{va:#x}"),
682        "mapped NEON tramp for hypercall"
683    );
684}
685
686#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
687#[must_use]
688pub fn hypercall_entry_addr() -> u64 {
689    0
690}
691
692// ── trampoline machine code (verified via rustc/objdump aarch64) ─────────────
693
694/// Offset of the 8-byte absolute `kh_trampoline_dispatch` pointer.
695///
696/// Layout (see `TRAMP_BYTES`): `bti c` + save GPRs/NEON/FPCR + `blr` dispatch
697/// + restore + `ret` + `.quad` reloc @ 0x100.
698#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
699const TRAMP_RELOC_OFF: usize = 0x100;
700
701/// Exact opcodes from assembled trampoline (`as` / objdump on Linux aarch64).
702///
703/// Saves **all Q0–Q31 + FPCR/FPSR** around the Rust call: Darwin `svc` preserves
704/// SIMD state; compression workers (7zz MT) rely on that. A bare GPR-only
705/// trampoline clobbered NEON and faulted under `-mmt>1`.
706#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
707#[rustfmt::skip]
708const TRAMP_BYTES: &[u8] = &[
709    0x5f, 0x24, 0x03, 0xd5, 0xff, 0x03, 0x0a, 0xd1, 0xe8, 0x27, 0x00, 0xa9, 0xea, 0x2f, 0x01, 0xa9,
710    0xec, 0x37, 0x02, 0xa9, 0xee, 0x3f, 0x03, 0xa9, 0xf1, 0x4b, 0x04, 0xa9, 0xfe, 0x2b, 0x00, 0xf9,
711    0x08, 0x44, 0x3b, 0xd5, 0x29, 0x44, 0x3b, 0xd5, 0xe8, 0x27, 0x06, 0xa9, 0xe0, 0x87, 0x03, 0xad,
712    0xe2, 0x8f, 0x04, 0xad, 0xe4, 0x97, 0x05, 0xad, 0xe6, 0x9f, 0x06, 0xad, 0xe8, 0xa7, 0x07, 0xad,
713    0xea, 0xaf, 0x08, 0xad, 0xec, 0xb7, 0x09, 0xad, 0xee, 0xbf, 0x0a, 0xad, 0xf0, 0xc7, 0x0b, 0xad,
714    0xf2, 0xcf, 0x0c, 0xad, 0xf4, 0xd7, 0x0d, 0xad, 0xf6, 0xdf, 0x0e, 0xad, 0xf8, 0xe7, 0x0f, 0xad,
715    0xfa, 0xef, 0x10, 0xad, 0xfc, 0xf7, 0x11, 0xad, 0xfe, 0xff, 0x12, 0xad, 0xe7, 0x03, 0x10, 0xaa,
716    0x88, 0x04, 0x00, 0x58, 0x00, 0x01, 0x3f, 0xd6, 0x02, 0x42, 0x3b, 0xd5, 0x23, 0x00, 0x80, 0x52,
717    0x63, 0x88, 0x63, 0xd3, 0x42, 0x00, 0x23, 0x8a, 0x41, 0x00, 0x00, 0xb4, 0x42, 0x00, 0x03, 0xaa,
718    0x02, 0x42, 0x1b, 0xd5, 0xe8, 0x27, 0x46, 0xa9, 0x08, 0x44, 0x1b, 0xd5, 0x29, 0x44, 0x1b, 0xd5,
719    0xe0, 0x87, 0x43, 0xad, 0xe2, 0x8f, 0x44, 0xad, 0xe4, 0x97, 0x45, 0xad, 0xe6, 0x9f, 0x46, 0xad,
720    0xe8, 0xa7, 0x47, 0xad, 0xea, 0xaf, 0x48, 0xad, 0xec, 0xb7, 0x49, 0xad, 0xee, 0xbf, 0x4a, 0xad,
721    0xf0, 0xc7, 0x4b, 0xad, 0xf2, 0xcf, 0x4c, 0xad, 0xf4, 0xd7, 0x4d, 0xad, 0xf6, 0xdf, 0x4e, 0xad,
722    0xf8, 0xe7, 0x4f, 0xad, 0xfa, 0xef, 0x50, 0xad, 0xfc, 0xf7, 0x51, 0xad, 0xfe, 0xff, 0x52, 0xad,
723    0xfe, 0x2b, 0x40, 0xf9, 0xf1, 0x4b, 0x44, 0xa9, 0xee, 0x3f, 0x43, 0xa9, 0xec, 0x37, 0x42, 0xa9,
724    0xea, 0x2f, 0x41, 0xa9, 0xe8, 0x27, 0x40, 0xa9, 0xff, 0x03, 0x0a, 0x91, 0xc0, 0x03, 0x5f, 0xd6,
725    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // reloc .quad @ 0x100
726];
727/// Installs the `SIGTRAP` handler used by the micro execution backend.
728pub fn install_trap_handlers(config: &TrapConfig) -> Result<(), TrapError> {
729    MAX_EVENTS.store(
730        u64::try_from(config.max_events).unwrap_or(256),
731        Ordering::SeqCst,
732    );
733    syscall::reset_syscall_state(config.max_syscalls);
734
735    #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
736    {
737        let _ = config;
738        Err(TrapError::UnsupportedArch)
739    }
740
741    #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
742    {
743        if HANDLER_INSTALLED.swap(true, Ordering::SeqCst) {
744            reset_trap_state();
745            syscall::reset_syscall_state(config.max_syscalls);
746            return Ok(());
747        }
748
749        // Guest faults used to leave multi‑MB `core` files in the workspace cwd.
750        // Faults go through our handler → `_exit`; disable dumps as belt-and-braces.
751        disable_core_dumps();
752
753        // SAFETY: process-wide SIGTRAP / fault handlers with SA_SIGINFO.
754        unsafe {
755            let mut sa: libc::sigaction = std::mem::zeroed();
756            sa.sa_flags = libc::SA_SIGINFO;
757            #[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
758            {
759                sa.sa_sigaction = trap_sigaction as *const () as usize;
760            }
761            libc::sigemptyset(std::ptr::addr_of_mut!(sa.sa_mask));
762            if libc::sigaction(libc::SIGTRAP, std::ptr::addr_of!(sa), std::ptr::null_mut()) != 0 {
763                HANDLER_INSTALLED.store(false, Ordering::SeqCst);
764                return Err(TrapError::SignalSetup(io::Error::last_os_error()));
765            }
766
767            // Guest faults: print PC/addr so `kh run` diagnoses unbound GOT / bad heap.
768            let mut fault: libc::sigaction = std::mem::zeroed();
769            fault.sa_flags = libc::SA_SIGINFO;
770            #[allow(unknown_lints, clippy::as_conversions, function_casts_as_integer)]
771            {
772                fault.sa_sigaction = guest_fault_sigaction as *const () as usize;
773            }
774            libc::sigemptyset(std::ptr::addr_of_mut!(fault.sa_mask));
775            for sig in [libc::SIGSEGV, libc::SIGBUS] {
776                if libc::sigaction(sig, std::ptr::addr_of!(fault), std::ptr::null_mut()) != 0 {
777                    HANDLER_INSTALLED.store(false, Ordering::SeqCst);
778                    return Err(TrapError::SignalSetup(io::Error::last_os_error()));
779                }
780            }
781        }
782        reset_trap_state();
783        Ok(())
784    }
785}
786
787/// Sets `RLIMIT_CORE` soft+hard to 0 so guest crashes never litter the repo with
788/// `core` dumps (we already print a guest fault summary and `_exit`).
789#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
790fn disable_core_dumps() {
791    // SAFETY: process-wide rlimit; best-effort, ignore failure.
792    unsafe {
793        let lim = libc::rlimit {
794            rlim_cur: 0,
795            rlim_max: 0,
796        };
797        let _ = libc::setrlimit(libc::RLIMIT_CORE, std::ptr::addr_of!(lim));
798    }
799}
800
801/// Logs guest `SIGSEGV` / `SIGBUS` (PC, fault address, key regs) then exits.
802#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
803unsafe extern "C" fn guest_fault_sigaction(
804    signo: libc::c_int,
805    info: *mut libc::siginfo_t,
806    ctx: *mut libc::c_void,
807) {
808    // May run under guest TPIDR; restore host before formatting / write.
809    crate::tls::enter_host_tls();
810    let name = match signo {
811        libc::SIGSEGV => "SIGSEGV",
812        libc::SIGBUS => "SIGBUS",
813        _ => "FAULT",
814    };
815    let mut pc = 0_u64;
816    let mut sp = 0_u64;
817    let mut x0 = 0_u64;
818    let mut x1 = 0_u64;
819    let mut x2 = 0_u64;
820    let mut x8 = 0_u64;
821    let mut x16 = 0_u64;
822    let mut x19 = 0_u64;
823    let mut x3 = 0_u64;
824    let mut x4 = 0_u64;
825    let mut x5 = 0_u64;
826    let mut x6 = 0_u64;
827    let mut x7 = 0_u64;
828    let mut x9 = 0_u64;
829    let mut x20 = 0_u64;
830    let mut x21 = 0_u64;
831    let mut x22 = 0_u64;
832    let mut x23 = 0_u64;
833    let mut x24 = 0_u64;
834    let mut x25 = 0_u64;
835    let mut x26 = 0_u64;
836    let mut x27 = 0_u64;
837    let mut x28 = 0_u64;
838    let mut x29 = 0_u64;
839    let mut lr = 0_u64;
840    if !ctx.is_null() {
841        // SAFETY: kernel `ucontext_t` for SA_SIGINFO.
842        let uctx = unsafe { &*ctx.cast::<libc::ucontext_t>() };
843        let m = &uctx.uc_mcontext;
844        pc = m.pc;
845        sp = m.sp;
846        x0 = m.regs[0];
847        x1 = m.regs[1];
848        x2 = m.regs[2];
849        x3 = m.regs[3];
850        x4 = m.regs[4];
851        x5 = m.regs[5];
852        x6 = m.regs[6];
853        x7 = m.regs[7];
854        x8 = m.regs[8];
855        x9 = m.regs[9];
856        x16 = m.regs[16];
857        x19 = m.regs[19];
858        x20 = m.regs[20];
859        x21 = m.regs[21];
860        x22 = m.regs[22];
861        x23 = m.regs[23];
862        x24 = m.regs[24];
863        x25 = m.regs[25];
864        x26 = m.regs[26];
865        x27 = m.regs[27];
866        x28 = m.regs[28];
867        x29 = m.regs[29];
868        lr = m.regs[30];
869    }
870    let addr = if info.is_null() {
871        0_u64
872    } else {
873        // SAFETY: kernel-provided siginfo.
874        let p = unsafe { (*info).si_addr() };
875        u64::try_from(p.addr()).unwrap_or(0)
876    };
877    let msg = format!(
878        "error: guest {name} pc={pc:#x} addr={addr:#x} sp={sp:#x} lr={lr:#x} \
879         x0={x0:#x} x1={x1:#x} x2={x2:#x} x3={x3:#x} x4={x4:#x} x5={x5:#x} \
880         x6={x6:#x} x7={x7:#x} x8={x8:#x} x9={x9:#x} x16={x16:#x} \
881         x19={x19:#x} x20={x20:#x} x21={x21:#x} x22={x22:#x} x23={x23:#x} \
882         x24={x24:#x} x25={x25:#x} x26={x26:#x} x27={x27:#x} x28={x28:#x} x29={x29:#x}\n"
883    );
884    drop(io::stderr().write_all(msg.as_bytes()));
885    // Best-effort: dump AArch64 insn at PC via /proc/self/mem (no raw guest deref).
886    if pc != 0 {
887        if let Ok(mut f) = std::fs::File::open("/proc/self/mem") {
888            use std::io::{Read, Seek, SeekFrom};
889            let mut insn = [0_u8; 4];
890            if f.seek(SeekFrom::Start(pc)).is_ok() && f.read_exact(&mut insn).is_ok() {
891                let w = u32::from_le_bytes(insn);
892                let m = format!("  insn@{pc:#x} = {w:#010x} bytes={insn:02x?}\n");
893                drop(io::stderr().write_all(m.as_bytes()));
894            }
895        }
896    }
897    // Best-effort: which mapping owns PC / fault address / key regs.
898    if let Ok(maps) = std::fs::read_to_string("/proc/self/maps") {
899        for &va in &[pc, addr, lr, x0, x1, x2, x16, x19, x25] {
900            if va == 0 {
901                continue;
902            }
903            for line in maps.lines() {
904                let Some((range, rest)) = line.split_once(' ') else {
905                    continue;
906                };
907                let Some((a, b)) = range.split_once('-') else {
908                    continue;
909                };
910                let (Ok(lo), Ok(hi)) = (u64::from_str_radix(a, 16), u64::from_str_radix(b, 16))
911                else {
912                    continue;
913                };
914                if va >= lo && va < hi {
915                    let off = va.saturating_sub(lo);
916                    let m = format!("  map {va:#x} = {lo:#x}+{off:#x} {rest}\n");
917                    drop(io::stderr().write_all(m.as_bytes()));
918                    break;
919                }
920            }
921        }
922    }
923    drop(io::stderr().flush());
924    if TRACE_ON_EXIT.load(Ordering::SeqCst) {
925        dump_events_before_exit();
926    }
927    // SAFETY: hard-stop after unrecoverable guest fault.
928    unsafe { libc::_exit(128_i32.saturating_add(signo)) };
929}
930
931/// Drains recorded trap events (for `kh trace`).
932#[must_use]
933pub fn take_trace_events() -> Vec<TrapEvent> {
934    EVENTS
935        .lock()
936        .map(|mut g| std::mem::take(&mut *g))
937        .unwrap_or_default()
938}
939
940/// Whether the guest requested exit.
941#[must_use]
942pub fn exit_requested() -> bool {
943    EXIT_REQUESTED.load(Ordering::SeqCst)
944}
945
946/// Exit code from the last guest `exit` trap.
947#[must_use]
948pub fn take_exit_code() -> Option<i32> {
949    if EXIT_REQUESTED.load(Ordering::SeqCst) {
950        Some(EXIT_CODE.load(Ordering::SeqCst))
951    } else {
952        None
953    }
954}
955
956/// Resets exit / event state before a new run.
957pub fn reset_trap_state() {
958    EXIT_REQUESTED.store(false, Ordering::SeqCst);
959    EXIT_CODE.store(0, Ordering::SeqCst);
960    EVENT_COUNT.store(0, Ordering::SeqCst);
961    if let Ok(mut guard) = EVENTS.lock() {
962        guard.clear();
963    }
964}
965
966/// When set, the exit trap prints recorded events to stdout before `_exit`.
967pub fn set_trace_on_exit(json: bool) {
968    TRACE_ON_EXIT.store(true, Ordering::SeqCst);
969    TRACE_JSON.store(json, Ordering::SeqCst);
970}
971
972/// Clears the trace-on-exit flag (default for plain `kh run`).
973pub fn clear_trace_on_exit() {
974    TRACE_ON_EXIT.store(false, Ordering::SeqCst);
975    TRACE_JSON.store(false, Ordering::SeqCst);
976}
977
978/// Sets the expected guest exit code for the micro gate (`--expect-code`).
979pub fn set_expect_code(code: i32) {
980    EXPECT_CODE.store(code, Ordering::SeqCst);
981}
982
983/// Disables the expect-code gate.
984pub fn clear_expect_code() {
985    EXPECT_CODE.store(i32::MIN, Ordering::SeqCst);
986}
987
988/// Terminates the host process as if the guest called BSD `exit(code)`.
989///
990/// Used after `LC_MAIN` returns (dyld calls `exit` with `main`'s return value).
991/// Honours `--expect-code` and optional trace-on-exit dump.
992pub fn finish_with_exit_code(code: i32) -> ! {
993    EXIT_CODE.store(code, Ordering::SeqCst);
994    EXIT_REQUESTED.store(true, Ordering::SeqCst);
995    if TRACE_ON_EXIT.load(Ordering::SeqCst) {
996        dump_events_before_exit();
997    }
998    // Host TPIDR: opt-in diagnosis (futex classes; boundary crossing counts).
999    crate::syscall::dump_futex_stats_if_enabled();
1000    crate::syscall::dump_boundary_stats_if_enabled();
1001    let expect = EXPECT_CODE.load(Ordering::SeqCst);
1002    let status = if expect == i32::MIN || expect == code {
1003        code
1004    } else {
1005        let msg = format!("error: guest exit code {code} != expected {expect}\n");
1006        drop(io::stderr().write_all(msg.as_bytes()));
1007        1
1008    };
1009    // SAFETY: intentional process termination after guest main return / exit.
1010    unsafe { libc::_exit(status) };
1011}
1012
1013#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
1014unsafe extern "C" fn trap_sigaction(
1015    signo: libc::c_int,
1016    _info: *mut libc::siginfo_t,
1017    ctx: *mut libc::c_void,
1018) {
1019    if signo != libc::SIGTRAP || ctx.is_null() {
1020        return;
1021    }
1022
1023    // Guest may have TPIDR_EL0 = guest TSD; restore host TLS before Rust/libc.
1024    crate::tls::enter_host_tls();
1025
1026    // SAFETY: kernel provides a valid `ucontext_t` for SA_SIGINFO deliveries.
1027    let uctx = unsafe { &mut *ctx.cast::<libc::ucontext_t>() };
1028    let m = &mut uctx.uc_mcontext;
1029    let pc = m.pc;
1030    // Only pull the regs dispatch needs; avoid extra loads on the hot path.
1031    let x0 = m.regs[0];
1032    let x1 = m.regs[1];
1033    let x2 = m.regs[2];
1034    let x3 = m.regs[3];
1035    let x4 = m.regs[4];
1036    let x5 = m.regs[5];
1037    let x6 = m.regs[6];
1038    let x16 = m.regs[16];
1039
1040    let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
1041    let result = syscall::dispatch(SyscallArgs {
1042        pc,
1043        number: sys_no,
1044        x0,
1045        x1,
1046        x2,
1047        x3,
1048        x4,
1049        x5,
1050        x6,
1051    });
1052
1053    // Zero-cost when max_events == 0 (plain `kh run`).
1054    if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
1055        maybe_push_event(
1056            pc,
1057            sys_no,
1058            result.name,
1059            x0,
1060            x1,
1061            x2,
1062            result.retval,
1063            result.error,
1064        );
1065    }
1066
1067    if let Some(ret) = result.retval {
1068        m.regs[0] = ret;
1069    }
1070    // Darwin arm64: error → set C; success with write-back → clear C.
1071    if result.error {
1072        m.pstate |= PSTATE_C;
1073    } else if result.retval.is_some() {
1074        m.pstate &= !PSTATE_C;
1075    }
1076
1077    match result.outcome {
1078        TrapOutcome::Exit { code } => {
1079            finish_with_exit_code(code);
1080        }
1081        TrapOutcome::Continue => {
1082            m.pc = pc.wrapping_add(4);
1083            // Return to guest with guest TPIDR restored.
1084            crate::tls::leave_host_tls();
1085        }
1086        TrapOutcome::ThreadExit => {
1087            // Stay on host TLS; exit trampoline publishes join then pthread_exit.
1088            // Redirect ucontext to a host `pthread_exit` trampoline on this
1089            // thread's original stack (see `crate::thread`). Main thread
1090            // (no host frame) falls back to process exit.
1091            if !crate::thread::redirect_ucontext_to_host_exit(m) {
1092                finish_with_exit_code(0);
1093            }
1094        }
1095    }
1096}
1097
1098fn dump_events_before_exit() {
1099    use std::fmt::Write as _;
1100
1101    let json = TRACE_JSON.load(Ordering::SeqCst);
1102    let events = EVENTS.lock().map(|g| g.clone()).unwrap_or_default();
1103    if json {
1104        let mut out = String::from("{\"events\":[");
1105        for (i, e) in events.iter().enumerate() {
1106            if i > 0 {
1107                out.push(',');
1108            }
1109            let sys = e
1110                .syscall
1111                .map_or_else(|| "null".to_owned(), |s| s.to_string());
1112            let ret = e
1113                .retval
1114                .map_or_else(|| "null".to_owned(), |v| v.to_string());
1115            let _ = write!(
1116                out,
1117                "{{\"pc\":{},\"syscall\":{},\"name\":\"{}\",\"arg0\":{},\"arg1\":{},\"arg2\":{},\"retval\":{},\"error\":{}}}",
1118                e.pc, sys, e.name, e.arg0, e.arg1, e.arg2, ret, e.error
1119            );
1120        }
1121        out.push_str("]}\n");
1122        drop(io::stdout().write_all(out.as_bytes()));
1123        drop(io::stdout().flush());
1124    } else {
1125        let header = format!("trace events: {}\n", events.len());
1126        drop(io::stdout().write_all(header.as_bytes()));
1127        for (i, e) in events.iter().enumerate() {
1128            let ret = e
1129                .retval
1130                .map_or_else(|| "-".to_owned(), |v| format!("{v:#x}"));
1131            let err = if e.error { " err" } else { "" };
1132            let line = format!(
1133                "  [{i}] pc={:#x} {} sys={:?} args=({:#x}, {:#x}, {:#x}) ret={ret}{err}\n",
1134                e.pc, e.name, e.syscall, e.arg0, e.arg1, e.arg2
1135            );
1136            drop(io::stdout().write_all(line.as_bytes()));
1137        }
1138        drop(io::stdout().flush());
1139    }
1140}
1141
1142#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
1143#[allow(clippy::too_many_arguments)]
1144fn maybe_push_event(
1145    pc: u64,
1146    sys_no: u32,
1147    name: &'static str,
1148    arg0: u64,
1149    arg1: u64,
1150    arg2: u64,
1151    retval: Option<u64>,
1152    error: bool,
1153) {
1154    let max = MAX_EVENTS.load(Ordering::Relaxed);
1155    // `kh run` sets max_events=0: single load, no String/Mutex.
1156    if max == 0 {
1157        return;
1158    }
1159    let n = EVENT_COUNT.fetch_add(1, Ordering::Relaxed);
1160    if n >= max {
1161        return;
1162    }
1163    if let Ok(mut guard) = EVENTS.lock()
1164        && u64::try_from(guard.len()).unwrap_or(u64::MAX) < max
1165    {
1166        guard.push(TrapEvent {
1167            pc,
1168            syscall: Some(sys_no),
1169            name: name.to_owned(),
1170            arg0,
1171            arg1,
1172            arg2,
1173            retval,
1174            error,
1175        });
1176    }
1177}
1178
1179#[cfg(test)]
1180mod tests {
1181    use super::*;
1182
1183    #[test]
1184    fn detects_svc_encoding() {
1185        assert!(is_svc(0xD400_0001));
1186        let svc80 = 0xD400_0001 | (0x80_u32 << 5);
1187        assert!(is_svc(svc80));
1188        assert!(!is_svc(0xD503_201F));
1189        assert!(!is_svc(brk_encoding(BRK_TRAP_IMM)));
1190    }
1191
1192    #[test]
1193    fn brk_encoding_roundtrip_imm() {
1194        let enc = brk_encoding(BRK_TRAP_IMM);
1195        let imm = (enc >> 5) & 0xFFFF;
1196        assert_eq!(imm, BRK_TRAP_IMM);
1197    }
1198}