Skip to main content

kh_runtime/
trap.rs

1//! Trap backend: rewrite Darwin `svc` for Linux aarch64 translation.
2//!
3//! Linux aarch64 treats every `svc` as a Linux syscall (number in `x8`), which
4//! is incompatible with Darwin (number in `x16`, `svc #0x80`).
5//!
6//! Prefer **userspace trampoline** so the common path never hits `SIGTRAP`:
7//! each `svc` becomes `b veneer` (no link — **preserves `x30`/LR**), and the
8//! per-site veneer saves LR, `bl`s the shared dispatcher, restores LR, then
9//! `b`s back to the instruction after the original `svc`. Fall back to
10//! `brk #IMM` + `SIGTRAP` when a branch is out of range.
11//!
12//! Live handlers require **Linux aarch64**. Patching works on any host.
13#![allow(unsafe_code)]
14
15use std::io::{self, Write};
16use std::sync::Mutex;
17use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering};
18
19use thiserror::Error;
20
21use crate::mem::{
22    MapError, MappedRegion, VM_PROT_EXECUTE, VM_PROT_READ, VM_PROT_WRITE, mprotect_darwin,
23    mprotect_rw,
24};
25use crate::syscall;
26#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
27use crate::syscall::SyscallArgs;
28
29/// Marker immediate for patched traps (`brk #0x0F00`).
30pub const BRK_TRAP_IMM: u32 = 0x0F00;
31
32/// AArch64 `svc #imm16` base encoding: `0xD4000001 | (imm16 << 5)`.
33const SVC_MASK: u32 = 0xFFE0_001F;
34const SVC_BASE: u32 = 0xD400_0001;
35
36/// AArch64 `brk #imm16`: `0xD4200000 | (imm16 << 5)`.
37const fn brk_encoding(imm16: u32) -> u32 {
38    0xD420_0000 | ((imm16 & 0xFFFF) << 5)
39}
40
41/// AArch64 PC-relative branch imm26 (±128 MiB), or `None` if out of range.
42#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
43#[allow(
44    clippy::as_conversions,
45    clippy::cast_possible_truncation,
46    clippy::cast_sign_loss,
47    clippy::manual_range_contains
48)]
49fn branch_imm26(from: u64, to: u64) -> Option<u32> {
50    let delta = i64::try_from(to).ok()?.wrapping_sub(i64::try_from(from).ok()?);
51    if delta & 3 != 0 {
52        return None;
53    }
54    let imm = delta >> 2;
55    // Imm26 signed range for B/BL.
56    if imm < -(1_i64 << 25) || imm >= (1_i64 << 25) {
57        return None;
58    }
59    // Truncate to 26-bit two's complement field (intentional).
60    Some((imm as u32) & 0x03FF_FFFF)
61}
62
63/// AArch64 `BL` (imm26) from `from` to `to`.
64#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
65fn bl_encoding(from: u64, to: u64) -> Option<u32> {
66    Some(0x9400_0000 | branch_imm26(from, to)?)
67}
68
69/// AArch64 `B` (imm26, **no link** — preserves `x30`) from `from` to `to`.
70#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
71fn b_encoding(from: u64, to: u64) -> Option<u32> {
72    Some(0x1400_0000 | branch_imm26(from, to)?)
73}
74
75/// AArch64 `PSTATE` / CPSR carry flag (bit 29 of NZCV).
76///
77/// Darwin arm64 syscall convention: set on error (errno in `x0`), clear on success.
78pub const PSTATE_C: u64 = 1 << 29;
79
80/// One observed trap / translated syscall event.
81#[derive(Debug, Clone, PartialEq, Eq)]
82pub struct TrapEvent {
83    /// Guest PC at the trap (instruction address).
84    pub pc: u64,
85    /// Darwin BSD syscall number (`x16`) when recognized; otherwise `None`.
86    pub syscall: Option<u32>,
87    /// Short label (`exit`, `write`, `unknown`, …).
88    pub name: String,
89    /// First argument (`x0`) when applicable.
90    pub arg0: u64,
91    /// Second argument (`x1`) when applicable.
92    pub arg1: u64,
93    /// Third argument (`x2`) when applicable.
94    pub arg2: u64,
95    /// Return value written to `x0`, if any.
96    pub retval: Option<u64>,
97    /// Darwin error path: carry set, `retval` is positive errno.
98    pub error: bool,
99}
100
101/// Outcome of handling a trap.
102#[derive(Debug, Clone, Copy, PartialEq, Eq)]
103pub enum TrapOutcome {
104    /// Guest called BSD `exit` (or the host forced process termination).
105    Exit {
106        /// Process exit code.
107        code: i32,
108    },
109    /// Guest should resume after the trap instruction.
110    Continue,
111    /// Guest called `bsdthread_terminate` (or equivalent): end this host thread only.
112    ThreadExit,
113}
114
115/// Configuration for the trap backend.
116#[derive(Debug, Clone)]
117pub struct TrapConfig {
118    /// Maximum number of events to retain for `kh trace`.
119    pub max_events: usize,
120    /// Maximum syscalls before forced exit (safety cap).
121    pub max_syscalls: usize,
122}
123
124impl Default for TrapConfig {
125    fn default() -> Self {
126        Self {
127            max_events: 256,
128            max_syscalls: 256,
129        }
130    }
131}
132
133/// Trap subsystem errors.
134#[derive(Debug, Error)]
135pub enum TrapError {
136    /// Failed to install signal handlers.
137    #[error("failed to install trap signal handler: {0}")]
138    SignalSetup(#[source] io::Error),
139
140    /// Memory protect / patch failure.
141    #[error(transparent)]
142    Map(#[from] MapError),
143
144    /// Architecture / OS is not supported for live traps.
145    #[error("trap backend requires Linux aarch64 for live execution")]
146    UnsupportedArch,
147}
148
149#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
150static HANDLER_INSTALLED: AtomicBool = AtomicBool::new(false);
151static EXIT_CODE: AtomicI32 = AtomicI32::new(0);
152static EXIT_REQUESTED: AtomicBool = AtomicBool::new(false);
153static EVENT_COUNT: AtomicU64 = AtomicU64::new(0);
154static MAX_EVENTS: AtomicU64 = AtomicU64::new(256);
155static TRACE_ON_EXIT: AtomicBool = AtomicBool::new(false);
156static TRACE_JSON: AtomicBool = AtomicBool::new(false);
157/// When set (not `i32::MIN`): if guest exit code differs, process exits 1.
158static EXPECT_CODE: AtomicI32 = AtomicI32::new(i32::MIN);
159static EVENTS: Mutex<Vec<TrapEvent>> = Mutex::new(Vec::new());
160
161/// Patches every AArch64 `svc` in executable regions.
162///
163/// On Linux aarch64: prefers `bl` into a guest-mapped trampoline (no signal).
164/// Falls back to `brk #IMM` for out-of-range sites.
165pub fn patch_svc_to_brk(regions: &mut [MappedRegion]) -> Result<usize, TrapError> {
166    #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
167    {
168        patch_svc_linux(regions)
169    }
170    #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
171    {
172        patch_svc_brk_only(regions)
173    }
174}
175
176#[cfg_attr(
177    all(target_arch = "aarch64", target_os = "linux"),
178    allow(dead_code)
179)]
180fn patch_svc_brk_only(regions: &mut [MappedRegion]) -> Result<usize, TrapError> {
181    let mut patched = 0_usize;
182    for region in regions.iter_mut() {
183        if region.prot & VM_PROT_EXECUTE == 0 {
184            continue;
185        }
186        mprotect_rw(region)?;
187        let guest_base = region.guest_addr;
188        let bytes = region.host_bytes_mut();
189        let mut off = 0_usize;
190        while off.saturating_add(4) <= bytes.len() {
191            let end = off.saturating_add(4);
192            let Some(word) = bytes.get(off..end) else {
193                break;
194            };
195            let Ok(word_bytes) = <[u8; 4]>::try_from(word) else {
196                break;
197            };
198            let insn = u32::from_le_bytes(word_bytes);
199            if is_svc(insn) {
200                let brk = brk_encoding(BRK_TRAP_IMM).to_le_bytes();
201                if let Some(slot) = bytes.get_mut(off..end) {
202                    slot.copy_from_slice(&brk);
203                    patched = patched.saturating_add(1);
204                }
205            }
206            let _ = guest_base;
207            off = end;
208        }
209        let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
210        mprotect_darwin(region, restore)?;
211        region.prot = restore;
212    }
213    Ok(patched)
214}
215
216const fn is_svc(insn: u32) -> bool {
217    (insn & SVC_MASK) == SVC_BASE
218}
219
220/// Linux aarch64: optional userspace trampoline (`svc`→`b veneer`).
221///
222/// Enable with `KAKEHASHI_TRAMPOLINE=1`. Default is classic `brk` + `SIGTRAP`
223/// (safe fallback when freestanding hypercall is off). Prefer hypercall for
224/// multi-thread compression; the veneer path remains opt-in.
225///
226/// **Why a veneer (not a direct `bl`)?** Darwin `svc` preserves `x30` (LR). A
227/// direct `bl trampoline` overwrites LR with the resume PC; the libSystem stub
228/// then `ret`s to itself and spins. The veneer saves/restores the real LR.
229///
230/// The shared dispatcher also saves **Q0–Q31 + FPCR/FPSR** (Darwin `svc`
231/// preserves SIMD; Rust would otherwise clobber NEON used by 7zz workers).
232#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
233fn patch_svc_linux(regions: &mut [MappedRegion]) -> Result<usize, TrapError> {
234    if !trampoline_enabled() {
235        return patch_svc_brk_only(regions);
236    }
237    let mut patched = 0_usize;
238    for region in regions.iter_mut() {
239        if region.prot & VM_PROT_EXECUTE == 0 {
240            continue;
241        }
242        mprotect_rw(region)?;
243        let guest_base = region.guest_addr;
244        // Collect svc offsets first.
245        let mut sites = Vec::new();
246        {
247            let bytes = region.host_bytes();
248            let mut off = 0_usize;
249            while off.saturating_add(4) <= bytes.len() {
250                let end = off.saturating_add(4);
251                if let Some(word) = bytes.get(off..end)
252                    && let Ok(word_bytes) = <[u8; 4]>::try_from(word)
253                {
254                    let insn = u32::from_le_bytes(word_bytes);
255                    if is_svc(insn) {
256                        sites.push(off);
257                    }
258                }
259                off = end;
260            }
261        }
262        if sites.is_empty() {
263            let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
264            mprotect_darwin(region, restore)?;
265            region.prot = restore;
266            continue;
267        }
268
269        patched = patched.saturating_add(patch_region_with_veneers(region, guest_base, &sites));
270    }
271    Ok(patched)
272}
273
274/// Per-site veneer size: `str lr; bl shared; ldr lr; b site+4`.
275#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
276const VENEER_BYTES: usize = 16;
277
278/// How many page slots to probe above/below an RX region for a veneer hub.
279#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
280const VENEER_PROBE_RADIUS: u64 = 128;
281
282/// Round `len` up to a multiple of `page` (page must be a power of two ≥ 1).
283#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
284#[inline]
285fn align_up_to_page(len: usize, page: usize) -> usize {
286    if page <= 1 {
287        return len.max(1);
288    }
289    let mask = page.saturating_sub(1);
290    len.saturating_add(mask) & !mask
291}
292
293// libgcc / compiler-rt symbol linked on Linux aarch64 toolchains.
294#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
295unsafe extern "C" {
296    fn __clear_cache(start: *mut libc::c_void, end: *mut libc::c_void);
297}
298
299/// Opt-in trampoline: `KAKEHASHI_TRAMPOLINE=1|true|yes|on`.
300#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
301fn trampoline_enabled() -> bool {
302    match std::env::var_os("KAKEHASHI_TRAMPOLINE") {
303        None => false,
304        Some(v) => {
305            v == "1"
306                || v.eq_ignore_ascii_case("true")
307                || v.eq_ignore_ascii_case("yes")
308                || v.eq_ignore_ascii_case("on")
309        }
310    }
311}
312
313/// Flush D/I caches after writing executable code (aarch64 requirement).
314#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
315fn clear_icache(ptr: *mut u8, len: usize) {
316    if ptr.is_null() || len == 0 {
317        return;
318    }
319    // SAFETY: range is a live mapping the caller owns and just wrote.
320    unsafe {
321        let start = ptr.cast::<libc::c_void>();
322        let end = ptr.wrapping_add(len).cast::<libc::c_void>();
323        __clear_cache(start, end);
324    }
325}
326
327/// Drop trampoline bookkeeping (owned pages are munmap'd with the address space).
328pub fn clear_trampoline_cache() {
329    // Veneer hubs live only as `register_owned` mappings; nothing else to clear.
330}
331
332/// Patch one RX region: map shared dispatcher + per-site veneers near it.
333///
334/// Returns the number of sites rewritten (`b veneer` or fallback `brk`).
335#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
336fn patch_region_with_veneers(
337    region: &mut MappedRegion,
338    guest_base: u64,
339    sites: &[usize],
340) -> usize {
341    let n_sites = sites.len();
342    let Some(hub) = install_veneer_hub(guest_base, region.host_len(), n_sites) else {
343        return patch_sites_brk(region, sites);
344    };
345
346    let mut patched = 0_usize;
347    // SAFETY: hub mapping is RW until we mprotect RX below; length is hub.map_len.
348    let slice = unsafe { std::slice::from_raw_parts_mut(hub.ptr, hub.map_len) };
349
350    for (i, &off) in sites.iter().enumerate() {
351        let site_va = guest_base.saturating_add(u64::try_from(off).unwrap_or(0));
352        let site_ret = site_va.saturating_add(4);
353        let veneer_off = hub.veneer0.saturating_add(i.saturating_mul(VENEER_BYTES));
354        let veneer_va = hub.base_va.saturating_add(u64::try_from(veneer_off).unwrap_or(0));
355        let end = off.saturating_add(4);
356
357        let ok = write_veneer(slice, veneer_off, veneer_va, hub.shared_va, site_ret)
358            .and_then(|()| b_encoding(site_va, veneer_va))
359            .and_then(|enc| {
360                let bytes = region.host_bytes_mut();
361                let slot = bytes.get_mut(off..end)?;
362                slot.copy_from_slice(&enc.to_le_bytes());
363                Some(())
364            })
365            .is_some();
366
367        if !ok {
368            let bytes = region.host_bytes_mut();
369            let brk = brk_encoding(BRK_TRAP_IMM).to_le_bytes();
370            if let Some(slot) = bytes.get_mut(off..end) {
371                slot.copy_from_slice(&brk);
372            }
373        }
374        patched = patched.saturating_add(1);
375    }
376
377    // W^X on hub + I-cache; register owned RX.
378    let prot_rx = libc::PROT_READ | libc::PROT_EXEC;
379    if !crate::host::mprotect(hub.ptr, hub.map_len, prot_rx) {
380        let _ = crate::host::munmap(hub.ptr, hub.map_len);
381        return patch_sites_brk(region, sites);
382    }
383    clear_icache(hub.ptr, hub.map_len);
384    crate::mem::register_owned(
385        hub.base_va,
386        hub.ptr,
387        hub.map_len,
388        VM_PROT_READ | VM_PROT_EXECUTE,
389    );
390
391    let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
392    drop(mprotect_darwin(region, restore));
393    region.prot = restore;
394    let host = region.host_addr();
395    let len = region.host_len();
396    if let (Ok(base), true) = (usize::try_from(host), len > 0) {
397        let p = std::ptr::with_exposed_provenance_mut::<u8>(base);
398        clear_icache(p, len);
399    }
400    patched
401}
402
403#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
404fn patch_sites_brk(region: &mut MappedRegion, sites: &[usize]) -> usize {
405    let mut patched = 0_usize;
406    let bytes = region.host_bytes_mut();
407    for &off in sites {
408        let end = off.saturating_add(4);
409        let brk = brk_encoding(BRK_TRAP_IMM).to_le_bytes();
410        if let Some(slot) = bytes.get_mut(off..end) {
411            slot.copy_from_slice(&brk);
412            patched = patched.saturating_add(1);
413        }
414    }
415    let restore = (region.prot | VM_PROT_READ | VM_PROT_EXECUTE) & !VM_PROT_WRITE;
416    drop(mprotect_darwin(region, restore));
417    region.prot = restore;
418    patched
419}
420
421/// Shared dispatcher + veneer pool mapping.
422#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
423struct VeneerHub {
424    ptr: *mut u8,
425    map_len: usize,
426    base_va: u64,
427    shared_va: u64,
428    veneer0: usize,
429}
430
431/// Map RW hub near the RX region (enough room for `n_sites` veneers).
432#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
433fn install_veneer_hub(region_guest: u64, region_len: usize, n_sites: usize) -> Option<VeneerHub> {
434    let region_end = region_guest.saturating_add(u64::try_from(region_len).unwrap_or(0));
435    let site_lo = region_guest;
436    let site_hi = region_end.saturating_sub(4).max(region_guest);
437
438    let page = crate::host::page_size().unwrap_or(4096);
439    let page_u = u64::try_from(page).unwrap_or(4096);
440    let code = trampoline_template();
441    let veneer0 = code.len().saturating_add(15) & !15_usize;
442    let need = veneer0.saturating_add(n_sites.saturating_mul(VENEER_BYTES));
443    let map_len = align_up_to_page(need, page).max(page);
444
445    let flags_fixed = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | crate::host::fixed_map_flag();
446    let flags_free = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
447    let prot_rw = libc::PROT_READ | libc::PROT_WRITE;
448
449    let after0 = region_end.saturating_add(page_u.saturating_sub(1)) & !(page_u.saturating_sub(1));
450    let before0 = region_guest.saturating_sub(page_u) & !(page_u.saturating_sub(1));
451
452    for i in 0..VENEER_PROBE_RADIUS {
453        let mut candidates = [after0.saturating_add(i.saturating_mul(page_u)), 0_u64];
454        if before0 >= i.saturating_mul(page_u) {
455            candidates[1] = before0.saturating_sub(i.saturating_mul(page_u));
456        }
457        for &prefer in &candidates {
458            if prefer == 0 {
459                continue;
460            }
461            // Sites must `b` to veneers in this hub; veneers `bl` shared at base.
462            if b_encoding(site_lo, prefer).is_none() || b_encoding(site_hi, prefer).is_none() {
463                continue;
464            }
465            if let Some(ptr) =
466                crate::host::mmap(Some(prefer), map_len, prot_rw, flags_fixed, -1, 0)
467            {
468                return init_hub_slice(ptr, map_len, code, veneer0);
469            }
470        }
471    }
472
473    let ptr = crate::host::mmap(None, map_len, prot_rw, flags_free, -1, 0)?;
474    let va = crate::host::ptr_addr_u64(ptr);
475    if b_encoding(site_lo, va).is_none() || b_encoding(site_hi, va).is_none() {
476        let _ = crate::host::munmap(ptr, map_len);
477        return None;
478    }
479    init_hub_slice(ptr, map_len, code, veneer0)
480}
481
482#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
483fn init_hub_slice(ptr: *mut u8, map_len: usize, code: &[u8], veneer0: usize) -> Option<VeneerHub> {
484    // SAFETY: fresh RW mapping of `map_len`.
485    let slice = unsafe { std::slice::from_raw_parts_mut(ptr, map_len) };
486    if code.len() > slice.len() || veneer0 > slice.len() {
487        let _ = crate::host::munmap(ptr, map_len);
488        return None;
489    }
490    slice.fill(0);
491    slice.get_mut(..code.len())?.copy_from_slice(code);
492    let reloc_off = trampoline_reloc_offset();
493    if let Some(slot) = slice.get_mut(reloc_off..reloc_off.saturating_add(8)) {
494        slot.copy_from_slice(&trampoline_rust_entry_addr().to_le_bytes());
495    } else {
496        let _ = crate::host::munmap(ptr, map_len);
497        return None;
498    }
499    let base_va = crate::host::ptr_addr_u64(ptr);
500    Some(VeneerHub {
501        ptr,
502        map_len,
503        base_va,
504        shared_va: base_va,
505        veneer0,
506    })
507}
508
509/// Emit one veneer: save LR → bl shared → restore LR → b resume.
510#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
511fn write_veneer(
512    slice: &mut [u8],
513    veneer_off: usize,
514    veneer_va: u64,
515    shared_va: u64,
516    site_ret: u64,
517) -> Option<()> {
518    // str x30, [sp, #-16]!
519    let str_lr: u32 = 0xf81f_0ffe;
520    // ldr x30, [sp], #16
521    let ldr_lr: u32 = 0xf841_07fe;
522    let bl = bl_encoding(veneer_va.saturating_add(4), shared_va)?;
523    let b_back = b_encoding(veneer_va.saturating_add(12), site_ret)?;
524    let words = [str_lr, bl, ldr_lr, b_back];
525    let end = veneer_off.saturating_add(VENEER_BYTES);
526    let slot = slice.get_mut(veneer_off..end)?;
527    for (i, w) in words.iter().enumerate() {
528        let o = i.saturating_mul(4);
529        slot.get_mut(o..o.saturating_add(4))?.copy_from_slice(&w.to_le_bytes());
530    }
531    Some(())
532}
533
534/// Verified aarch64 trampoline (no host-relative BL).
535///
536/// Layout: code bytes + 8-byte absolute pointer (`kh_trampoline_dispatch`).
537#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
538fn trampoline_template() -> &'static [u8] {
539    TRAMP_BYTES
540}
541
542#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
543fn trampoline_reloc_offset() -> usize {
544    TRAMP_RELOC_OFF
545}
546
547#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
548#[allow(clippy::as_conversions, function_casts_as_integer)]
549fn trampoline_rust_entry_addr() -> u64 {
550    u64::try_from(kh_trampoline_dispatch as usize).unwrap_or(0)
551}
552
553/// Return value of the trampoline dispatch (x0=retval, x1=error flag).
554#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
555#[repr(C)]
556pub struct TrampRet {
557    /// Value for guest `x0`.
558    pub retval: u64,
559    /// Non-zero → set Darwin carry (error path).
560    pub error: u64,
561}
562
563/// Called from the guest-mapped trampoline (AAPCS64).
564///
565/// # Safety
566///
567/// Caller must preserve guest SIMD/GPRs around the call (veneer trampoline or
568/// freestanding NEON shim). Arguments are Darwin BSD syscall registers;
569/// `x16` holds the syscall number (or AAPCS 8th arg after the hypercall
570/// prologue). May end the current host worker thread on
571/// `bsdthread_terminate` (`SYS_exit`, not `pthread_exit`), or `_exit` the
572/// process on guest `exit`.
573///
574/// Prefer [`kh_hypercall_entry`] for freestanding libSystem: that path switches
575/// onto a host alt stack first so multi-thread guests never run host Rust on
576/// guest worker stacks.
577#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
578#[unsafe(no_mangle)]
579pub unsafe extern "C" fn kh_trampoline_dispatch(
580    x0: u64,
581    x1: u64,
582    x2: u64,
583    x3: u64,
584    x4: u64,
585    x5: u64,
586    x6: u64,
587    x16: u64,
588) -> TrampRet {
589    let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
590    let result = syscall::dispatch(SyscallArgs {
591        pc: 0,
592        number: sys_no,
593        x0,
594        x1,
595        x2,
596        x3,
597        x4,
598        x5,
599        x6,
600    });
601    if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
602        maybe_push_event(0, sys_no, result.name, x0, x1, x2, result.retval, result.error);
603    }
604    match result.outcome {
605        TrapOutcome::Exit { code } => finish_with_exit_code(code),
606        TrapOutcome::ThreadExit => {
607            // End this host worker only (same as bsdthread_terminate).
608            crate::thread::exit_current_guest_worker();
609        }
610        TrapOutcome::Continue => {}
611    }
612    TrampRet {
613        retval: result.retval.unwrap_or(0),
614        error: u64::from(result.error),
615    }
616}
617
618// Freestanding hypercall entry (`kh_hypercall_entry`):
619// Prefer host alt stack (so join/terminate never runs host Rust on a guest
620// stack that `pthread_join` may unmap).
621//
622// **NEON:** Darwin `svc` preserves Q0–Q31. We must too. Any `bl` into Rust/C
623// (TLS enter, alt-stack map) clobbers caller-saved SIMD under AAPCS64, so full
624// Q0–Q31 + FPCR/FPSR are saved on the *guest* prolog frame *before* any `bl`.
625// TRAMP_BYTES still saves around the dispatch body (belt-and-braces).
626//
627// ABI: x0–x6 + number in x7 → TrampRet in x0/x1.
628// Guest frame layout (640 B):
629//   [0]   x29,x30
630//   [16]  x0,x1  [32] x2,x3  [48] x4,x5  [64] x6,x7
631//   [80]  pad
632//   [96]  q0..q31 (512 B)
633//   [608] fpcr,fpsr
634#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
635std::arch::global_asm!(
636    r#"
637    .text
638    .align 2
639    .global kh_hypercall_entry
640    .type kh_hypercall_entry, %function
641kh_hypercall_entry:
642    // Guest prolog: GPRs + full NEON *before* any bl (NEON must survive host TLS).
643    sub sp, sp, #640
644    stp x29, x30, [sp, #0]
645    stp x0, x1, [sp, #16]
646    stp x2, x3, [sp, #32]
647    stp x4, x5, [sp, #48]
648    stp x6, x7, [sp, #64]
649    mov x29, sp
650    stp q0, q1, [sp, #96]
651    stp q2, q3, [sp, #128]
652    stp q4, q5, [sp, #160]
653    stp q6, q7, [sp, #192]
654    stp q8, q9, [sp, #224]
655    stp q10, q11, [sp, #256]
656    stp q12, q13, [sp, #288]
657    stp q14, q15, [sp, #320]
658    stp q16, q17, [sp, #352]
659    stp q18, q19, [sp, #384]
660    stp q20, q21, [sp, #416]
661    stp q22, q23, [sp, #448]
662    stp q24, q25, [sp, #480]
663    stp q26, q27, [sp, #512]
664    stp q28, q29, [sp, #544]
665    stp q30, q31, [sp, #576]
666    mrs x9, fpcr
667    mrs x10, fpsr
668    // #608 exceeds signed LDP imm7*8 range (±512); use base+add.
669    add x11, sp, #608
670    stp x9, x10, [x11]
671
672    // Host glibc TLS *before* alt-stack lookup/alloc (mmap/mutex need host TPIDR).
673    bl kh_tls_enter_host
674    bl kh_host_alt_sp
675    cbz x0, 1f
676    mov x9, sp                 // guest prolog frame
677    mov sp, x0                 // host alt stack
678    ldr x30, [x9, #8]          // reload LR → freestanding thin caller
679    // Host frame: [guest_frame, lr]
680    stp x9, x30, [sp, #-16]!
681    // Reload guest frame ptr / args (enter_host + alt_sp clobbered temps).
682    ldr x9, [sp]
683    ldp x0, x1, [x9, #16]
684    ldp x2, x3, [x9, #32]
685    ldp x4, x5, [x9, #48]
686    ldp x6, x7, [x9, #64]
687    mov x16, x7
688    bl kh_neon_tramp_entry
689    // Preserve TrampRet (x0/x1) across TLS restore.
690    stp x0, x1, [sp, #-16]!
691    bl kh_tls_leave_host
692    ldp x0, x1, [sp], #16
693    ldp x9, x30, [sp], #16
694    mov sp, x9
695    // Restore guest NEON from prolog (pre-bl snapshot).
696    ldp q0, q1, [sp, #96]
697    ldp q2, q3, [sp, #128]
698    ldp q4, q5, [sp, #160]
699    ldp q6, q7, [sp, #192]
700    ldp q8, q9, [sp, #224]
701    ldp q10, q11, [sp, #256]
702    ldp q12, q13, [sp, #288]
703    ldp q14, q15, [sp, #320]
704    ldp q16, q17, [sp, #352]
705    ldp q18, q19, [sp, #384]
706    ldp q20, q21, [sp, #416]
707    ldp q22, q23, [sp, #448]
708    ldp q24, q25, [sp, #480]
709    ldp q26, q27, [sp, #512]
710    ldp q28, q29, [sp, #544]
711    ldp q30, q31, [sp, #576]
712    add x11, sp, #608
713    ldp x9, x10, [x11]
714    msr fpcr, x9
715    msr fpsr, x10
716    ldr x29, [sp, #0]
717    add sp, sp, #640
718    ret
7191:
720    // No alt stack: dispatch on the guest stack (ST / fallback only).
721    // Already on host TLS from the enter above.
722    ldp x0, x1, [sp, #16]
723    ldp x2, x3, [sp, #32]
724    ldp x4, x5, [sp, #48]
725    ldp x6, x7, [sp, #64]
726    mov x16, x7
727    // Preserve real LR across the call (still on guest prolog frame).
728    ldr x30, [sp, #8]
729    bl kh_neon_tramp_entry
730    stp x0, x1, [sp, #-16]!
731    bl kh_tls_leave_host
732    ldp x0, x1, [sp], #16
733    // Restore guest NEON from prolog.
734    ldp q0, q1, [sp, #96]
735    ldp q2, q3, [sp, #128]
736    ldp q4, q5, [sp, #160]
737    ldp q6, q7, [sp, #192]
738    ldp q8, q9, [sp, #224]
739    ldp q10, q11, [sp, #256]
740    ldp q12, q13, [sp, #288]
741    ldp q14, q15, [sp, #320]
742    ldp q16, q17, [sp, #352]
743    ldp q18, q19, [sp, #384]
744    ldp q20, q21, [sp, #416]
745    ldp q22, q23, [sp, #448]
746    ldp q24, q25, [sp, #480]
747    ldp q26, q27, [sp, #512]
748    ldp q28, q29, [sp, #544]
749    ldp q30, q31, [sp, #576]
750    add x11, sp, #608
751    ldp x9, x10, [x11]
752    msr fpcr, x9
753    msr fpsr, x10
754    ldp x29, x30, [sp, #0]
755    add sp, sp, #640
756    ret
757    .size kh_hypercall_entry, .-kh_hypercall_entry
758    "#
759);
760
761// `kh_neon_tramp_entry` + `KH_NEON_TRAMP_VA` (filled by `ensure_neon_tramp`).
762#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
763std::arch::global_asm!(
764    r#"
765    .text
766    .align 2
767    .global kh_neon_tramp_entry
768    .type kh_neon_tramp_entry, %function
769kh_neon_tramp_entry:
770    adrp x17, KH_NEON_TRAMP_VA
771    ldr x17, [x17, :lo12:KH_NEON_TRAMP_VA]
772    cbz x17, 1f
773    br x17
7741:
775    mov x7, x16
776    b kh_trampoline_dispatch
777    .size kh_neon_tramp_entry, .-kh_neon_tramp_entry
778
779    .data
780    .align 3
781    .global KH_NEON_TRAMP_VA
782KH_NEON_TRAMP_VA:
783    .quad 0
784    "#
785);
786
787#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
788unsafe extern "C" {
789    fn kh_hypercall_entry();
790    static mut KH_NEON_TRAMP_VA: u64;
791}
792
793/// Address of freestanding hypercall entry (alt stack + NEON tramp + dispatch).
794#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
795#[must_use]
796pub fn hypercall_entry_addr() -> u64 {
797    ensure_neon_tramp();
798    #[allow(clippy::as_conversions, function_casts_as_integer)]
799    {
800        u64::try_from(kh_hypercall_entry as usize).unwrap_or(0)
801    }
802}
803
804/// Map TRAMP_BYTES once and publish its VA to `KH_NEON_TRAMP_VA`.
805#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
806fn ensure_neon_tramp() {
807    static ONCE: AtomicU64 = AtomicU64::new(0);
808    static LOCK: Mutex<()> = Mutex::new(());
809
810    if ONCE.load(Ordering::Acquire) != 0 {
811        return;
812    }
813    let _g = LOCK
814        .lock()
815        .unwrap_or_else(std::sync::PoisonError::into_inner);
816    if ONCE.load(Ordering::Acquire) != 0 {
817        return;
818    }
819
820    let code = trampoline_template();
821    let page = crate::host::page_size().unwrap_or(4096);
822    let map_len = align_up_to_page(code.len(), page).max(page);
823    let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
824    let prot_rw = libc::PROT_READ | libc::PROT_WRITE;
825    let Some(ptr) = crate::host::mmap(None, map_len, prot_rw, flags, -1, 0) else {
826        tracing::error!("NEON tramp mmap failed");
827        return;
828    };
829    // SAFETY: fresh RW mapping.
830    let slice = unsafe { std::slice::from_raw_parts_mut(ptr, map_len) };
831    slice.fill(0);
832    if let Some(dst) = slice.get_mut(..code.len()) {
833        dst.copy_from_slice(code);
834    } else {
835        let _ = crate::host::munmap(ptr, map_len);
836        return;
837    }
838    let reloc_off = trampoline_reloc_offset();
839    if let Some(slot) = slice.get_mut(reloc_off..reloc_off.saturating_add(8)) {
840        slot.copy_from_slice(&trampoline_rust_entry_addr().to_le_bytes());
841    } else {
842        let _ = crate::host::munmap(ptr, map_len);
843        return;
844    }
845    let prot_rx = libc::PROT_READ | libc::PROT_EXEC;
846    if !crate::host::mprotect(ptr, map_len, prot_rx) {
847        tracing::error!("NEON tramp mprotect RX failed");
848        let _ = crate::host::munmap(ptr, map_len);
849        return;
850    }
851    clear_icache(ptr, map_len);
852    let va = crate::host::ptr_addr_u64(ptr);
853    // SAFETY: symbol from global_asm; written once under LOCK before guest entry.
854    unsafe {
855        KH_NEON_TRAMP_VA = va;
856    }
857    ONCE.store(va, Ordering::Release);
858    tracing::info!(va = format_args!("{va:#x}"), "mapped NEON tramp for hypercall");
859}
860
861#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
862#[must_use]
863pub fn hypercall_entry_addr() -> u64 {
864    0
865}
866
867// ── trampoline machine code (verified via rustc/objdump aarch64) ─────────────
868
869/// Offset of the 8-byte absolute `kh_trampoline_dispatch` pointer.
870///
871/// Layout (see `TRAMP_BYTES`): `bti c` + save GPRs/NEON/FPCR + `blr` dispatch
872/// + restore + `ret` + `.quad` reloc @ 0x100.
873#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
874const TRAMP_RELOC_OFF: usize = 0x100;
875
876/// Exact opcodes from assembled trampoline (`as` / objdump on Linux aarch64).
877///
878/// Saves **all Q0–Q31 + FPCR/FPSR** around the Rust call: Darwin `svc` preserves
879/// SIMD state; compression workers (7zz MT) rely on that. A bare GPR-only
880/// trampoline clobbered NEON and faulted under `-mmt>1`.
881#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
882#[rustfmt::skip]
883const TRAMP_BYTES: &[u8] = &[
884    0x5f, 0x24, 0x03, 0xd5, 0xff, 0x03, 0x0a, 0xd1, 0xe8, 0x27, 0x00, 0xa9, 0xea, 0x2f, 0x01, 0xa9,
885    0xec, 0x37, 0x02, 0xa9, 0xee, 0x3f, 0x03, 0xa9, 0xf1, 0x4b, 0x04, 0xa9, 0xfe, 0x2b, 0x00, 0xf9,
886    0x08, 0x44, 0x3b, 0xd5, 0x29, 0x44, 0x3b, 0xd5, 0xe8, 0x27, 0x06, 0xa9, 0xe0, 0x87, 0x03, 0xad,
887    0xe2, 0x8f, 0x04, 0xad, 0xe4, 0x97, 0x05, 0xad, 0xe6, 0x9f, 0x06, 0xad, 0xe8, 0xa7, 0x07, 0xad,
888    0xea, 0xaf, 0x08, 0xad, 0xec, 0xb7, 0x09, 0xad, 0xee, 0xbf, 0x0a, 0xad, 0xf0, 0xc7, 0x0b, 0xad,
889    0xf2, 0xcf, 0x0c, 0xad, 0xf4, 0xd7, 0x0d, 0xad, 0xf6, 0xdf, 0x0e, 0xad, 0xf8, 0xe7, 0x0f, 0xad,
890    0xfa, 0xef, 0x10, 0xad, 0xfc, 0xf7, 0x11, 0xad, 0xfe, 0xff, 0x12, 0xad, 0xe7, 0x03, 0x10, 0xaa,
891    0x88, 0x04, 0x00, 0x58, 0x00, 0x01, 0x3f, 0xd6, 0x02, 0x42, 0x3b, 0xd5, 0x23, 0x00, 0x80, 0x52,
892    0x63, 0x88, 0x63, 0xd3, 0x42, 0x00, 0x23, 0x8a, 0x41, 0x00, 0x00, 0xb4, 0x42, 0x00, 0x03, 0xaa,
893    0x02, 0x42, 0x1b, 0xd5, 0xe8, 0x27, 0x46, 0xa9, 0x08, 0x44, 0x1b, 0xd5, 0x29, 0x44, 0x1b, 0xd5,
894    0xe0, 0x87, 0x43, 0xad, 0xe2, 0x8f, 0x44, 0xad, 0xe4, 0x97, 0x45, 0xad, 0xe6, 0x9f, 0x46, 0xad,
895    0xe8, 0xa7, 0x47, 0xad, 0xea, 0xaf, 0x48, 0xad, 0xec, 0xb7, 0x49, 0xad, 0xee, 0xbf, 0x4a, 0xad,
896    0xf0, 0xc7, 0x4b, 0xad, 0xf2, 0xcf, 0x4c, 0xad, 0xf4, 0xd7, 0x4d, 0xad, 0xf6, 0xdf, 0x4e, 0xad,
897    0xf8, 0xe7, 0x4f, 0xad, 0xfa, 0xef, 0x50, 0xad, 0xfc, 0xf7, 0x51, 0xad, 0xfe, 0xff, 0x52, 0xad,
898    0xfe, 0x2b, 0x40, 0xf9, 0xf1, 0x4b, 0x44, 0xa9, 0xee, 0x3f, 0x43, 0xa9, 0xec, 0x37, 0x42, 0xa9,
899    0xea, 0x2f, 0x41, 0xa9, 0xe8, 0x27, 0x40, 0xa9, 0xff, 0x03, 0x0a, 0x91, 0xc0, 0x03, 0x5f, 0xd6,
900    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // reloc .quad @ 0x100
901];
902/// Installs the `SIGTRAP` handler used by the micro execution backend.
903pub fn install_trap_handlers(config: &TrapConfig) -> Result<(), TrapError> {
904    MAX_EVENTS.store(
905        u64::try_from(config.max_events).unwrap_or(256),
906        Ordering::SeqCst,
907    );
908    syscall::reset_syscall_state(config.max_syscalls);
909
910    #[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
911    {
912        let _ = config;
913        Err(TrapError::UnsupportedArch)
914    }
915
916    #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
917    {
918        if HANDLER_INSTALLED.swap(true, Ordering::SeqCst) {
919            reset_trap_state();
920            syscall::reset_syscall_state(config.max_syscalls);
921            return Ok(());
922        }
923
924        // Guest faults used to leave multi‑MB `core` files in the workspace cwd.
925        // Faults go through our handler → `_exit`; disable dumps as belt-and-braces.
926        disable_core_dumps();
927
928        // SAFETY: process-wide SIGTRAP / fault handlers with SA_SIGINFO.
929        unsafe {
930            let mut sa: libc::sigaction = std::mem::zeroed();
931            sa.sa_flags = libc::SA_SIGINFO;
932            #[allow(clippy::as_conversions, function_casts_as_integer)]
933            {
934                sa.sa_sigaction = trap_sigaction as *const () as usize;
935            }
936            libc::sigemptyset(std::ptr::addr_of_mut!(sa.sa_mask));
937            if libc::sigaction(libc::SIGTRAP, std::ptr::addr_of!(sa), std::ptr::null_mut()) != 0 {
938                HANDLER_INSTALLED.store(false, Ordering::SeqCst);
939                return Err(TrapError::SignalSetup(io::Error::last_os_error()));
940            }
941
942            // Guest faults: print PC/addr so `kh run` diagnoses unbound GOT / bad heap.
943            let mut fault: libc::sigaction = std::mem::zeroed();
944            fault.sa_flags = libc::SA_SIGINFO;
945            #[allow(clippy::as_conversions, function_casts_as_integer)]
946            {
947                fault.sa_sigaction = guest_fault_sigaction as *const () as usize;
948            }
949            libc::sigemptyset(std::ptr::addr_of_mut!(fault.sa_mask));
950            for sig in [libc::SIGSEGV, libc::SIGBUS] {
951                if libc::sigaction(sig, std::ptr::addr_of!(fault), std::ptr::null_mut()) != 0 {
952                    HANDLER_INSTALLED.store(false, Ordering::SeqCst);
953                    return Err(TrapError::SignalSetup(io::Error::last_os_error()));
954                }
955            }
956        }
957        reset_trap_state();
958        Ok(())
959    }
960}
961
962/// Sets `RLIMIT_CORE` soft+hard to 0 so guest crashes never litter the repo with
963/// `core` dumps (we already print a guest fault summary and `_exit`).
964#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
965fn disable_core_dumps() {
966    // SAFETY: process-wide rlimit; best-effort, ignore failure.
967    unsafe {
968        let lim = libc::rlimit {
969            rlim_cur: 0,
970            rlim_max: 0,
971        };
972        let _ = libc::setrlimit(libc::RLIMIT_CORE, std::ptr::addr_of!(lim));
973    }
974}
975
976/// Logs guest `SIGSEGV` / `SIGBUS` (PC, fault address, key regs) then exits.
977#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
978unsafe extern "C" fn guest_fault_sigaction(
979    signo: libc::c_int,
980    info: *mut libc::siginfo_t,
981    ctx: *mut libc::c_void,
982) {
983    // May run under guest TPIDR; restore host before formatting / write.
984    crate::tls::enter_host_tls();
985    let name = match signo {
986        libc::SIGSEGV => "SIGSEGV",
987        libc::SIGBUS => "SIGBUS",
988        _ => "FAULT",
989    };
990    let mut pc = 0_u64;
991    let mut sp = 0_u64;
992    let mut x0 = 0_u64;
993    let mut x1 = 0_u64;
994    let mut x8 = 0_u64;
995    let mut x16 = 0_u64;
996    let mut lr = 0_u64;
997    if !ctx.is_null() {
998        // SAFETY: kernel `ucontext_t` for SA_SIGINFO.
999        let uctx = unsafe { &*ctx.cast::<libc::ucontext_t>() };
1000        let m = &uctx.uc_mcontext;
1001        pc = m.pc;
1002        sp = m.sp;
1003        x0 = m.regs[0];
1004        x1 = m.regs[1];
1005        x8 = m.regs[8];
1006        x16 = m.regs[16];
1007        lr = m.regs[30];
1008    }
1009    let addr = if info.is_null() {
1010        0_u64
1011    } else {
1012        // SAFETY: kernel-provided siginfo.
1013        let p = unsafe { (*info).si_addr() };
1014        u64::try_from(p.addr()).unwrap_or(0)
1015    };
1016    let msg = format!(
1017        "error: guest {name} pc={pc:#x} addr={addr:#x} sp={sp:#x} lr={lr:#x} \
1018         x0={x0:#x} x1={x1:#x} x8={x8:#x} x16={x16:#x}\n"
1019    );
1020    drop(io::stderr().write_all(msg.as_bytes()));
1021    // Best-effort: which mapping owns PC / fault address (helps MT diagnosis).
1022    if let Ok(maps) = std::fs::read_to_string("/proc/self/maps") {
1023        for &va in &[pc, addr, lr, x16] {
1024            for line in maps.lines() {
1025                let Some((range, rest)) = line.split_once(' ') else {
1026                    continue;
1027                };
1028                let Some((a, b)) = range.split_once('-') else {
1029                    continue;
1030                };
1031                let (Ok(lo), Ok(hi)) = (u64::from_str_radix(a, 16), u64::from_str_radix(b, 16))
1032                else {
1033                    continue;
1034                };
1035                if va >= lo && va < hi {
1036                    let off = va.saturating_sub(lo);
1037                    let m = format!("  map {va:#x} = {lo:#x}+{off:#x} {rest}\n");
1038                    drop(io::stderr().write_all(m.as_bytes()));
1039                    break;
1040                }
1041            }
1042        }
1043    }
1044    drop(io::stderr().flush());
1045    if TRACE_ON_EXIT.load(Ordering::SeqCst) {
1046        dump_events_before_exit();
1047    }
1048    // SAFETY: hard-stop after unrecoverable guest fault.
1049    unsafe { libc::_exit(128_i32.saturating_add(signo)) };
1050}
1051
1052/// Drains recorded trap events (for `kh trace`).
1053#[must_use]
1054pub fn take_trace_events() -> Vec<TrapEvent> {
1055    EVENTS
1056        .lock()
1057        .map(|mut g| std::mem::take(&mut *g))
1058        .unwrap_or_default()
1059}
1060
1061/// Whether the guest requested exit.
1062#[must_use]
1063pub fn exit_requested() -> bool {
1064    EXIT_REQUESTED.load(Ordering::SeqCst)
1065}
1066
1067/// Exit code from the last guest `exit` trap.
1068#[must_use]
1069pub fn take_exit_code() -> Option<i32> {
1070    if EXIT_REQUESTED.load(Ordering::SeqCst) {
1071        Some(EXIT_CODE.load(Ordering::SeqCst))
1072    } else {
1073        None
1074    }
1075}
1076
1077/// Resets exit / event state before a new run.
1078pub fn reset_trap_state() {
1079    EXIT_REQUESTED.store(false, Ordering::SeqCst);
1080    EXIT_CODE.store(0, Ordering::SeqCst);
1081    EVENT_COUNT.store(0, Ordering::SeqCst);
1082    if let Ok(mut guard) = EVENTS.lock() {
1083        guard.clear();
1084    }
1085}
1086
1087/// When set, the exit trap prints recorded events to stdout before `_exit`.
1088pub fn set_trace_on_exit(json: bool) {
1089    TRACE_ON_EXIT.store(true, Ordering::SeqCst);
1090    TRACE_JSON.store(json, Ordering::SeqCst);
1091}
1092
1093/// Clears the trace-on-exit flag (default for plain `kh run`).
1094pub fn clear_trace_on_exit() {
1095    TRACE_ON_EXIT.store(false, Ordering::SeqCst);
1096    TRACE_JSON.store(false, Ordering::SeqCst);
1097}
1098
1099/// Sets the expected guest exit code for the micro gate (`--expect-code`).
1100pub fn set_expect_code(code: i32) {
1101    EXPECT_CODE.store(code, Ordering::SeqCst);
1102}
1103
1104/// Disables the expect-code gate.
1105pub fn clear_expect_code() {
1106    EXPECT_CODE.store(i32::MIN, Ordering::SeqCst);
1107}
1108
1109/// Terminates the host process as if the guest called BSD `exit(code)`.
1110///
1111/// Used after `LC_MAIN` returns (dyld calls `exit` with `main`'s return value).
1112/// Honours `--expect-code` and optional trace-on-exit dump.
1113pub fn finish_with_exit_code(code: i32) -> ! {
1114    EXIT_CODE.store(code, Ordering::SeqCst);
1115    EXIT_REQUESTED.store(true, Ordering::SeqCst);
1116    if TRACE_ON_EXIT.load(Ordering::SeqCst) {
1117        dump_events_before_exit();
1118    }
1119    let expect = EXPECT_CODE.load(Ordering::SeqCst);
1120    let status = if expect == i32::MIN || expect == code {
1121        code
1122    } else {
1123        let msg = format!("error: guest exit code {code} != expected {expect}\n");
1124        drop(io::stderr().write_all(msg.as_bytes()));
1125        1
1126    };
1127    // SAFETY: intentional process termination after guest main return / exit.
1128    unsafe { libc::_exit(status) };
1129}
1130
1131#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
1132unsafe extern "C" fn trap_sigaction(
1133    signo: libc::c_int,
1134    _info: *mut libc::siginfo_t,
1135    ctx: *mut libc::c_void,
1136) {
1137    if signo != libc::SIGTRAP || ctx.is_null() {
1138        return;
1139    }
1140
1141    // Guest may have TPIDR_EL0 = guest TSD; restore host TLS before Rust/libc.
1142    crate::tls::enter_host_tls();
1143
1144    // SAFETY: kernel provides a valid `ucontext_t` for SA_SIGINFO deliveries.
1145    let uctx = unsafe { &mut *ctx.cast::<libc::ucontext_t>() };
1146    let m = &mut uctx.uc_mcontext;
1147    let pc = m.pc;
1148    // Only pull the regs dispatch needs; avoid extra loads on the hot path.
1149    let x0 = m.regs[0];
1150    let x1 = m.regs[1];
1151    let x2 = m.regs[2];
1152    let x3 = m.regs[3];
1153    let x4 = m.regs[4];
1154    let x5 = m.regs[5];
1155    let x6 = m.regs[6];
1156    let x16 = m.regs[16];
1157
1158    let sys_no = u32::try_from(x16 & 0xFFFF_FFFF).unwrap_or(u32::MAX);
1159    let result = syscall::dispatch(SyscallArgs {
1160        pc,
1161        number: sys_no,
1162        x0,
1163        x1,
1164        x2,
1165        x3,
1166        x4,
1167        x5,
1168        x6,
1169    });
1170
1171    // Zero-cost when max_events == 0 (plain `kh run`).
1172    if MAX_EVENTS.load(Ordering::Relaxed) != 0 {
1173        maybe_push_event(pc, sys_no, result.name, x0, x1, x2, result.retval, result.error);
1174    }
1175
1176    if let Some(ret) = result.retval {
1177        m.regs[0] = ret;
1178    }
1179    // Darwin arm64: error → set C; success with write-back → clear C.
1180    if result.error {
1181        m.pstate |= PSTATE_C;
1182    } else if result.retval.is_some() {
1183        m.pstate &= !PSTATE_C;
1184    }
1185
1186    match result.outcome {
1187        TrapOutcome::Exit { code } => {
1188            finish_with_exit_code(code);
1189        }
1190        TrapOutcome::Continue => {
1191            m.pc = pc.wrapping_add(4);
1192            // Return to guest with guest TPIDR restored.
1193            crate::tls::leave_host_tls();
1194        }
1195        TrapOutcome::ThreadExit => {
1196            // Stay on host TLS; exit trampoline publishes join then pthread_exit.
1197            // Redirect ucontext to a host `pthread_exit` trampoline on this
1198            // thread's original stack (see `crate::thread`). Main thread
1199            // (no host frame) falls back to process exit.
1200            if !crate::thread::redirect_ucontext_to_host_exit(m) {
1201                finish_with_exit_code(0);
1202            }
1203        }
1204    }
1205}
1206
1207fn dump_events_before_exit() {
1208    use std::fmt::Write as _;
1209
1210    let json = TRACE_JSON.load(Ordering::SeqCst);
1211    let events = EVENTS.lock().map(|g| g.clone()).unwrap_or_default();
1212    if json {
1213        let mut out = String::from("{\"events\":[");
1214        for (i, e) in events.iter().enumerate() {
1215            if i > 0 {
1216                out.push(',');
1217            }
1218            let sys = e
1219                .syscall
1220                .map_or_else(|| "null".to_owned(), |s| s.to_string());
1221            let ret = e
1222                .retval
1223                .map_or_else(|| "null".to_owned(), |v| v.to_string());
1224            let _ = write!(
1225                out,
1226                "{{\"pc\":{},\"syscall\":{},\"name\":\"{}\",\"arg0\":{},\"arg1\":{},\"arg2\":{},\"retval\":{},\"error\":{}}}",
1227                e.pc, sys, e.name, e.arg0, e.arg1, e.arg2, ret, e.error
1228            );
1229        }
1230        out.push_str("]}\n");
1231        drop(io::stdout().write_all(out.as_bytes()));
1232        drop(io::stdout().flush());
1233    } else {
1234        let header = format!("trace events: {}\n", events.len());
1235        drop(io::stdout().write_all(header.as_bytes()));
1236        for (i, e) in events.iter().enumerate() {
1237            let ret = e
1238                .retval
1239                .map_or_else(|| "-".to_owned(), |v| format!("{v:#x}"));
1240            let err = if e.error { " err" } else { "" };
1241            let line = format!(
1242                "  [{i}] pc={:#x} {} sys={:?} args=({:#x}, {:#x}, {:#x}) ret={ret}{err}\n",
1243                e.pc, e.name, e.syscall, e.arg0, e.arg1, e.arg2
1244            );
1245            drop(io::stdout().write_all(line.as_bytes()));
1246        }
1247        drop(io::stdout().flush());
1248    }
1249}
1250
1251#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
1252#[allow(clippy::too_many_arguments)]
1253fn maybe_push_event(
1254    pc: u64,
1255    sys_no: u32,
1256    name: &'static str,
1257    arg0: u64,
1258    arg1: u64,
1259    arg2: u64,
1260    retval: Option<u64>,
1261    error: bool,
1262) {
1263    let max = MAX_EVENTS.load(Ordering::Relaxed);
1264    // `kh run` sets max_events=0: single load, no String/Mutex.
1265    if max == 0 {
1266        return;
1267    }
1268    let n = EVENT_COUNT.fetch_add(1, Ordering::Relaxed);
1269    if n >= max {
1270        return;
1271    }
1272    if let Ok(mut guard) = EVENTS.lock()
1273        && u64::try_from(guard.len()).unwrap_or(u64::MAX) < max
1274    {
1275        guard.push(TrapEvent {
1276            pc,
1277            syscall: Some(sys_no),
1278            name: name.to_owned(),
1279            arg0,
1280            arg1,
1281            arg2,
1282            retval,
1283            error,
1284        });
1285    }
1286}
1287
1288#[cfg(test)]
1289mod tests {
1290    use super::*;
1291
1292    #[test]
1293    fn detects_svc_encoding() {
1294        assert!(is_svc(0xD400_0001));
1295        let svc80 = 0xD400_0001 | (0x80_u32 << 5);
1296        assert!(is_svc(svc80));
1297        assert!(!is_svc(0xD503_201F));
1298        assert!(!is_svc(brk_encoding(BRK_TRAP_IMM)));
1299    }
1300
1301    #[test]
1302    fn brk_encoding_roundtrip_imm() {
1303        let enc = brk_encoding(BRK_TRAP_IMM);
1304        let imm = (enc >> 5) & 0xFFFF;
1305        assert_eq!(imm, BRK_TRAP_IMM);
1306    }
1307}