Skip to main content

kh_runtime/mem/
map.rs

1//! Guest virtual memory backed by host `mmap` / `mprotect` / `munmap`.
2//!
3//! Mapping policy:
4//! - Lengths and host addresses are rounded to the **host** page size.
5//! - Preferred guest VAs are tried first (`MAP_FIXED_NOREPLACE` on Linux).
6//! - On failure, a **contiguous host span** covering the whole image is
7//!   reserved, a uniform slide is derived, and each segment is placed with
8//!   `MAP_FIXED` inside that span. (Placing only the first segment free and
9//!   then `MAP_FIXED_NOREPLACE` neighbours fails on Linux when ASLR put the
10//!   first mapping next to another VMA — common for preferred base `0` dylibs.)
11//! - `__PAGEZERO`-style regions (no access, no file, huge span) are skipped.
12//! - File-backed content: **host-page-aligned** full segments map the file
13//!   once with final prot (no anon+overlay, no bulk I-cache flush). Partial
14//!   edge pages still copy into an anon map. BSS beyond `filesize` stays
15//!   anonymous zeros. Interiors that are only page-aligned still use
16//!   `MAP_PRIVATE` file remap over anon (CoW on first write).
17//! - I-cache flush only after **userspace stores** into RX (edge fills /
18//!   residual `svc`→`brk` patches) — pure file maps stay demand-paged.
19
20use std::fs::File;
21use std::io::{Read, Seek, SeekFrom};
22use std::os::fd::AsRawFd;
23use std::ptr;
24
25use thiserror::Error;
26
27use crate::host;
28
29use super::layout::{HostPageSize, PageLayout};
30
31/// Darwin `VM_PROT_*` bits (same numeric values as XNU).
32pub const VM_PROT_READ: u32 = 0x1;
33pub const VM_PROT_WRITE: u32 = 0x2;
34pub const VM_PROT_EXECUTE: u32 = 0x4;
35
36/// One mapped guest region (host-owned).
37#[derive(Debug)]
38pub struct MappedRegion {
39    /// Segment / region name (for diagnostics).
40    pub name: String,
41    /// Guest virtual address (preferred + slide).
42    pub guest_addr: u64,
43    /// Host mapping base pointer.
44    ptr: *mut u8,
45    /// Host mapping length in bytes (host-page multiple).
46    len: usize,
47    /// Final protection bits (Darwin `VM_PROT_*`).
48    pub prot: u32,
49    /// File bytes copied into the mapping (0 for pure BSS / anonymous).
50    pub file_bytes: u64,
51    /// Virtual size requested by the image (may be smaller than host map).
52    pub vmsize: u64,
53    /// Relative `(offset, len)` within [`Self::file_bytes`] for residual `svc`
54    /// scan (instruction sections). Empty → scan whole `file_bytes`.
55    pub svc_scan_ranges: Vec<(u64, u64)>,
56}
57
58// SAFETY: `MappedRegion` uniquely owns the mapping; the pointer is never shared
59// across threads without external synchronization. `Send` is required so
60// `GuestMemory` can move between threads before execution starts.
61unsafe impl Send for MappedRegion {}
62
63impl MappedRegion {
64    /// Host base as a mutable byte slice covering the whole mapping.
65    #[must_use]
66    pub fn host_bytes_mut(&mut self) -> &mut [u8] {
67        // SAFETY: we own this mapping for `len` bytes; it is RW until final
68        // protect, and callers only use this before RX mprotect or on RW pages.
69        unsafe { std::slice::from_raw_parts_mut(self.ptr, self.len) }
70    }
71
72    /// Host base as an immutable byte slice.
73    #[must_use]
74    pub fn host_bytes(&self) -> &[u8] {
75        // SAFETY: mapping is live and owned; length is exact.
76        unsafe { std::slice::from_raw_parts(self.ptr, self.len) }
77    }
78
79    /// Host virtual address of the mapping base.
80    #[must_use]
81    pub fn host_addr(&self) -> u64 {
82        host::ptr_addr_u64(self.ptr)
83    }
84
85    /// Host mapping length.
86    #[must_use]
87    pub const fn host_len(&self) -> usize {
88        self.len
89    }
90}
91
92impl Drop for MappedRegion {
93    fn drop(&mut self) {
94        if !self.ptr.is_null() && self.len > 0 {
95            if !host::munmap(self.ptr, self.len) {
96                tracing::warn!(
97                    name = %self.name,
98                    len = self.len,
99                    "munmap failed during MappedRegion drop"
100                );
101            }
102            self.ptr = ptr::null_mut();
103        }
104    }
105}
106
107/// Request to map one segment-like region from a Mach-O image.
108#[derive(Debug, Clone)]
109pub struct MapRequest {
110    /// Segment name.
111    pub name: String,
112    /// Preferred guest VA (slide 0).
113    pub preferred_va: u64,
114    /// Virtual size (`vmsize`).
115    pub vmsize: u64,
116    /// File offset of the first mapped byte (`fileoff`).
117    pub fileoff: u64,
118    /// Bytes to load from the file (`filesize`).
119    pub filesize: u64,
120    /// Initial protection (`initprot`, Darwin bits).
121    pub initprot: u32,
122    /// Maximum protection (`maxprot`, Darwin bits).
123    pub maxprot: u32,
124    /// Relative `(offset, len)` ranges within mapped file content to scan for
125    /// residual Darwin `svc` (instruction sections only).
126    ///
127    /// Empty → trap backend falls back to the whole `filesize` span (fixtures
128    /// without section flags, or unknown layout).
129    pub svc_scan_ranges: Vec<(u64, u64)>,
130}
131
132impl MapRequest {
133    /// True when this region should not be mapped (null-catch / empty).
134    #[must_use]
135    pub fn should_skip(&self) -> bool {
136        if self.vmsize == 0 {
137            return true;
138        }
139        // PAGEZERO is typically 4 GiB — never materialize it.
140        self.name == "__PAGEZERO"
141            || (self.initprot == 0 && self.fileoff == 0 && self.filesize == 0)
142    }
143}
144
145/// Fully mapped guest image address space (slide applied).
146#[derive(Debug)]
147pub struct GuestMemory {
148    regions: Vec<MappedRegion>,
149    /// Byte slide applied to preferred guest VAs (`actual = preferred + slide`).
150    slide: u64,
151    host: HostPageSize,
152    preferred_base: u64,
153}
154
155/// Errors while mapping guest memory.
156#[derive(Debug, Error)]
157pub enum MapError {
158    /// Invalid argument (zero length, overflow, …).
159    #[error("invalid map request: {0}")]
160    Invalid(&'static str),
161
162    /// `mmap` / `mprotect` failed.
163    #[error("mmap/mprotect failed for {name}: {source}")]
164    Sys {
165        /// Region name.
166        name: String,
167        /// Underlying OS error.
168        #[source]
169        source: std::io::Error,
170    },
171
172    /// File I/O while filling a mapping.
173    #[error("file I/O while mapping {name}: {source}")]
174    File {
175        /// Region name.
176        name: String,
177        /// Underlying I/O error.
178        #[source]
179        source: std::io::Error,
180    },
181
182    /// Could not place the image (fixed and slid attempts failed).
183    #[error("could not place guest image in host VA space")]
184    PlacementFailed,
185}
186
187/// Upper bound on a single slid reservation (guards sparse / pathological layouts).
188const MAX_SLIDE_SPAN: u64 = 512 * 1024 * 1024; // 512 MiB
189
190impl GuestMemory {
191    /// Maps all non-skipped requests into the host address space.
192    ///
193    /// Tries preferred addresses first; on failure reserves a contiguous host
194    /// span for the whole image and places each segment with a uniform slide
195    /// (guest VA == host VA after placement).
196    pub fn map_image(
197        host: HostPageSize,
198        preferred_base: u64,
199        requests: &[MapRequest],
200        file: &mut File,
201    ) -> Result<Self, MapError> {
202        let mut src = SegmentSource::File(file);
203        Self::map_image_with(host, preferred_base, requests, &mut src)
204    }
205
206    /// Like [`Self::map_image`], but fills segments from an in-memory container
207    /// (same bytes already read for Mach-O parse). Avoids a second disk pass
208    /// for large tools (any guest, not tool-specific).
209    pub fn map_image_bytes(
210        host: HostPageSize,
211        preferred_base: u64,
212        requests: &[MapRequest],
213        bytes: &[u8],
214    ) -> Result<Self, MapError> {
215        let mut src = SegmentSource::Bytes(bytes);
216        Self::map_image_with(host, preferred_base, requests, &mut src)
217    }
218
219    fn map_image_with(
220        host: HostPageSize,
221        preferred_base: u64,
222        requests: &[MapRequest],
223        src: &mut SegmentSource<'_>,
224    ) -> Result<Self, MapError> {
225        let active: Vec<&MapRequest> = requests.iter().filter(|r| !r.should_skip()).collect();
226        if active.is_empty() {
227            return Ok(Self {
228                regions: Vec::new(),
229                slide: 0,
230                host,
231                preferred_base,
232            });
233        }
234
235        // Attempt 1: preferred VAs (slide = 0, never clobber host maps).
236        match Self::try_map_all(host, preferred_base, 0, &active, src, FixedMode::NoReplace) {
237            Ok(regions) => {
238                return Ok(Self {
239                    regions,
240                    slide: 0,
241                    host,
242                    preferred_base,
243                });
244            }
245            Err(err) => {
246                tracing::debug!(error = %err, "preferred-base map failed; trying slid placement");
247            }
248        }
249
250        // Attempt 2: reserve full image span, slide every segment into it.
251        match Self::try_map_slid_reserved(host, preferred_base, &active, src) {
252            Ok(mem) => return Ok(mem),
253            Err(err) => {
254                tracing::debug!(error = %err, "reserved-span slid map failed; trying first-region slide");
255            }
256        }
257
258        // Attempt 3 (legacy): free-place first region, fixed-relative rest.
259        // Works when the image is a single segment or the kernel left a hole.
260        Self::try_map_slid_first_region(host, preferred_base, &active, src)
261    }
262
263    /// Kernel-chosen contiguous span for the whole image, then `MAP_FIXED` each segment.
264    fn try_map_slid_reserved(
265        host: HostPageSize,
266        preferred_base: u64,
267        active: &[&MapRequest],
268        src: &mut SegmentSource<'_>,
269    ) -> Result<Self, MapError> {
270        let (span_lo, span_hi) = image_preferred_span(host, active)?;
271        let span_len_u64 = span_hi.saturating_sub(span_lo);
272        if span_len_u64 == 0 {
273            return Err(MapError::Invalid("empty image span"));
274        }
275        if span_len_u64 > MAX_SLIDE_SPAN {
276            return Err(MapError::Invalid(
277                "image span too large for slid reservation",
278            ));
279        }
280        let span_len =
281            usize::try_from(span_len_u64).map_err(|_| MapError::Invalid("span too large"))?;
282
283        // Reserve with PROT_NONE so the VA range cannot be stolen before we carve it.
284        let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
285        let Some(reserve_ptr) = host::mmap(None, span_len, libc::PROT_NONE, flags, -1, 0) else {
286            return Err(MapError::Sys {
287                name: "__RESERVE".into(),
288                source: std::io::Error::last_os_error(),
289            });
290        };
291        let reserve_base = host::ptr_addr_u64(reserve_ptr);
292        let slide = reserve_base.wrapping_sub(span_lo);
293
294        tracing::debug!(
295            span_lo = format_args!("{span_lo:#x}"),
296            span_hi = format_args!("{span_hi:#x}"),
297            reserve = format_args!("{reserve_base:#x}"),
298            slide = format_args!("{slide:#x}"),
299            "reserved contiguous host span for slid image"
300        );
301
302        // Carve segments with MAP_FIXED (replaces PROT_NONE pages in place).
303        match Self::try_map_all(
304            host,
305            preferred_base,
306            slide,
307            active,
308            src,
309            FixedMode::Replace,
310        ) {
311            Ok(regions) => {
312                // Carve-outs leave PROT_NONE residue in gaps; release it so the
313                // address space does not retain untracked maps.
314                unmap_span_gaps(reserve_base, span_len_u64, &regions);
315                Ok(Self {
316                    regions,
317                    slide,
318                    host,
319                    preferred_base,
320                })
321            }
322            Err(err) => {
323                // Successful segment maps were already dropped by `try_map_all`.
324                // Drop any leftover PROT_NONE from the original reservation.
325                // (May no-op for ranges already replaced+dropped.)
326                let _ = host::munmap(reserve_ptr, span_len);
327                Err(err)
328            }
329        }
330    }
331
332    /// Free-place the first region; map the rest at preferred + slide (fixed, no-replace).
333    fn try_map_slid_first_region(
334        host: HostPageSize,
335        preferred_base: u64,
336        active: &[&MapRequest],
337        src: &mut SegmentSource<'_>,
338    ) -> Result<Self, MapError> {
339        let first = active
340            .first()
341            .copied()
342            .ok_or(MapError::Invalid("no active regions"))?;
343        let first_map = map_one(host, first, first.preferred_va, src, FixedMode::Free)?;
344        let slide = first_map.guest_addr.wrapping_sub(first.preferred_va);
345
346        let mut regions = vec![first_map];
347        for req in active.iter().skip(1) {
348            let target = req.preferred_va.wrapping_add(slide);
349            match map_one(host, req, target, src, FixedMode::NoReplace) {
350                Ok(region) => regions.push(region),
351                Err(err) => {
352                    drop(regions);
353                    tracing::debug!(error = %err, "slid fixed map failed");
354                    return Err(MapError::PlacementFailed);
355                }
356            }
357        }
358
359        Ok(Self {
360            regions,
361            slide,
362            host,
363            preferred_base,
364        })
365    }
366
367    fn try_map_all(
368        host: HostPageSize,
369        preferred_base: u64,
370        slide: u64,
371        active: &[&MapRequest],
372        src: &mut SegmentSource<'_>,
373        fixed: FixedMode,
374    ) -> Result<Vec<MappedRegion>, MapError> {
375        let mut regions = Vec::with_capacity(active.len());
376        for req in active {
377            let target = req.preferred_va.wrapping_add(slide);
378            match map_one(host, req, target, src, fixed) {
379                Ok(region) => regions.push(region),
380                Err(err) => {
381                    drop(regions);
382                    return Err(err);
383                }
384            }
385        }
386        let _ = preferred_base;
387        Ok(regions)
388    }
389
390    /// Applied slide in bytes.
391    #[must_use]
392    pub const fn slide(&self) -> u64 {
393        self.slide
394    }
395
396    /// Preferred base used when planning.
397    #[must_use]
398    pub const fn preferred_base(&self) -> u64 {
399        self.preferred_base
400    }
401
402    /// Host page size policy.
403    #[must_use]
404    pub const fn host(&self) -> HostPageSize {
405        self.host
406    }
407
408    /// Mapped regions in plan order (skips omitted).
409    #[must_use]
410    pub fn regions(&self) -> &[MappedRegion] {
411        &self.regions
412    }
413
414    /// Mutable access to regions (e.g. SVC rewrite before protect).
415    pub fn regions_mut(&mut self) -> &mut [MappedRegion] {
416        &mut self.regions
417    }
418
419    /// Translates a guest VA to a host pointer if it falls in a mapped region.
420    #[must_use]
421    pub fn guest_to_host(&self, guest_va: u64) -> Option<*const u8> {
422        for region in &self.regions {
423            let start = region.guest_addr;
424            let end = start.saturating_add(region.vmsize);
425            if guest_va >= start && guest_va < end {
426                let offset = usize::try_from(guest_va.saturating_sub(start)).ok()?;
427                if offset >= region.len {
428                    return None;
429                }
430                // SAFETY: offset < len, pointer derived from owned mapping base.
431                return Some(unsafe { region.ptr.add(offset) });
432            }
433        }
434        None
435    }
436
437    /// Translates guest VA to a mutable host pointer.
438    #[must_use]
439    pub fn guest_to_host_mut(&mut self, guest_va: u64) -> Option<*mut u8> {
440        for region in &mut self.regions {
441            let start = region.guest_addr;
442            let end = start.saturating_add(region.vmsize);
443            if guest_va >= start && guest_va < end {
444                let offset = usize::try_from(guest_va.saturating_sub(start)).ok()?;
445                if offset >= region.len {
446                    return None;
447                }
448                return Some(unsafe { region.ptr.add(offset) });
449            }
450        }
451        None
452    }
453
454    /// Reads a little-endian `u32` from a mapped guest VA.
455    #[must_use]
456    pub fn read_u32_le(&self, guest_va: u64) -> Option<u32> {
457        let mut buf = [0_u8; 4];
458        self.read_exact(guest_va, &mut buf)?;
459        Some(u32::from_le_bytes(buf))
460    }
461
462    /// Reads a little-endian `u64` from a mapped guest VA.
463    #[must_use]
464    pub fn read_u64_le(&self, guest_va: u64) -> Option<u64> {
465        let mut buf = [0_u8; 8];
466        self.read_exact(guest_va, &mut buf)?;
467        Some(u64::from_le_bytes(buf))
468    }
469
470    /// Writes a little-endian `u64` to a mapped guest VA.
471    ///
472    /// Caller must ensure the region is writable (`mprotect_rw` if needed).
473    pub fn write_u64_le(&mut self, guest_va: u64, value: u64) -> Option<()> {
474        self.write_exact(guest_va, &value.to_le_bytes())
475    }
476
477    /// Copies `buf.len()` bytes from a mapped guest VA into `buf`.
478    pub fn read_exact(&self, guest_va: u64, buf: &mut [u8]) -> Option<()> {
479        if buf.is_empty() {
480            return Some(());
481        }
482        for region in &self.regions {
483            let start = region.guest_addr;
484            let end = start.saturating_add(region.vmsize);
485            if guest_va >= start && guest_va < end {
486                let offset = usize::try_from(guest_va.saturating_sub(start)).ok()?;
487                let need = buf.len();
488                let avail = region.len.saturating_sub(offset);
489                if need > avail {
490                    return None;
491                }
492                let host = region.host_bytes();
493                let src = host.get(offset..offset.saturating_add(need))?;
494                buf.copy_from_slice(src);
495                return Some(());
496            }
497        }
498        None
499    }
500
501    /// Copies `buf` into a mapped guest VA (region must be host-writable).
502    pub fn write_exact(&mut self, guest_va: u64, buf: &[u8]) -> Option<()> {
503        if buf.is_empty() {
504            return Some(());
505        }
506        for region in &mut self.regions {
507            let start = region.guest_addr;
508            let end = start.saturating_add(region.vmsize);
509            if guest_va >= start && guest_va < end {
510                let offset = usize::try_from(guest_va.saturating_sub(start)).ok()?;
511                let need = buf.len();
512                let avail = region.len.saturating_sub(offset);
513                if need > avail {
514                    return None;
515                }
516                let host = region.host_bytes_mut();
517                let dst = host.get_mut(offset..offset.saturating_add(need))?;
518                dst.copy_from_slice(buf);
519                return Some(());
520            }
521        }
522        None
523    }
524
525    /// Testing helper: pretends the image was slid by `delta` without remapping.
526    ///
527    /// Updates [`Self::slide`] and each region's `guest_addr`. Host pointers are
528    /// unchanged (identity is intentionally broken). Used by loader unit tests when
529    /// true preferred-base collisions are unavailable (macOS `MAP_FIXED` clobbers
530    /// blockers). Not for production load paths.
531    #[doc(hidden)]
532    pub fn test_offset_guest_vas(&mut self, delta: u64) {
533        self.slide = self.slide.wrapping_add(delta);
534        for region in &mut self.regions {
535            region.guest_addr = region.guest_addr.wrapping_add(delta);
536        }
537    }
538}
539
540/// How to place a single region relative to a preferred / slid guest VA.
541#[derive(Debug, Clone, Copy, PartialEq, Eq)]
542enum FixedMode {
543    /// Kernel chooses address (`guest_addr` becomes host base).
544    Free,
545    /// Fixed address; do not clobber existing maps (`MAP_FIXED_NOREPLACE` on Linux).
546    NoReplace,
547    /// Fixed address; replace any mapping already at that VA (`MAP_FIXED`).
548    /// Used after a PROT_NONE span reservation so carve-outs succeed.
549    Replace,
550}
551
552/// Where segment file content comes from (disk or already-read container).
553enum SegmentSource<'a> {
554    File(&'a mut File),
555    Bytes(&'a [u8]),
556}
557
558impl SegmentSource<'_> {
559    fn copy_into(
560        &mut self,
561        fileoff: u64,
562        dst: &mut [u8],
563        region_name: &str,
564    ) -> Result<(), MapError> {
565        match self {
566            SegmentSource::File(file) => {
567                file.seek(SeekFrom::Start(fileoff))
568                    .map_err(|source| MapError::File {
569                        name: region_name.to_owned(),
570                        source,
571                    })?;
572                file.read_exact(dst).map_err(|source| MapError::File {
573                    name: region_name.to_owned(),
574                    source,
575                })
576            }
577            SegmentSource::Bytes(bytes) => {
578                let start = usize::try_from(fileoff).map_err(|_| MapError::Invalid("fileoff"))?;
579                let end = start
580                    .checked_add(dst.len())
581                    .ok_or(MapError::Invalid("file range overflow"))?;
582                let Some(src) = bytes.get(start..end) else {
583                    return Err(MapError::Invalid("fileoff past container end"));
584                };
585                dst.copy_from_slice(src);
586                Ok(())
587            }
588        }
589    }
590}
591
592fn map_one(
593    host: HostPageSize,
594    req: &MapRequest,
595    target_guest: u64,
596    src: &mut SegmentSource<'_>,
597    mode: FixedMode,
598) -> Result<MappedRegion, MapError> {
599    let host_page = host.bytes();
600    let map_len_u64 = PageLayout::align_up(req.vmsize, host_page)
601        .ok_or(MapError::Invalid("vmsize align overflow"))?;
602    let map_len =
603        usize::try_from(map_len_u64).map_err(|_| MapError::Invalid("vmsize too large"))?;
604    if map_len == 0 {
605        return Err(MapError::Invalid("zero map length"));
606    }
607
608    let fixed = matches!(mode, FixedMode::NoReplace | FixedMode::Replace);
609    let fixed_addr = if fixed { Some(target_guest) } else { None };
610
611    // Fast path: whole host map is one host-page-aligned file image.
612    // CLT tools (clang __TEXT ~100+ MiB) hit this — avoid anon+overlay and
613    // bulk I-cache flush so the guest stays demand-paged until first use.
614    if let SegmentSource::File(file) = src
615        && let Some(region) =
616            try_map_pure_file(host, req, target_guest, file, mode, map_len, fixed)
617    {
618        return Ok(region);
619    }
620
621    let mut flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
622    match mode {
623        FixedMode::Free => {}
624        FixedMode::NoReplace => flags |= host::fixed_map_flag(),
625        FixedMode::Replace => flags |= libc::MAP_FIXED,
626    }
627
628    // Map RW first so we can fill file content; tighten with mprotect after.
629    let prot_rw = libc::PROT_READ | libc::PROT_WRITE;
630
631    let Some(base) = host::mmap(fixed_addr, map_len, prot_rw, flags, -1, 0) else {
632        return Err(MapError::Sys {
633            name: req.name.clone(),
634            source: std::io::Error::last_os_error(),
635        });
636    };
637
638    let actual_host = host::ptr_addr_u64(base);
639    // For non-fixed maps, guest VA becomes the host address (identity slide).
640    let guest_addr = if fixed { target_guest } else { actual_host };
641
642    let mut region = MappedRegion {
643        name: req.name.clone(),
644        guest_addr,
645        ptr: base,
646        len: map_len,
647        prot: req.initprot,
648        file_bytes: 0,
649        vmsize: req.vmsize,
650        svc_scan_ranges: req.svc_scan_ranges.clone(),
651    };
652
653    // True when userspace stores wrote instruction (or any) bytes into the map.
654    let mut dirtied = false;
655
656    // Fill file-backed bytes: prefer page-aligned file MAP_PRIVATE (any guest).
657    if req.filesize > 0 {
658        let copy_len = usize::try_from(req.filesize.min(req.vmsize))
659            .map_err(|_| MapError::Invalid("filesize too large"))?;
660        if copy_len > map_len {
661            return Err(MapError::Invalid("filesize exceeds map length"));
662        }
663        let fill = match src {
664            SegmentSource::File(file) => fill_segment_file(
665                file,
666                base,
667                map_len,
668                req.fileoff,
669                copy_len,
670                host_page,
671                &req.name,
672            )?,
673            SegmentSource::Bytes(bytes) => {
674                fill_segment_bytes(bytes, base, map_len, req.fileoff, copy_len, host_page)?
675            }
676        };
677        if fill.ok {
678            dirtied = fill.dirtied;
679        } else {
680            let dst = region.host_bytes_mut();
681            let Some(slice) = dst.get_mut(..copy_len) else {
682                return Err(MapError::Invalid("copy slice out of range"));
683            };
684            src.copy_into(req.fileoff, slice, &req.name)?;
685            dirtied = true;
686        }
687        region.file_bytes = u64::try_from(copy_len).unwrap_or(0);
688    }
689
690    // Apply final protection. Empty initprot → PROT_NONE.
691    let host_prot = darwin_to_host_prot(req.initprot);
692    if !host::mprotect(base, map_len, host_prot) {
693        return Err(MapError::Sys {
694            name: req.name.clone(),
695            source: std::io::Error::last_os_error(),
696        });
697    }
698
699    // aarch64: only flush after userspace wrote executable bytes. Pure file
700    // MAP_PRIVATE overlays leave D/I coherent for demand faults; flushing a
701    // 100+ MiB TEXT forced every page into RSS at load (any large guest).
702    if dirtied && req.initprot & VM_PROT_EXECUTE != 0 {
703        let flush_len = if region.file_bytes == 0 {
704            let page = usize::try_from(host_page).unwrap_or(4096);
705            map_len.min(page).max(4)
706        } else {
707            usize::try_from(region.file_bytes)
708                .unwrap_or(0)
709                .min(map_len)
710                .max(4)
711        };
712        clear_icache(base, flush_len);
713    }
714
715    // When fixed, require exact placement.
716    if fixed && actual_host != target_guest {
717        return Err(MapError::Sys {
718            name: req.name.clone(),
719            source: std::io::Error::new(
720                std::io::ErrorKind::AddrNotAvailable,
721                format!("kernel placed map at {actual_host:#x}, wanted {target_guest:#x}"),
722            ),
723        });
724    }
725
726    Ok(region)
727}
728
729/// Map a whole segment as one `MAP_PRIVATE` file image with final prot.
730///
731/// Requires host-page-aligned `fileoff`, file bytes covering the full host map
732/// length (no BSS tail), and fixed guest VA page-aligned when fixed.
733fn try_map_pure_file(
734    host: HostPageSize,
735    req: &MapRequest,
736    target_guest: u64,
737    file: &File,
738    mode: FixedMode,
739    map_len: usize,
740    fixed: bool,
741) -> Option<MappedRegion> {
742    let host_page = u64::from(host.bytes());
743    if host_page == 0 {
744        return None;
745    }
746    let copy_u64 = req.filesize.min(req.vmsize);
747    if copy_u64 == 0 {
748        return None;
749    }
750    let Ok(copy_len) = usize::try_from(copy_u64) else {
751        return None;
752    };
753    // Full host mapping must be file-backed (no zero tail beyond filesize).
754    if copy_len != map_len {
755        return None;
756    }
757    if !req.fileoff.is_multiple_of(host_page) {
758        return None;
759    }
760    if fixed && !target_guest.is_multiple_of(host_page) {
761        return None;
762    }
763
764    let mut flags = libc::MAP_PRIVATE;
765    match mode {
766        FixedMode::Free => {}
767        FixedMode::NoReplace => flags |= host::fixed_map_flag(),
768        FixedMode::Replace => flags |= libc::MAP_FIXED,
769    }
770    let host_prot = darwin_to_host_prot(req.initprot);
771    let fixed_addr = if fixed { Some(target_guest) } else { None };
772    let fd = file.as_raw_fd();
773    let Ok(file_off) = i64::try_from(req.fileoff) else {
774        return None;
775    };
776    let Some(base) = host::mmap(fixed_addr, map_len, host_prot, flags, fd, file_off) else {
777        // Preferred-base collision etc. — caller falls back to anon+fill.
778        return None;
779    };
780    let actual_host = host::ptr_addr_u64(base);
781    if fixed && actual_host != target_guest {
782        let _ = host::munmap(base, map_len);
783        return None;
784    }
785    let guest_addr = if fixed { target_guest } else { actual_host };
786    Some(MappedRegion {
787        name: req.name.clone(),
788        guest_addr,
789        ptr: base,
790        len: map_len,
791        prot: req.initprot,
792        file_bytes: copy_u64,
793        vmsize: req.vmsize,
794        svc_scan_ranges: req.svc_scan_ranges.clone(),
795    })
796}
797
798/// Read `dst.len()` bytes from `file` at absolute `off`.
799fn read_file_range(file: &File, off: u64, dst: &mut [u8], name: &str) -> Result<(), MapError> {
800    let mut clone = file.try_clone().map_err(|source| MapError::File {
801        name: name.to_owned(),
802        source,
803    })?;
804    clone
805        .seek(SeekFrom::Start(off))
806        .map_err(|source| MapError::File {
807            name: name.to_owned(),
808            source,
809        })?;
810    clone.read_exact(dst).map_err(|source| MapError::File {
811        name: name.to_owned(),
812        source,
813    })
814}
815
816/// Result of filling a segment mapping from file or bytes.
817struct FillOutcome {
818    /// Fill completed (false → caller should fall back to full `copy_into`).
819    ok: bool,
820    /// Userspace stores touched the mapping (edge copies / memcpy).
821    dirtied: bool,
822}
823
824/// Fill `[0, copy_len)` of an anonymous mapping from a host file.
825///
826/// Host-page-aligned interior is remapped `MAP_PRIVATE` from the file (no
827/// full-TEXT `memcpy`). Partial edge pages are still copied.
828fn fill_segment_file(
829    file: &File,
830    base: *mut u8,
831    map_len: usize,
832    fileoff: u64,
833    copy_len: usize,
834    host_page: u32,
835    name: &str,
836) -> Result<FillOutcome, MapError> {
837    let page = u64::from(host_page);
838    if page == 0 || copy_len == 0 || copy_len > map_len {
839        return Ok(FillOutcome {
840            ok: false,
841            dirtied: false,
842        });
843    }
844
845    let content_lo = fileoff;
846    let content_hi = fileoff.saturating_add(u64::try_from(copy_len).unwrap_or(0));
847    // Full host pages strictly inside [content_lo, content_hi).
848    let aligned_lo = content_lo.div_ceil(page).saturating_mul(page);
849    let aligned_hi = content_hi
850        .checked_div(page)
851        .unwrap_or(0)
852        .saturating_mul(page);
853
854    // Mapping-relative offsets.
855    let lead = usize::try_from(aligned_lo.saturating_sub(content_lo)).unwrap_or(0);
856    let trail_off = usize::try_from(aligned_hi.saturating_sub(content_lo)).unwrap_or(0);
857    let mut dirtied = false;
858
859    // Interior: file-backed MAP_PRIVATE (kernel page cache / CoW).
860    if aligned_hi > aligned_lo {
861        let mid_len = usize::try_from(aligned_hi.saturating_sub(aligned_lo)).unwrap_or(0);
862        if mid_len == 0 || lead.saturating_add(mid_len) > map_len {
863            return Ok(FillOutcome {
864                ok: false,
865                dirtied: false,
866            });
867        }
868        // SAFETY: dest is within the anon mapping we just created.
869        let dest = unsafe { base.add(lead) };
870        let fd = file.as_raw_fd();
871        let flags = libc::MAP_PRIVATE | libc::MAP_FIXED;
872        let Some(mapped) = host::mmap(
873            Some(host::ptr_addr_u64(dest)),
874            mid_len,
875            libc::PROT_READ | libc::PROT_WRITE,
876            flags,
877            fd,
878            i64::try_from(aligned_lo).unwrap_or(0),
879        ) else {
880            return Err(MapError::Sys {
881                name: name.to_owned(),
882                source: std::io::Error::last_os_error(),
883            });
884        };
885        if mapped != dest {
886            let _ = host::munmap(mapped, mid_len);
887            return Err(MapError::Sys {
888                name: name.to_owned(),
889                source: std::io::Error::new(
890                    std::io::ErrorKind::AddrNotAvailable,
891                    "file-backed MAP_FIXED moved",
892                ),
893            });
894        }
895    }
896
897    // No full pages: whole range is partial — one copy.
898    if aligned_hi <= aligned_lo {
899        // SAFETY: base..base+copy_len inside map_len.
900        let dst = unsafe { std::slice::from_raw_parts_mut(base, copy_len) };
901        read_file_range(file, fileoff, dst, name)?;
902        return Ok(FillOutcome {
903            ok: true,
904            dirtied: true,
905        });
906    }
907
908    // Leading partial page.
909    if lead > 0 {
910        let n = lead.min(copy_len);
911        // SAFETY: lead < map_len when interior was validated (or no interior).
912        let dst = unsafe { std::slice::from_raw_parts_mut(base, n) };
913        read_file_range(file, fileoff, dst, name)?;
914        dirtied = true;
915    }
916    // Trailing partial page.
917    if trail_off < copy_len {
918        let n = copy_len.saturating_sub(trail_off);
919        if n > 0 && trail_off < map_len {
920            let len = n.min(map_len.saturating_sub(trail_off));
921            // SAFETY: trail_off+len <= map_len.
922            let dst = unsafe { std::slice::from_raw_parts_mut(base.add(trail_off), len) };
923            let seek_at = fileoff.saturating_add(u64::try_from(trail_off).unwrap_or(0));
924            read_file_range(file, seek_at, dst, name)?;
925            dirtied = true;
926        }
927    }
928
929    Ok(FillOutcome {
930        ok: true,
931        dirtied,
932    })
933}
934
935/// Fill from an in-memory container (mmap of the same file or heap).
936///
937/// Uses the same partial-page policy: full host pages are still `memcpy` from
938/// the container (already demand-paged); edges match. Avoids a second disk
939/// path when the loader holds `FileImage`.
940fn fill_segment_bytes(
941    bytes: &[u8],
942    base: *mut u8,
943    map_len: usize,
944    fileoff: u64,
945    copy_len: usize,
946    _host_page: u32,
947) -> Result<FillOutcome, MapError> {
948    let start = usize::try_from(fileoff).map_err(|_| MapError::Invalid("fileoff"))?;
949    let end = start
950        .checked_add(copy_len)
951        .ok_or(MapError::Invalid("file range overflow"))?;
952    let Some(src) = bytes.get(start..end) else {
953        return Err(MapError::Invalid("fileoff past container end"));
954    };
955    if copy_len > map_len {
956        return Err(MapError::Invalid("filesize exceeds map length"));
957    }
958    let dst = unsafe { std::slice::from_raw_parts_mut(base, copy_len) };
959    dst.copy_from_slice(src);
960    Ok(FillOutcome {
961        ok: true,
962        dirtied: true,
963    })
964}
965
966// libgcc / compiler-rt — same symbol as hypercall tramp setup.
967#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
968unsafe extern "C" {
969    fn __clear_cache(start: *mut libc::c_void, end: *mut libc::c_void);
970}
971
972/// Flush D/I caches after writing executable guest code (aarch64 requirement).
973#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
974fn clear_icache(ptr: *mut u8, len: usize) {
975    if ptr.is_null() || len == 0 {
976        return;
977    }
978    // SAFETY: range is the live mapping we just wrote and mprotected.
979    unsafe {
980        let start = ptr.cast::<libc::c_void>();
981        let end = ptr.wrapping_add(len).cast::<libc::c_void>();
982        __clear_cache(start, end);
983    }
984}
985
986#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
987fn clear_icache(_ptr: *mut u8, _len: usize) {}
988
989/// Preferred VA span covering every active segment (host-page aligned ends).
990fn image_preferred_span(
991    host: HostPageSize,
992    active: &[&MapRequest],
993) -> Result<(u64, u64), MapError> {
994    let host_page = host.bytes();
995    let mut lo = u64::MAX;
996    let mut hi = 0_u64;
997    for req in active {
998        lo = lo.min(req.preferred_va);
999        let end = req
1000            .preferred_va
1001            .checked_add(
1002                PageLayout::align_up(req.vmsize, host_page)
1003                    .ok_or(MapError::Invalid("vmsize align overflow"))?,
1004            )
1005            .ok_or(MapError::Invalid("segment end overflow"))?;
1006        hi = hi.max(end);
1007    }
1008    if lo == u64::MAX || hi <= lo {
1009        return Err(MapError::Invalid("no span from active segments"));
1010    }
1011    // Align span start down so MAP_FIXED bases stay host-page aligned when preferred is.
1012    let lo = PageLayout::align_down(lo, host_page).unwrap_or(lo);
1013    Ok((lo, hi))
1014}
1015
1016/// Best-effort: release PROT_NONE pages left in the reserved span outside mapped segments.
1017fn unmap_span_gaps(reserve_base: u64, span_len: u64, regions: &[MappedRegion]) {
1018    if span_len == 0 || regions.is_empty() {
1019        return;
1020    }
1021    let span_end = reserve_base.saturating_add(span_len);
1022    let mut covered: Vec<(u64, u64)> = regions
1023        .iter()
1024        .map(|r| {
1025            let start = r.guest_addr;
1026            let end = start.saturating_add(u64::try_from(r.len).unwrap_or(0));
1027            (start, end)
1028        })
1029        .collect();
1030    covered.sort_by_key(|(s, _)| *s);
1031
1032    let mut cursor = reserve_base;
1033    for (start, end) in covered {
1034        if start > cursor && start <= span_end {
1035            let gap_len = start.saturating_sub(cursor);
1036            if let Ok(len) = usize::try_from(gap_len)
1037                && len > 0
1038            {
1039                let ptr = host::u64_as_void_ptr(cursor).cast::<u8>();
1040                let _ = host::munmap(ptr, len);
1041            }
1042        }
1043        cursor = cursor.max(end);
1044    }
1045    if cursor < span_end {
1046        let gap_len = span_end.saturating_sub(cursor);
1047        if let Ok(len) = usize::try_from(gap_len)
1048            && len > 0
1049        {
1050            let ptr = host::u64_as_void_ptr(cursor).cast::<u8>();
1051            let _ = host::munmap(ptr, len);
1052        }
1053    }
1054}
1055
1056/// Converts Darwin `VM_PROT_*` to host `PROT_*`.
1057#[must_use]
1058pub const fn darwin_to_host_prot(darwin: u32) -> libc::c_int {
1059    if darwin == 0 {
1060        return libc::PROT_NONE;
1061    }
1062    let mut prot = 0;
1063    if darwin & VM_PROT_READ != 0 {
1064        prot |= libc::PROT_READ;
1065    }
1066    if darwin & VM_PROT_WRITE != 0 {
1067        prot |= libc::PROT_WRITE;
1068    }
1069    if darwin & VM_PROT_EXECUTE != 0 {
1070        prot |= libc::PROT_EXEC;
1071    }
1072    prot
1073}
1074
1075/// Maps an anonymous RW stack region (guest VA == host VA, kernel-chosen).
1076///
1077/// `size` is rounded up to the host page size. The returned region's
1078/// `guest_addr` is the low address of the mapping; the stack pointer should be
1079/// near `guest_addr + vmsize`.
1080pub fn map_stack(host: HostPageSize, size: u64) -> Result<MappedRegion, MapError> {
1081    let host_page = host.bytes();
1082    let map_len_u64 =
1083        PageLayout::align_up(size.max(1), host_page).ok_or(MapError::Invalid("stack align"))?;
1084    let map_len = usize::try_from(map_len_u64).map_err(|_| MapError::Invalid("stack too large"))?;
1085
1086    let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
1087    let prot = libc::PROT_READ | libc::PROT_WRITE;
1088    let Some(base) = host::mmap(None, map_len, prot, flags, -1, 0) else {
1089        return Err(MapError::Sys {
1090            name: "__STACK".into(),
1091            source: std::io::Error::last_os_error(),
1092        });
1093    };
1094    Ok(MappedRegion {
1095        name: "__STACK".into(),
1096        guest_addr: host::ptr_addr_u64(base),
1097        ptr: base,
1098        len: map_len,
1099        prot: VM_PROT_READ | VM_PROT_WRITE,
1100        file_bytes: 0,
1101        vmsize: map_len_u64,
1102        svc_scan_ranges: Vec::new(),
1103    })
1104}
1105
1106/// Temporarily makes a region read/write (for SVC patching / bind writes).
1107///
1108/// No-op when Darwin `prot` already includes write (host mapping is already RW).
1109pub fn mprotect_rw(region: &MappedRegion) -> Result<(), MapError> {
1110    if region.prot & VM_PROT_WRITE != 0 {
1111        return Ok(());
1112    }
1113    if host::mprotect(region.ptr, region.len, libc::PROT_READ | libc::PROT_WRITE) {
1114        Ok(())
1115    } else {
1116        Err(MapError::Sys {
1117            name: region.name.clone(),
1118            source: std::io::Error::last_os_error(),
1119        })
1120    }
1121}
1122
1123/// Restores Darwin-derived host protection on a region.
1124pub fn mprotect_darwin(region: &MappedRegion, darwin_prot: u32) -> Result<(), MapError> {
1125    let host_prot = darwin_to_host_prot(darwin_prot);
1126    if host::mprotect(region.ptr, region.len, host_prot) {
1127        Ok(())
1128    } else {
1129        Err(MapError::Sys {
1130            name: region.name.clone(),
1131            source: std::io::Error::last_os_error(),
1132        })
1133    }
1134}
1135
1136#[cfg(test)]
1137#[allow(clippy::expect_used, clippy::unwrap_used)]
1138mod tests {
1139    use super::*;
1140    use std::io::{Seek, SeekFrom, Write};
1141    use std::path::PathBuf;
1142
1143    fn temp_file_with(bytes: &[u8]) -> (File, PathBuf) {
1144        use std::fs::OpenOptions;
1145        let path = std::env::temp_dir().join(format!(
1146            "kh-map-test-{}-{}",
1147            std::process::id(),
1148            bytes.len()
1149        ));
1150        let mut f = OpenOptions::new()
1151            .read(true)
1152            .write(true)
1153            .create(true)
1154            .truncate(true)
1155            .open(&path)
1156            .expect("create temp");
1157        f.write_all(bytes).expect("write");
1158        f.seek(SeekFrom::Start(0)).expect("seek");
1159        (f, path)
1160    }
1161
1162    #[test]
1163    fn skip_pagezero() {
1164        let req = MapRequest {
1165            name: "__PAGEZERO".into(),
1166            preferred_va: 0,
1167            vmsize: 0x1_0000_0000,
1168            fileoff: 0,
1169            filesize: 0,
1170            initprot: 0,
1171            maxprot: 0,
1172            svc_scan_ranges: Vec::new(),
1173        };
1174        assert!(req.should_skip());
1175    }
1176
1177    #[test]
1178    fn map_anonymous_rw_region() {
1179        let host = HostPageSize::detect().expect("host page");
1180        let page = u64::from(host.bytes());
1181        let payload = b"hello-guest-map";
1182        let (mut file, path) = temp_file_with(payload);
1183
1184        // Prefer a high VA; may fail and slide — both OK for this test.
1185        let preferred = 0x0000_0001_0000_0000_u64;
1186        let reqs = [MapRequest {
1187            name: "__TEST".into(),
1188            preferred_va: preferred,
1189            vmsize: page,
1190            fileoff: 0,
1191            filesize: u64::try_from(payload.len()).unwrap(),
1192            initprot: VM_PROT_READ | VM_PROT_WRITE,
1193            maxprot: VM_PROT_READ | VM_PROT_WRITE,
1194            svc_scan_ranges: Vec::new(),
1195        }];
1196
1197        let mem = GuestMemory::map_image(host, preferred, &reqs, &mut file).expect("map");
1198        assert_eq!(mem.regions().len(), 1);
1199        let region = mem.regions().first().expect("region");
1200        assert!(region.host_len() >= payload.len());
1201        let got = region.host_bytes().get(..payload.len()).expect("bytes");
1202        assert_eq!(got, payload);
1203        assert_eq!(region.file_bytes, u64::try_from(payload.len()).unwrap());
1204        drop(mem);
1205        drop(std::fs::remove_file(path));
1206    }
1207
1208    #[test]
1209    fn prot_conversion() {
1210        assert_eq!(darwin_to_host_prot(0), libc::PROT_NONE);
1211        assert_eq!(darwin_to_host_prot(VM_PROT_READ), libc::PROT_READ);
1212        assert_eq!(
1213            darwin_to_host_prot(VM_PROT_READ | VM_PROT_EXECUTE),
1214            libc::PROT_READ | libc::PROT_EXEC
1215        );
1216    }
1217
1218    /// Real `kh-libsystem` ships with preferred base 0 (`__TEXT` at 0, BSS DATA,
1219    /// LINKEDIT). Preferred fixed maps fail on Linux; slid reserved-span must work.
1220    #[test]
1221    fn map_base0_multi_segment_like_libsystem() {
1222        let host = HostPageSize::detect().expect("host page");
1223        let page = u64::from(host.bytes());
1224        // Layout mirrors staged libSystem.B.dylib (16 KiB TEXT + ~272 KiB DATA BSS + LINKEDIT).
1225        let text_sz = page.max(0x4000);
1226        let data_sz = 0x4_4000_u64;
1227        let link_sz = page.max(0x4000);
1228        let text_va = 0_u64;
1229        let data_va = text_sz;
1230        let link_va = data_va + data_sz;
1231
1232        let payload = b"base0-text-payload";
1233        let (mut file, path) = temp_file_with(payload);
1234
1235        let reqs = [
1236            MapRequest {
1237                name: "__TEXT".into(),
1238                preferred_va: text_va,
1239                vmsize: text_sz,
1240                fileoff: 0,
1241                filesize: u64::try_from(payload.len()).unwrap(),
1242                initprot: VM_PROT_READ | VM_PROT_EXECUTE,
1243                maxprot: VM_PROT_READ | VM_PROT_EXECUTE,
1244                svc_scan_ranges: Vec::new(),
1245            },
1246            MapRequest {
1247                name: "__DATA".into(),
1248                preferred_va: data_va,
1249                vmsize: data_sz,
1250                fileoff: 0,
1251                filesize: 0,
1252                initprot: VM_PROT_READ | VM_PROT_WRITE,
1253                maxprot: VM_PROT_READ | VM_PROT_WRITE,
1254                svc_scan_ranges: Vec::new(),
1255            },
1256            MapRequest {
1257                name: "__LINKEDIT".into(),
1258                preferred_va: link_va,
1259                vmsize: link_sz,
1260                fileoff: 0,
1261                filesize: 0,
1262                initprot: VM_PROT_READ,
1263                maxprot: VM_PROT_READ,
1264                svc_scan_ranges: Vec::new(),
1265            },
1266        ];
1267
1268        let mem = GuestMemory::map_image(host, 0, &reqs, &mut file).expect("map base0 image");
1269        assert_eq!(mem.regions().len(), 3);
1270        // Relative layout must be preserved under slide.
1271        let text = mem.regions().first().expect("text");
1272        let data = mem.regions().get(1).expect("data");
1273        let link = mem.regions().get(2).expect("link");
1274        assert_eq!(
1275            data.guest_addr.wrapping_sub(text.guest_addr),
1276            data_va.wrapping_sub(text_va)
1277        );
1278        assert_eq!(
1279            link.guest_addr.wrapping_sub(text.guest_addr),
1280            link_va.wrapping_sub(text_va)
1281        );
1282        let got = text.host_bytes().get(..payload.len()).expect("text bytes");
1283        assert_eq!(got, payload);
1284        // Preferred base 0 never sticks on Linux; slide may be 0 only if the
1285        // kernel actually accepted fixed maps at 0 (rare). Either is fine.
1286        let _ = mem.slide();
1287        drop(mem);
1288        drop(std::fs::remove_file(path));
1289    }
1290
1291    #[test]
1292    fn image_span_covers_aligned_ends() {
1293        let host = HostPageSize::detect().expect("host page");
1294        let page = u64::from(host.bytes());
1295        let reqs = [
1296            MapRequest {
1297                name: "a".into(),
1298                preferred_va: 0,
1299                vmsize: page,
1300                fileoff: 0,
1301                filesize: 0,
1302                initprot: VM_PROT_READ,
1303                maxprot: VM_PROT_READ,
1304                svc_scan_ranges: Vec::new(),
1305            },
1306            MapRequest {
1307                name: "b".into(),
1308                preferred_va: page * 2,
1309                vmsize: page,
1310                fileoff: 0,
1311                filesize: 0,
1312                initprot: VM_PROT_READ,
1313                maxprot: VM_PROT_READ,
1314                svc_scan_ranges: Vec::new(),
1315            },
1316        ];
1317        let refs: Vec<&MapRequest> = reqs.iter().collect();
1318        let (lo, hi) = image_preferred_span(host, &refs).expect("span");
1319        assert_eq!(lo, 0);
1320        assert_eq!(hi, page * 3);
1321    }
1322}