1use std::fs::File;
21use std::io::{Read, Seek, SeekFrom};
22use std::os::fd::AsRawFd;
23use std::ptr;
24
25use thiserror::Error;
26
27use crate::host;
28
29use super::layout::{HostPageSize, PageLayout};
30
31pub const VM_PROT_READ: u32 = 0x1;
33pub const VM_PROT_WRITE: u32 = 0x2;
34pub const VM_PROT_EXECUTE: u32 = 0x4;
35
36#[derive(Debug)]
38pub struct MappedRegion {
39 pub name: String,
41 pub guest_addr: u64,
43 ptr: *mut u8,
45 len: usize,
47 pub prot: u32,
49 pub file_bytes: u64,
51 pub vmsize: u64,
53 pub svc_scan_ranges: Vec<(u64, u64)>,
56}
57
58unsafe impl Send for MappedRegion {}
62
63impl MappedRegion {
64 #[must_use]
66 pub fn host_bytes_mut(&mut self) -> &mut [u8] {
67 unsafe { std::slice::from_raw_parts_mut(self.ptr, self.len) }
70 }
71
72 #[must_use]
74 pub fn host_bytes(&self) -> &[u8] {
75 unsafe { std::slice::from_raw_parts(self.ptr, self.len) }
77 }
78
79 #[must_use]
81 pub fn host_addr(&self) -> u64 {
82 host::ptr_addr_u64(self.ptr)
83 }
84
85 #[must_use]
87 pub const fn host_len(&self) -> usize {
88 self.len
89 }
90}
91
92impl Drop for MappedRegion {
93 fn drop(&mut self) {
94 if !self.ptr.is_null() && self.len > 0 {
95 if !host::munmap(self.ptr, self.len) {
96 tracing::warn!(
97 name = %self.name,
98 len = self.len,
99 "munmap failed during MappedRegion drop"
100 );
101 }
102 self.ptr = ptr::null_mut();
103 }
104 }
105}
106
107#[derive(Debug, Clone)]
109pub struct MapRequest {
110 pub name: String,
112 pub preferred_va: u64,
114 pub vmsize: u64,
116 pub fileoff: u64,
118 pub filesize: u64,
120 pub initprot: u32,
122 pub maxprot: u32,
124 pub svc_scan_ranges: Vec<(u64, u64)>,
130}
131
132impl MapRequest {
133 #[must_use]
135 pub fn should_skip(&self) -> bool {
136 if self.vmsize == 0 {
137 return true;
138 }
139 self.name == "__PAGEZERO"
141 || (self.initprot == 0 && self.fileoff == 0 && self.filesize == 0)
142 }
143}
144
145#[derive(Debug)]
147pub struct GuestMemory {
148 regions: Vec<MappedRegion>,
149 slide: u64,
151 host: HostPageSize,
152 preferred_base: u64,
153}
154
155#[derive(Debug, Error)]
157pub enum MapError {
158 #[error("invalid map request: {0}")]
160 Invalid(&'static str),
161
162 #[error("mmap/mprotect failed for {name}: {source}")]
164 Sys {
165 name: String,
167 #[source]
169 source: std::io::Error,
170 },
171
172 #[error("file I/O while mapping {name}: {source}")]
174 File {
175 name: String,
177 #[source]
179 source: std::io::Error,
180 },
181
182 #[error("could not place guest image in host VA space")]
184 PlacementFailed,
185}
186
187const MAX_SLIDE_SPAN: u64 = 512 * 1024 * 1024; impl GuestMemory {
191 pub fn map_image(
197 host: HostPageSize,
198 preferred_base: u64,
199 requests: &[MapRequest],
200 file: &mut File,
201 ) -> Result<Self, MapError> {
202 let mut src = SegmentSource::File(file);
203 Self::map_image_with(host, preferred_base, requests, &mut src)
204 }
205
206 pub fn map_image_bytes(
210 host: HostPageSize,
211 preferred_base: u64,
212 requests: &[MapRequest],
213 bytes: &[u8],
214 ) -> Result<Self, MapError> {
215 let mut src = SegmentSource::Bytes(bytes);
216 Self::map_image_with(host, preferred_base, requests, &mut src)
217 }
218
219 fn map_image_with(
220 host: HostPageSize,
221 preferred_base: u64,
222 requests: &[MapRequest],
223 src: &mut SegmentSource<'_>,
224 ) -> Result<Self, MapError> {
225 let active: Vec<&MapRequest> = requests.iter().filter(|r| !r.should_skip()).collect();
226 if active.is_empty() {
227 return Ok(Self {
228 regions: Vec::new(),
229 slide: 0,
230 host,
231 preferred_base,
232 });
233 }
234
235 match Self::try_map_all(host, preferred_base, 0, &active, src, FixedMode::NoReplace) {
237 Ok(regions) => {
238 return Ok(Self {
239 regions,
240 slide: 0,
241 host,
242 preferred_base,
243 });
244 }
245 Err(err) => {
246 tracing::debug!(error = %err, "preferred-base map failed; trying slid placement");
247 }
248 }
249
250 match Self::try_map_slid_reserved(host, preferred_base, &active, src) {
252 Ok(mem) => return Ok(mem),
253 Err(err) => {
254 tracing::debug!(error = %err, "reserved-span slid map failed; trying first-region slide");
255 }
256 }
257
258 Self::try_map_slid_first_region(host, preferred_base, &active, src)
261 }
262
263 fn try_map_slid_reserved(
265 host: HostPageSize,
266 preferred_base: u64,
267 active: &[&MapRequest],
268 src: &mut SegmentSource<'_>,
269 ) -> Result<Self, MapError> {
270 let (span_lo, span_hi) = image_preferred_span(host, active)?;
271 let span_len_u64 = span_hi.saturating_sub(span_lo);
272 if span_len_u64 == 0 {
273 return Err(MapError::Invalid("empty image span"));
274 }
275 if span_len_u64 > MAX_SLIDE_SPAN {
276 return Err(MapError::Invalid(
277 "image span too large for slid reservation",
278 ));
279 }
280 let span_len =
281 usize::try_from(span_len_u64).map_err(|_| MapError::Invalid("span too large"))?;
282
283 let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
285 let Some(reserve_ptr) = host::mmap(None, span_len, libc::PROT_NONE, flags, -1, 0) else {
286 return Err(MapError::Sys {
287 name: "__RESERVE".into(),
288 source: std::io::Error::last_os_error(),
289 });
290 };
291 let reserve_base = host::ptr_addr_u64(reserve_ptr);
292 let slide = reserve_base.wrapping_sub(span_lo);
293
294 tracing::debug!(
295 span_lo = format_args!("{span_lo:#x}"),
296 span_hi = format_args!("{span_hi:#x}"),
297 reserve = format_args!("{reserve_base:#x}"),
298 slide = format_args!("{slide:#x}"),
299 "reserved contiguous host span for slid image"
300 );
301
302 match Self::try_map_all(
304 host,
305 preferred_base,
306 slide,
307 active,
308 src,
309 FixedMode::Replace,
310 ) {
311 Ok(regions) => {
312 unmap_span_gaps(reserve_base, span_len_u64, ®ions);
315 Ok(Self {
316 regions,
317 slide,
318 host,
319 preferred_base,
320 })
321 }
322 Err(err) => {
323 let _ = host::munmap(reserve_ptr, span_len);
327 Err(err)
328 }
329 }
330 }
331
332 fn try_map_slid_first_region(
334 host: HostPageSize,
335 preferred_base: u64,
336 active: &[&MapRequest],
337 src: &mut SegmentSource<'_>,
338 ) -> Result<Self, MapError> {
339 let first = active
340 .first()
341 .copied()
342 .ok_or(MapError::Invalid("no active regions"))?;
343 let first_map = map_one(host, first, first.preferred_va, src, FixedMode::Free)?;
344 let slide = first_map.guest_addr.wrapping_sub(first.preferred_va);
345
346 let mut regions = vec![first_map];
347 for req in active.iter().skip(1) {
348 let target = req.preferred_va.wrapping_add(slide);
349 match map_one(host, req, target, src, FixedMode::NoReplace) {
350 Ok(region) => regions.push(region),
351 Err(err) => {
352 drop(regions);
353 tracing::debug!(error = %err, "slid fixed map failed");
354 return Err(MapError::PlacementFailed);
355 }
356 }
357 }
358
359 Ok(Self {
360 regions,
361 slide,
362 host,
363 preferred_base,
364 })
365 }
366
367 fn try_map_all(
368 host: HostPageSize,
369 preferred_base: u64,
370 slide: u64,
371 active: &[&MapRequest],
372 src: &mut SegmentSource<'_>,
373 fixed: FixedMode,
374 ) -> Result<Vec<MappedRegion>, MapError> {
375 let mut regions = Vec::with_capacity(active.len());
376 for req in active {
377 let target = req.preferred_va.wrapping_add(slide);
378 match map_one(host, req, target, src, fixed) {
379 Ok(region) => regions.push(region),
380 Err(err) => {
381 drop(regions);
382 return Err(err);
383 }
384 }
385 }
386 let _ = preferred_base;
387 Ok(regions)
388 }
389
390 #[must_use]
392 pub const fn slide(&self) -> u64 {
393 self.slide
394 }
395
396 #[must_use]
398 pub const fn preferred_base(&self) -> u64 {
399 self.preferred_base
400 }
401
402 #[must_use]
404 pub const fn host(&self) -> HostPageSize {
405 self.host
406 }
407
408 #[must_use]
410 pub fn regions(&self) -> &[MappedRegion] {
411 &self.regions
412 }
413
414 pub fn regions_mut(&mut self) -> &mut [MappedRegion] {
416 &mut self.regions
417 }
418
419 #[must_use]
421 pub fn guest_to_host(&self, guest_va: u64) -> Option<*const u8> {
422 for region in &self.regions {
423 let start = region.guest_addr;
424 let end = start.saturating_add(region.vmsize);
425 if guest_va >= start && guest_va < end {
426 let offset = usize::try_from(guest_va.saturating_sub(start)).ok()?;
427 if offset >= region.len {
428 return None;
429 }
430 return Some(unsafe { region.ptr.add(offset) });
432 }
433 }
434 None
435 }
436
437 #[must_use]
439 pub fn guest_to_host_mut(&mut self, guest_va: u64) -> Option<*mut u8> {
440 for region in &mut self.regions {
441 let start = region.guest_addr;
442 let end = start.saturating_add(region.vmsize);
443 if guest_va >= start && guest_va < end {
444 let offset = usize::try_from(guest_va.saturating_sub(start)).ok()?;
445 if offset >= region.len {
446 return None;
447 }
448 return Some(unsafe { region.ptr.add(offset) });
449 }
450 }
451 None
452 }
453
454 #[must_use]
456 pub fn read_u32_le(&self, guest_va: u64) -> Option<u32> {
457 let mut buf = [0_u8; 4];
458 self.read_exact(guest_va, &mut buf)?;
459 Some(u32::from_le_bytes(buf))
460 }
461
462 #[must_use]
464 pub fn read_u64_le(&self, guest_va: u64) -> Option<u64> {
465 let mut buf = [0_u8; 8];
466 self.read_exact(guest_va, &mut buf)?;
467 Some(u64::from_le_bytes(buf))
468 }
469
470 pub fn write_u64_le(&mut self, guest_va: u64, value: u64) -> Option<()> {
474 self.write_exact(guest_va, &value.to_le_bytes())
475 }
476
477 pub fn read_exact(&self, guest_va: u64, buf: &mut [u8]) -> Option<()> {
479 if buf.is_empty() {
480 return Some(());
481 }
482 for region in &self.regions {
483 let start = region.guest_addr;
484 let end = start.saturating_add(region.vmsize);
485 if guest_va >= start && guest_va < end {
486 let offset = usize::try_from(guest_va.saturating_sub(start)).ok()?;
487 let need = buf.len();
488 let avail = region.len.saturating_sub(offset);
489 if need > avail {
490 return None;
491 }
492 let host = region.host_bytes();
493 let src = host.get(offset..offset.saturating_add(need))?;
494 buf.copy_from_slice(src);
495 return Some(());
496 }
497 }
498 None
499 }
500
501 pub fn write_exact(&mut self, guest_va: u64, buf: &[u8]) -> Option<()> {
503 if buf.is_empty() {
504 return Some(());
505 }
506 for region in &mut self.regions {
507 let start = region.guest_addr;
508 let end = start.saturating_add(region.vmsize);
509 if guest_va >= start && guest_va < end {
510 let offset = usize::try_from(guest_va.saturating_sub(start)).ok()?;
511 let need = buf.len();
512 let avail = region.len.saturating_sub(offset);
513 if need > avail {
514 return None;
515 }
516 let host = region.host_bytes_mut();
517 let dst = host.get_mut(offset..offset.saturating_add(need))?;
518 dst.copy_from_slice(buf);
519 return Some(());
520 }
521 }
522 None
523 }
524
525 #[doc(hidden)]
532 pub fn test_offset_guest_vas(&mut self, delta: u64) {
533 self.slide = self.slide.wrapping_add(delta);
534 for region in &mut self.regions {
535 region.guest_addr = region.guest_addr.wrapping_add(delta);
536 }
537 }
538}
539
540#[derive(Debug, Clone, Copy, PartialEq, Eq)]
542enum FixedMode {
543 Free,
545 NoReplace,
547 Replace,
550}
551
552enum SegmentSource<'a> {
554 File(&'a mut File),
555 Bytes(&'a [u8]),
556}
557
558impl SegmentSource<'_> {
559 fn copy_into(
560 &mut self,
561 fileoff: u64,
562 dst: &mut [u8],
563 region_name: &str,
564 ) -> Result<(), MapError> {
565 match self {
566 SegmentSource::File(file) => {
567 file.seek(SeekFrom::Start(fileoff))
568 .map_err(|source| MapError::File {
569 name: region_name.to_owned(),
570 source,
571 })?;
572 file.read_exact(dst).map_err(|source| MapError::File {
573 name: region_name.to_owned(),
574 source,
575 })
576 }
577 SegmentSource::Bytes(bytes) => {
578 let start = usize::try_from(fileoff).map_err(|_| MapError::Invalid("fileoff"))?;
579 let end = start
580 .checked_add(dst.len())
581 .ok_or(MapError::Invalid("file range overflow"))?;
582 let Some(src) = bytes.get(start..end) else {
583 return Err(MapError::Invalid("fileoff past container end"));
584 };
585 dst.copy_from_slice(src);
586 Ok(())
587 }
588 }
589 }
590}
591
592fn map_one(
593 host: HostPageSize,
594 req: &MapRequest,
595 target_guest: u64,
596 src: &mut SegmentSource<'_>,
597 mode: FixedMode,
598) -> Result<MappedRegion, MapError> {
599 let host_page = host.bytes();
600 let map_len_u64 = PageLayout::align_up(req.vmsize, host_page)
601 .ok_or(MapError::Invalid("vmsize align overflow"))?;
602 let map_len =
603 usize::try_from(map_len_u64).map_err(|_| MapError::Invalid("vmsize too large"))?;
604 if map_len == 0 {
605 return Err(MapError::Invalid("zero map length"));
606 }
607
608 let fixed = matches!(mode, FixedMode::NoReplace | FixedMode::Replace);
609 let fixed_addr = if fixed { Some(target_guest) } else { None };
610
611 if let SegmentSource::File(file) = src
615 && let Some(region) =
616 try_map_pure_file(host, req, target_guest, file, mode, map_len, fixed)
617 {
618 return Ok(region);
619 }
620
621 let mut flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
622 match mode {
623 FixedMode::Free => {}
624 FixedMode::NoReplace => flags |= host::fixed_map_flag(),
625 FixedMode::Replace => flags |= libc::MAP_FIXED,
626 }
627
628 let prot_rw = libc::PROT_READ | libc::PROT_WRITE;
630
631 let Some(base) = host::mmap(fixed_addr, map_len, prot_rw, flags, -1, 0) else {
632 return Err(MapError::Sys {
633 name: req.name.clone(),
634 source: std::io::Error::last_os_error(),
635 });
636 };
637
638 let actual_host = host::ptr_addr_u64(base);
639 let guest_addr = if fixed { target_guest } else { actual_host };
641
642 let mut region = MappedRegion {
643 name: req.name.clone(),
644 guest_addr,
645 ptr: base,
646 len: map_len,
647 prot: req.initprot,
648 file_bytes: 0,
649 vmsize: req.vmsize,
650 svc_scan_ranges: req.svc_scan_ranges.clone(),
651 };
652
653 let mut dirtied = false;
655
656 if req.filesize > 0 {
658 let copy_len = usize::try_from(req.filesize.min(req.vmsize))
659 .map_err(|_| MapError::Invalid("filesize too large"))?;
660 if copy_len > map_len {
661 return Err(MapError::Invalid("filesize exceeds map length"));
662 }
663 let fill = match src {
664 SegmentSource::File(file) => fill_segment_file(
665 file,
666 base,
667 map_len,
668 req.fileoff,
669 copy_len,
670 host_page,
671 &req.name,
672 )?,
673 SegmentSource::Bytes(bytes) => {
674 fill_segment_bytes(bytes, base, map_len, req.fileoff, copy_len, host_page)?
675 }
676 };
677 if fill.ok {
678 dirtied = fill.dirtied;
679 } else {
680 let dst = region.host_bytes_mut();
681 let Some(slice) = dst.get_mut(..copy_len) else {
682 return Err(MapError::Invalid("copy slice out of range"));
683 };
684 src.copy_into(req.fileoff, slice, &req.name)?;
685 dirtied = true;
686 }
687 region.file_bytes = u64::try_from(copy_len).unwrap_or(0);
688 }
689
690 let host_prot = darwin_to_host_prot(req.initprot);
692 if !host::mprotect(base, map_len, host_prot) {
693 return Err(MapError::Sys {
694 name: req.name.clone(),
695 source: std::io::Error::last_os_error(),
696 });
697 }
698
699 if dirtied && req.initprot & VM_PROT_EXECUTE != 0 {
703 let flush_len = if region.file_bytes == 0 {
704 let page = usize::try_from(host_page).unwrap_or(4096);
705 map_len.min(page).max(4)
706 } else {
707 usize::try_from(region.file_bytes)
708 .unwrap_or(0)
709 .min(map_len)
710 .max(4)
711 };
712 clear_icache(base, flush_len);
713 }
714
715 if fixed && actual_host != target_guest {
717 return Err(MapError::Sys {
718 name: req.name.clone(),
719 source: std::io::Error::new(
720 std::io::ErrorKind::AddrNotAvailable,
721 format!("kernel placed map at {actual_host:#x}, wanted {target_guest:#x}"),
722 ),
723 });
724 }
725
726 Ok(region)
727}
728
729fn try_map_pure_file(
734 host: HostPageSize,
735 req: &MapRequest,
736 target_guest: u64,
737 file: &File,
738 mode: FixedMode,
739 map_len: usize,
740 fixed: bool,
741) -> Option<MappedRegion> {
742 let host_page = u64::from(host.bytes());
743 if host_page == 0 {
744 return None;
745 }
746 let copy_u64 = req.filesize.min(req.vmsize);
747 if copy_u64 == 0 {
748 return None;
749 }
750 let Ok(copy_len) = usize::try_from(copy_u64) else {
751 return None;
752 };
753 if copy_len != map_len {
755 return None;
756 }
757 if !req.fileoff.is_multiple_of(host_page) {
758 return None;
759 }
760 if fixed && !target_guest.is_multiple_of(host_page) {
761 return None;
762 }
763
764 let mut flags = libc::MAP_PRIVATE;
765 match mode {
766 FixedMode::Free => {}
767 FixedMode::NoReplace => flags |= host::fixed_map_flag(),
768 FixedMode::Replace => flags |= libc::MAP_FIXED,
769 }
770 let host_prot = darwin_to_host_prot(req.initprot);
771 let fixed_addr = if fixed { Some(target_guest) } else { None };
772 let fd = file.as_raw_fd();
773 let Ok(file_off) = i64::try_from(req.fileoff) else {
774 return None;
775 };
776 let Some(base) = host::mmap(fixed_addr, map_len, host_prot, flags, fd, file_off) else {
777 return None;
779 };
780 let actual_host = host::ptr_addr_u64(base);
781 if fixed && actual_host != target_guest {
782 let _ = host::munmap(base, map_len);
783 return None;
784 }
785 let guest_addr = if fixed { target_guest } else { actual_host };
786 Some(MappedRegion {
787 name: req.name.clone(),
788 guest_addr,
789 ptr: base,
790 len: map_len,
791 prot: req.initprot,
792 file_bytes: copy_u64,
793 vmsize: req.vmsize,
794 svc_scan_ranges: req.svc_scan_ranges.clone(),
795 })
796}
797
798fn read_file_range(file: &File, off: u64, dst: &mut [u8], name: &str) -> Result<(), MapError> {
800 let mut clone = file.try_clone().map_err(|source| MapError::File {
801 name: name.to_owned(),
802 source,
803 })?;
804 clone
805 .seek(SeekFrom::Start(off))
806 .map_err(|source| MapError::File {
807 name: name.to_owned(),
808 source,
809 })?;
810 clone.read_exact(dst).map_err(|source| MapError::File {
811 name: name.to_owned(),
812 source,
813 })
814}
815
816struct FillOutcome {
818 ok: bool,
820 dirtied: bool,
822}
823
824fn fill_segment_file(
829 file: &File,
830 base: *mut u8,
831 map_len: usize,
832 fileoff: u64,
833 copy_len: usize,
834 host_page: u32,
835 name: &str,
836) -> Result<FillOutcome, MapError> {
837 let page = u64::from(host_page);
838 if page == 0 || copy_len == 0 || copy_len > map_len {
839 return Ok(FillOutcome {
840 ok: false,
841 dirtied: false,
842 });
843 }
844
845 let content_lo = fileoff;
846 let content_hi = fileoff.saturating_add(u64::try_from(copy_len).unwrap_or(0));
847 let aligned_lo = content_lo.div_ceil(page).saturating_mul(page);
849 let aligned_hi = content_hi
850 .checked_div(page)
851 .unwrap_or(0)
852 .saturating_mul(page);
853
854 let lead = usize::try_from(aligned_lo.saturating_sub(content_lo)).unwrap_or(0);
856 let trail_off = usize::try_from(aligned_hi.saturating_sub(content_lo)).unwrap_or(0);
857 let mut dirtied = false;
858
859 if aligned_hi > aligned_lo {
861 let mid_len = usize::try_from(aligned_hi.saturating_sub(aligned_lo)).unwrap_or(0);
862 if mid_len == 0 || lead.saturating_add(mid_len) > map_len {
863 return Ok(FillOutcome {
864 ok: false,
865 dirtied: false,
866 });
867 }
868 let dest = unsafe { base.add(lead) };
870 let fd = file.as_raw_fd();
871 let flags = libc::MAP_PRIVATE | libc::MAP_FIXED;
872 let Some(mapped) = host::mmap(
873 Some(host::ptr_addr_u64(dest)),
874 mid_len,
875 libc::PROT_READ | libc::PROT_WRITE,
876 flags,
877 fd,
878 i64::try_from(aligned_lo).unwrap_or(0),
879 ) else {
880 return Err(MapError::Sys {
881 name: name.to_owned(),
882 source: std::io::Error::last_os_error(),
883 });
884 };
885 if mapped != dest {
886 let _ = host::munmap(mapped, mid_len);
887 return Err(MapError::Sys {
888 name: name.to_owned(),
889 source: std::io::Error::new(
890 std::io::ErrorKind::AddrNotAvailable,
891 "file-backed MAP_FIXED moved",
892 ),
893 });
894 }
895 }
896
897 if aligned_hi <= aligned_lo {
899 let dst = unsafe { std::slice::from_raw_parts_mut(base, copy_len) };
901 read_file_range(file, fileoff, dst, name)?;
902 return Ok(FillOutcome {
903 ok: true,
904 dirtied: true,
905 });
906 }
907
908 if lead > 0 {
910 let n = lead.min(copy_len);
911 let dst = unsafe { std::slice::from_raw_parts_mut(base, n) };
913 read_file_range(file, fileoff, dst, name)?;
914 dirtied = true;
915 }
916 if trail_off < copy_len {
918 let n = copy_len.saturating_sub(trail_off);
919 if n > 0 && trail_off < map_len {
920 let len = n.min(map_len.saturating_sub(trail_off));
921 let dst = unsafe { std::slice::from_raw_parts_mut(base.add(trail_off), len) };
923 let seek_at = fileoff.saturating_add(u64::try_from(trail_off).unwrap_or(0));
924 read_file_range(file, seek_at, dst, name)?;
925 dirtied = true;
926 }
927 }
928
929 Ok(FillOutcome {
930 ok: true,
931 dirtied,
932 })
933}
934
935fn fill_segment_bytes(
941 bytes: &[u8],
942 base: *mut u8,
943 map_len: usize,
944 fileoff: u64,
945 copy_len: usize,
946 _host_page: u32,
947) -> Result<FillOutcome, MapError> {
948 let start = usize::try_from(fileoff).map_err(|_| MapError::Invalid("fileoff"))?;
949 let end = start
950 .checked_add(copy_len)
951 .ok_or(MapError::Invalid("file range overflow"))?;
952 let Some(src) = bytes.get(start..end) else {
953 return Err(MapError::Invalid("fileoff past container end"));
954 };
955 if copy_len > map_len {
956 return Err(MapError::Invalid("filesize exceeds map length"));
957 }
958 let dst = unsafe { std::slice::from_raw_parts_mut(base, copy_len) };
959 dst.copy_from_slice(src);
960 Ok(FillOutcome {
961 ok: true,
962 dirtied: true,
963 })
964}
965
966#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
968unsafe extern "C" {
969 fn __clear_cache(start: *mut libc::c_void, end: *mut libc::c_void);
970}
971
972#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
974fn clear_icache(ptr: *mut u8, len: usize) {
975 if ptr.is_null() || len == 0 {
976 return;
977 }
978 unsafe {
980 let start = ptr.cast::<libc::c_void>();
981 let end = ptr.wrapping_add(len).cast::<libc::c_void>();
982 __clear_cache(start, end);
983 }
984}
985
986#[cfg(not(all(target_arch = "aarch64", target_os = "linux")))]
987fn clear_icache(_ptr: *mut u8, _len: usize) {}
988
989fn image_preferred_span(
991 host: HostPageSize,
992 active: &[&MapRequest],
993) -> Result<(u64, u64), MapError> {
994 let host_page = host.bytes();
995 let mut lo = u64::MAX;
996 let mut hi = 0_u64;
997 for req in active {
998 lo = lo.min(req.preferred_va);
999 let end = req
1000 .preferred_va
1001 .checked_add(
1002 PageLayout::align_up(req.vmsize, host_page)
1003 .ok_or(MapError::Invalid("vmsize align overflow"))?,
1004 )
1005 .ok_or(MapError::Invalid("segment end overflow"))?;
1006 hi = hi.max(end);
1007 }
1008 if lo == u64::MAX || hi <= lo {
1009 return Err(MapError::Invalid("no span from active segments"));
1010 }
1011 let lo = PageLayout::align_down(lo, host_page).unwrap_or(lo);
1013 Ok((lo, hi))
1014}
1015
1016fn unmap_span_gaps(reserve_base: u64, span_len: u64, regions: &[MappedRegion]) {
1018 if span_len == 0 || regions.is_empty() {
1019 return;
1020 }
1021 let span_end = reserve_base.saturating_add(span_len);
1022 let mut covered: Vec<(u64, u64)> = regions
1023 .iter()
1024 .map(|r| {
1025 let start = r.guest_addr;
1026 let end = start.saturating_add(u64::try_from(r.len).unwrap_or(0));
1027 (start, end)
1028 })
1029 .collect();
1030 covered.sort_by_key(|(s, _)| *s);
1031
1032 let mut cursor = reserve_base;
1033 for (start, end) in covered {
1034 if start > cursor && start <= span_end {
1035 let gap_len = start.saturating_sub(cursor);
1036 if let Ok(len) = usize::try_from(gap_len)
1037 && len > 0
1038 {
1039 let ptr = host::u64_as_void_ptr(cursor).cast::<u8>();
1040 let _ = host::munmap(ptr, len);
1041 }
1042 }
1043 cursor = cursor.max(end);
1044 }
1045 if cursor < span_end {
1046 let gap_len = span_end.saturating_sub(cursor);
1047 if let Ok(len) = usize::try_from(gap_len)
1048 && len > 0
1049 {
1050 let ptr = host::u64_as_void_ptr(cursor).cast::<u8>();
1051 let _ = host::munmap(ptr, len);
1052 }
1053 }
1054}
1055
1056#[must_use]
1058pub const fn darwin_to_host_prot(darwin: u32) -> libc::c_int {
1059 if darwin == 0 {
1060 return libc::PROT_NONE;
1061 }
1062 let mut prot = 0;
1063 if darwin & VM_PROT_READ != 0 {
1064 prot |= libc::PROT_READ;
1065 }
1066 if darwin & VM_PROT_WRITE != 0 {
1067 prot |= libc::PROT_WRITE;
1068 }
1069 if darwin & VM_PROT_EXECUTE != 0 {
1070 prot |= libc::PROT_EXEC;
1071 }
1072 prot
1073}
1074
1075pub fn map_stack(host: HostPageSize, size: u64) -> Result<MappedRegion, MapError> {
1081 let host_page = host.bytes();
1082 let map_len_u64 =
1083 PageLayout::align_up(size.max(1), host_page).ok_or(MapError::Invalid("stack align"))?;
1084 let map_len = usize::try_from(map_len_u64).map_err(|_| MapError::Invalid("stack too large"))?;
1085
1086 let flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS;
1087 let prot = libc::PROT_READ | libc::PROT_WRITE;
1088 let Some(base) = host::mmap(None, map_len, prot, flags, -1, 0) else {
1089 return Err(MapError::Sys {
1090 name: "__STACK".into(),
1091 source: std::io::Error::last_os_error(),
1092 });
1093 };
1094 Ok(MappedRegion {
1095 name: "__STACK".into(),
1096 guest_addr: host::ptr_addr_u64(base),
1097 ptr: base,
1098 len: map_len,
1099 prot: VM_PROT_READ | VM_PROT_WRITE,
1100 file_bytes: 0,
1101 vmsize: map_len_u64,
1102 svc_scan_ranges: Vec::new(),
1103 })
1104}
1105
1106pub fn mprotect_rw(region: &MappedRegion) -> Result<(), MapError> {
1110 if region.prot & VM_PROT_WRITE != 0 {
1111 return Ok(());
1112 }
1113 if host::mprotect(region.ptr, region.len, libc::PROT_READ | libc::PROT_WRITE) {
1114 Ok(())
1115 } else {
1116 Err(MapError::Sys {
1117 name: region.name.clone(),
1118 source: std::io::Error::last_os_error(),
1119 })
1120 }
1121}
1122
1123pub fn mprotect_darwin(region: &MappedRegion, darwin_prot: u32) -> Result<(), MapError> {
1125 let host_prot = darwin_to_host_prot(darwin_prot);
1126 if host::mprotect(region.ptr, region.len, host_prot) {
1127 Ok(())
1128 } else {
1129 Err(MapError::Sys {
1130 name: region.name.clone(),
1131 source: std::io::Error::last_os_error(),
1132 })
1133 }
1134}
1135
1136#[cfg(test)]
1137#[allow(clippy::expect_used, clippy::unwrap_used)]
1138mod tests {
1139 use super::*;
1140 use std::io::{Seek, SeekFrom, Write};
1141 use std::path::PathBuf;
1142
1143 fn temp_file_with(bytes: &[u8]) -> (File, PathBuf) {
1144 use std::fs::OpenOptions;
1145 let path = std::env::temp_dir().join(format!(
1146 "kh-map-test-{}-{}",
1147 std::process::id(),
1148 bytes.len()
1149 ));
1150 let mut f = OpenOptions::new()
1151 .read(true)
1152 .write(true)
1153 .create(true)
1154 .truncate(true)
1155 .open(&path)
1156 .expect("create temp");
1157 f.write_all(bytes).expect("write");
1158 f.seek(SeekFrom::Start(0)).expect("seek");
1159 (f, path)
1160 }
1161
1162 #[test]
1163 fn skip_pagezero() {
1164 let req = MapRequest {
1165 name: "__PAGEZERO".into(),
1166 preferred_va: 0,
1167 vmsize: 0x1_0000_0000,
1168 fileoff: 0,
1169 filesize: 0,
1170 initprot: 0,
1171 maxprot: 0,
1172 svc_scan_ranges: Vec::new(),
1173 };
1174 assert!(req.should_skip());
1175 }
1176
1177 #[test]
1178 fn map_anonymous_rw_region() {
1179 let host = HostPageSize::detect().expect("host page");
1180 let page = u64::from(host.bytes());
1181 let payload = b"hello-guest-map";
1182 let (mut file, path) = temp_file_with(payload);
1183
1184 let preferred = 0x0000_0001_0000_0000_u64;
1186 let reqs = [MapRequest {
1187 name: "__TEST".into(),
1188 preferred_va: preferred,
1189 vmsize: page,
1190 fileoff: 0,
1191 filesize: u64::try_from(payload.len()).unwrap(),
1192 initprot: VM_PROT_READ | VM_PROT_WRITE,
1193 maxprot: VM_PROT_READ | VM_PROT_WRITE,
1194 svc_scan_ranges: Vec::new(),
1195 }];
1196
1197 let mem = GuestMemory::map_image(host, preferred, &reqs, &mut file).expect("map");
1198 assert_eq!(mem.regions().len(), 1);
1199 let region = mem.regions().first().expect("region");
1200 assert!(region.host_len() >= payload.len());
1201 let got = region.host_bytes().get(..payload.len()).expect("bytes");
1202 assert_eq!(got, payload);
1203 assert_eq!(region.file_bytes, u64::try_from(payload.len()).unwrap());
1204 drop(mem);
1205 drop(std::fs::remove_file(path));
1206 }
1207
1208 #[test]
1209 fn prot_conversion() {
1210 assert_eq!(darwin_to_host_prot(0), libc::PROT_NONE);
1211 assert_eq!(darwin_to_host_prot(VM_PROT_READ), libc::PROT_READ);
1212 assert_eq!(
1213 darwin_to_host_prot(VM_PROT_READ | VM_PROT_EXECUTE),
1214 libc::PROT_READ | libc::PROT_EXEC
1215 );
1216 }
1217
1218 #[test]
1221 fn map_base0_multi_segment_like_libsystem() {
1222 let host = HostPageSize::detect().expect("host page");
1223 let page = u64::from(host.bytes());
1224 let text_sz = page.max(0x4000);
1226 let data_sz = 0x4_4000_u64;
1227 let link_sz = page.max(0x4000);
1228 let text_va = 0_u64;
1229 let data_va = text_sz;
1230 let link_va = data_va + data_sz;
1231
1232 let payload = b"base0-text-payload";
1233 let (mut file, path) = temp_file_with(payload);
1234
1235 let reqs = [
1236 MapRequest {
1237 name: "__TEXT".into(),
1238 preferred_va: text_va,
1239 vmsize: text_sz,
1240 fileoff: 0,
1241 filesize: u64::try_from(payload.len()).unwrap(),
1242 initprot: VM_PROT_READ | VM_PROT_EXECUTE,
1243 maxprot: VM_PROT_READ | VM_PROT_EXECUTE,
1244 svc_scan_ranges: Vec::new(),
1245 },
1246 MapRequest {
1247 name: "__DATA".into(),
1248 preferred_va: data_va,
1249 vmsize: data_sz,
1250 fileoff: 0,
1251 filesize: 0,
1252 initprot: VM_PROT_READ | VM_PROT_WRITE,
1253 maxprot: VM_PROT_READ | VM_PROT_WRITE,
1254 svc_scan_ranges: Vec::new(),
1255 },
1256 MapRequest {
1257 name: "__LINKEDIT".into(),
1258 preferred_va: link_va,
1259 vmsize: link_sz,
1260 fileoff: 0,
1261 filesize: 0,
1262 initprot: VM_PROT_READ,
1263 maxprot: VM_PROT_READ,
1264 svc_scan_ranges: Vec::new(),
1265 },
1266 ];
1267
1268 let mem = GuestMemory::map_image(host, 0, &reqs, &mut file).expect("map base0 image");
1269 assert_eq!(mem.regions().len(), 3);
1270 let text = mem.regions().first().expect("text");
1272 let data = mem.regions().get(1).expect("data");
1273 let link = mem.regions().get(2).expect("link");
1274 assert_eq!(
1275 data.guest_addr.wrapping_sub(text.guest_addr),
1276 data_va.wrapping_sub(text_va)
1277 );
1278 assert_eq!(
1279 link.guest_addr.wrapping_sub(text.guest_addr),
1280 link_va.wrapping_sub(text_va)
1281 );
1282 let got = text.host_bytes().get(..payload.len()).expect("text bytes");
1283 assert_eq!(got, payload);
1284 let _ = mem.slide();
1287 drop(mem);
1288 drop(std::fs::remove_file(path));
1289 }
1290
1291 #[test]
1292 fn image_span_covers_aligned_ends() {
1293 let host = HostPageSize::detect().expect("host page");
1294 let page = u64::from(host.bytes());
1295 let reqs = [
1296 MapRequest {
1297 name: "a".into(),
1298 preferred_va: 0,
1299 vmsize: page,
1300 fileoff: 0,
1301 filesize: 0,
1302 initprot: VM_PROT_READ,
1303 maxprot: VM_PROT_READ,
1304 svc_scan_ranges: Vec::new(),
1305 },
1306 MapRequest {
1307 name: "b".into(),
1308 preferred_va: page * 2,
1309 vmsize: page,
1310 fileoff: 0,
1311 filesize: 0,
1312 initprot: VM_PROT_READ,
1313 maxprot: VM_PROT_READ,
1314 svc_scan_ranges: Vec::new(),
1315 },
1316 ];
1317 let refs: Vec<&MapRequest> = reqs.iter().collect();
1318 let (lo, hi) = image_preferred_span(host, &refs).expect("span");
1319 assert_eq!(lo, 0);
1320 assert_eq!(hi, page * 3);
1321 }
1322}