Skip to main content

hyperlight_host/mem/
shared_mem.rs

1// SPDX-License-Identifier: Apache-2.0
2// Copyright 2025 The Hyperlight Authors.
3
4use std::ffi::c_void;
5use std::io::Error;
6use std::mem::{align_of, size_of};
7#[cfg(unix)]
8use std::ptr::null_mut;
9use std::sync::atomic::Ordering;
10use std::sync::{Arc, RwLock};
11
12use bytemuck::Pod;
13use thiserror::Error;
14use tracing::{Span, instrument};
15#[cfg(target_os = "windows")]
16use windows::Win32::Foundation::{CloseHandle, HANDLE, INVALID_HANDLE_VALUE};
17#[cfg(target_os = "windows")]
18use windows::Win32::System::Memory::PAGE_READWRITE;
19#[cfg(target_os = "windows")]
20use windows::Win32::System::Memory::{
21    CreateFileMappingA, FILE_MAP_ALL_ACCESS, MEM_PRESERVE_PLACEHOLDER, MEM_RELEASE,
22    MEM_REPLACE_PLACEHOLDER, MEM_RESERVE, MEM_RESERVE_PLACEHOLDER, MEMORY_MAPPED_VIEW_ADDRESS,
23    MapViewOfFile, MapViewOfFile3, PAGE_NOACCESS, PAGE_PROTECTION_FLAGS, PAGE_READONLY,
24    UnmapViewOfFile, VIRTUAL_ALLOCATION_TYPE, VIRTUAL_FREE_TYPE, VirtualAlloc2, VirtualFree,
25    VirtualProtect,
26};
27#[cfg(target_os = "windows")]
28use windows::core::PCSTR;
29
30use super::memory_region::{
31    HostGuestMemoryRegion, MemoryRegion, MemoryRegionFlags, MemoryRegionKind, MemoryRegionType,
32};
33use crate::log_then_return;
34
35type Result<T> = core::result::Result<T, SharedMemoryError>;
36
37/// This is just an alias for std::backtrace::Backtrace that we
38/// introduce to stop thiserror from using its backtrace
39/// functionality, which depends on nightly APIs.
40type ThisErrorHackBacktrace = std::backtrace::Backtrace;
41
42/// An error encountered while setting up or manipulating a shared memory region
43#[derive(Error, Debug)]
44pub enum SharedMemoryError {
45    /// Some operation on the shared memory attempted to read or write
46    /// out of bounds
47    #[error("Cannot access a value with size {0} at offset {1} in memory of size {2}")]
48    Bounds(usize, usize, usize),
49
50    /// An atomic access was not aligned for its atomic type.
51    #[error("Atomic access at offset {0} is not aligned to {1} bytes")]
52    AtomicAccessUnaligned(usize, usize),
53
54    /// An atomic load used an unsupported ordering.
55    #[error("Invalid atomic load ordering: {0:?}")]
56    InvalidAtomicLoadOrdering(Ordering),
57
58    /// An atomic store used an unsupported ordering.
59    #[error("Invalid atomic store ordering: {0:?}")]
60    InvalidAtomicStoreOrdering(Ordering),
61
62    /// When creating a memory with contents from a file, metadata for
63    /// that file could not be read
64    #[error("Could not access metadata for file: {0}")]
65    FileMetadata(std::io::Error),
66
67    /// When creating a memory with contents from a file, that file
68    /// was logically larger than the range of a usize.
69    #[error("File size exceeded usize: {0}")]
70    FileTooLarge(std::num::TryFromIntError),
71
72    /// The locking discipline used to enforce temporary exclusive
73    /// access by the host to a shared memory failed in some way
74    #[error("Could not acquire memory lock: {0} at {1}")]
75    LockError(String, ThisErrorHackBacktrace),
76
77    /// A request to allocate a shared memory could not be fulfilled
78    /// by the host operating system
79    #[error("Memory Allocation Failed with OS Error {0:?}.")]
80    MemoryAllocationFailed(Option<i32>),
81
82    /// A ruqest to allocate a shared shared memory had an invalid
83    /// size, due either to bounds or alignment.
84    #[error(
85        "Memory request does not satisfy constraints: 0x{1:x} < 0x{0:x} <= 0x{2:x} && 0x{0:x} % 0x{3:x} = 0"
86    )]
87    MemoryRequest(usize, usize, usize, usize),
88
89    /// An request to mmap a file or create an anonymous memory could
90    /// not be fulfilled by the host operating system
91    #[error("mmap failed with os error {0:?}")]
92    MmapFailed(Option<i32>),
93
94    /// A request to change the host-side permissions on the guard
95    /// pages of a shared memory region could not be fulfilled by the
96    /// host operating system
97    #[error("mprotect failed with os error {0:?}")]
98    MprotectFailed(Option<i32>),
99
100    /// A Windows virtual memory API call failed
101    #[cfg(target_os = "windows")]
102    #[error("Windows API Error Result {0:?}")]
103    WindowsAPIError(#[from] windows_result::Error),
104
105    /// Calling code attempted to take exclusive (write) access to a
106    /// [`ReadonlySharedMemory`].
107    #[error("Cannot take exclusive access to a ReadonlySharedMemory")]
108    ReadonlySharedMemoryExclusiveRequest,
109
110    /// An error was encountered when trying to convert a slice of raw
111    /// bytes into some logical data
112    #[error("Error reading slice {0}")]
113    TryFromSlice(#[from] std::array::TryFromSliceError),
114
115    /// An error was encountered during a routine error conversion
116    /// that should have been infallible
117    #[error("Error reading int: {0}")]
118    TryFromInt(#[from] std::num::TryFromIntError),
119}
120impl<T> From<std::sync::TryLockError<T>> for SharedMemoryError {
121    fn from(e: std::sync::TryLockError<T>) -> SharedMemoryError {
122        SharedMemoryError::LockError(format!("{:?}", e), std::backtrace::Backtrace::capture())
123    }
124}
125
126/// Makes sure that the given `offset` and `size` are within the bounds of the memory with size `mem_size`.
127macro_rules! bounds_check {
128    ($offset:expr, $size:expr, $mem_size:expr) => {
129        if $offset.checked_add($size).is_none_or(|end| end > $mem_size) {
130            return Err(SharedMemoryError::Bounds($offset, $size, $mem_size));
131        }
132    };
133}
134
135mod atomic_access {
136    pub trait Sealed {}
137}
138
139/// An integer atomic supported by [`HostSharedMemory`] atomic operations.
140///
141/// This trait is sealed and implemented for the standard signed and unsigned
142/// integer atomic types.
143#[allow(private_bounds)]
144pub trait AtomicAccess: atomic_access::Sealed {
145    /// The integer stored by this atomic type.
146    type Value: Copy;
147
148    /// Load the atomic value with `ordering`.
149    #[doc(hidden)]
150    fn load(&self, ordering: Ordering) -> Self::Value;
151
152    /// Store `value` with `ordering`.
153    #[doc(hidden)]
154    fn store(&self, value: Self::Value, ordering: Ordering);
155}
156
157macro_rules! impl_atomic_access {
158    ($atomic:ty, $value:ty) => {
159        impl atomic_access::Sealed for $atomic {}
160
161        impl AtomicAccess for $atomic {
162            type Value = $value;
163
164            fn load(&self, ordering: Ordering) -> Self::Value {
165                <$atomic>::load(self, ordering)
166            }
167
168            fn store(&self, value: Self::Value, ordering: Ordering) {
169                <$atomic>::store(self, value, ordering);
170            }
171        }
172    };
173}
174
175/// generates a reader function for the given type
176macro_rules! generate_reader {
177    ($fname:ident, $ty:ty) => {
178        /// Read a value of type `$ty` from the memory at the given offset.
179        #[allow(dead_code)]
180        #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
181        pub(crate) fn $fname(&self, offset: usize) -> Result<$ty> {
182            let data = self.as_slice();
183            bounds_check!(offset, std::mem::size_of::<$ty>(), data.len());
184            Ok(<$ty>::from_le_bytes(
185                data[offset..offset + std::mem::size_of::<$ty>()].try_into()?,
186            ))
187        }
188    };
189}
190
191/// generates a writer function for the given type
192macro_rules! generate_writer {
193    ($fname:ident, $ty:ty) => {
194        /// Write a value of type `$ty` to the memory at the given offset.
195        #[allow(dead_code)]
196        pub(crate) fn $fname(&mut self, offset: usize, value: $ty) -> Result<()> {
197            let data = self.as_mut_slice();
198            bounds_check!(offset, std::mem::size_of::<$ty>(), data.len());
199            data[offset..offset + std::mem::size_of::<$ty>()].copy_from_slice(&value.to_le_bytes());
200            Ok(())
201        }
202    };
203}
204
205impl_atomic_access!(std::sync::atomic::AtomicI8, i8);
206impl_atomic_access!(std::sync::atomic::AtomicI16, i16);
207impl_atomic_access!(std::sync::atomic::AtomicI32, i32);
208impl_atomic_access!(std::sync::atomic::AtomicI64, i64);
209impl_atomic_access!(std::sync::atomic::AtomicIsize, isize);
210impl_atomic_access!(std::sync::atomic::AtomicU8, u8);
211impl_atomic_access!(std::sync::atomic::AtomicU16, u16);
212impl_atomic_access!(std::sync::atomic::AtomicU32, u32);
213impl_atomic_access!(std::sync::atomic::AtomicU64, u64);
214impl_atomic_access!(std::sync::atomic::AtomicUsize, usize);
215
216/// A representation of a host mapping of a shared memory region,
217/// which will be released when this structure is Drop'd. This is not
218/// individually Clone (since it holds ownership of the mapping), or
219/// Send or Sync, since it doesn't ensure any particular synchronization.
220#[derive(Debug)]
221pub struct HostMapping {
222    #[cfg(not(target_os = "windows"))]
223    mmap: Mmap,
224    #[cfg(target_os = "windows")]
225    mapping: WindowsMapping,
226}
227
228/// Windows-side flavors of a [`HostMapping`].
229#[cfg(target_os = "windows")]
230#[derive(Debug)]
231enum WindowsMapping {
232    /// `[guard][blob][guard]` carved from a single anonymous file
233    /// mapping created via `CreateFileMappingA(INVALID_HANDLE_VALUE)`
234    /// and mapped with `MapViewOfFile`.
235    ///
236    /// ```text
237    /// |<------------------ view ------------------>|
238    /// [   guard   ][         blob         ][   guard   ]
239    /// ```
240    Anonymous {
241        view: MappedView,
242        file_mapping: FileMapping,
243    },
244    /// File-backed: a `VirtualAlloc2` placeholder is split in three.
245    /// The middle slot is replaced by a `MapViewOfFile3` view of the
246    /// file. The flanking placeholder slots remain unmapped and act
247    /// as the guard pages.
248    ///
249    /// ```text
250    /// [ leading ][         view         ][ trailing ]
251    /// ```
252    FileBacked {
253        leading: Placeholder,
254        view: MappedView,
255        trailing: Placeholder,
256        file_mapping: FileMapping,
257    },
258}
259
260impl HostMapping {
261    /// Base address of the host mapping, including the surrounding guard pages.
262    pub(crate) fn ptr(&self) -> *mut u8 {
263        #[cfg(not(target_os = "windows"))]
264        {
265            self.mmap.base as *mut u8
266        }
267        #[cfg(target_os = "windows")]
268        match &self.mapping {
269            WindowsMapping::Anonymous { view, .. } => view.addr as *mut u8,
270            WindowsMapping::FileBacked { leading, .. } => leading.addr as *mut u8,
271        }
272    }
273
274    /// Total size of the host mapping, including the surrounding guard pages.
275    pub(crate) fn size(&self) -> usize {
276        #[cfg(not(target_os = "windows"))]
277        {
278            self.mmap.len
279        }
280        #[cfg(target_os = "windows")]
281        match &self.mapping {
282            WindowsMapping::Anonymous { view, .. } => view.len,
283            WindowsMapping::FileBacked {
284                leading,
285                view,
286                trailing,
287                ..
288            } => leading.size + view.len + trailing.size,
289        }
290    }
291
292    /// Win32 file-mapping handle backing this mapping.
293    #[cfg(target_os = "windows")]
294    pub(crate) fn file_mapping_handle(&self) -> HANDLE {
295        match &self.mapping {
296            WindowsMapping::Anonymous { file_mapping, .. }
297            | WindowsMapping::FileBacked { file_mapping, .. } => file_mapping.0,
298        }
299    }
300}
301
302/// RAII guard for an `mmap` reservation. Calls `munmap` on drop.
303#[cfg(unix)]
304#[derive(Debug)]
305struct Mmap {
306    base: *mut c_void,
307    len: usize,
308}
309
310#[cfg(unix)]
311impl Drop for Mmap {
312    fn drop(&mut self) {
313        // SAFETY: `self.base` and `self.len` are exactly what was
314        // returned by the `mmap` that produced this `Mmap`, and that
315        // mapping has not been unmapped (we own it).
316        unsafe {
317            if libc::munmap(self.base, self.len) != 0 {
318                tracing::error!(
319                    "Mmap::drop: munmap failed: {:?}",
320                    std::io::Error::last_os_error()
321                );
322            }
323        }
324    }
325}
326
327/// RAII guard for a Win32 mapped view. Calls `UnmapViewOfFile` on drop.
328#[cfg(target_os = "windows")]
329#[derive(Debug)]
330struct MappedView {
331    addr: *mut c_void,
332    len: usize,
333}
334
335#[cfg(target_os = "windows")]
336impl Drop for MappedView {
337    fn drop(&mut self) {
338        let view = MEMORY_MAPPED_VIEW_ADDRESS { Value: self.addr };
339        // Plain `UnmapViewOfFile` fully releases the address range.
340        // `UnmapViewOfFile2(MEM_PRESERVE_PLACEHOLDER)` would convert
341        // it back into a placeholder for remapping, which is not
342        // what we want: the surrounding guard `Placeholder`s release
343        // their slots independently on drop.
344        // SAFETY: `self.addr` is the base address returned by the
345        // `MapViewOfFile` call that produced this `MappedView`, and
346        // the view has not been unmapped (we own it).
347        if let Err(e) = unsafe { UnmapViewOfFile(view) } {
348            tracing::error!(
349                "MappedView::drop(addr={:?}, len={}) UnmapViewOfFile failed: {:?}",
350                self.addr,
351                self.len,
352                e
353            );
354        }
355    }
356}
357
358/// Owns a Win32 file-mapping `HANDLE`. Calls `CloseHandle` on drop.
359#[cfg(target_os = "windows")]
360#[derive(Debug)]
361struct FileMapping(HANDLE);
362
363#[cfg(target_os = "windows")]
364impl Drop for FileMapping {
365    fn drop(&mut self) {
366        // SAFETY: `self.0` is a valid HANDLE returned by
367        // `CreateFileMappingA` that has not been closed (we own it).
368        unsafe {
369            if let Err(e) = CloseHandle(self.0) {
370                tracing::error!(
371                    "FileMapping::drop(handle={:?}) CloseHandle failed: {:?}",
372                    self.0,
373                    e
374                );
375            }
376        }
377    }
378}
379
380/// RAII guard for a `VirtualAlloc2` placeholder reservation. Owns
381/// the `[addr, addr + size)` range until split into smaller
382/// placeholders, replaced by a mapped view, or dropped (which calls
383/// `VirtualFree(MEM_RELEASE)`).
384#[cfg(target_os = "windows")]
385#[derive(Debug)]
386pub(crate) struct Placeholder {
387    addr: *mut c_void,
388    size: usize,
389}
390
391#[cfg(target_os = "windows")]
392impl Placeholder {
393    fn reserve(size: usize) -> Result<Self> {
394        // SAFETY: `VirtualAlloc2` with `MEM_RESERVE |
395        // MEM_RESERVE_PLACEHOLDER` and `PAGE_NOACCESS` only reserves
396        // address space. No pages are committed and no access is
397        // granted, so the call has no preconditions and the returned
398        // reservation cannot be misused from safe code.
399        let addr = unsafe {
400            VirtualAlloc2(
401                None,
402                None,
403                size,
404                VIRTUAL_ALLOCATION_TYPE(MEM_RESERVE.0 | MEM_RESERVE_PLACEHOLDER.0),
405                PAGE_NOACCESS.0,
406                None,
407            )
408        };
409        if addr.is_null() {
410            log_then_return!(SharedMemoryError::MemoryAllocationFailed(
411                Error::last_os_error().raw_os_error()
412            ));
413        }
414        Ok(Placeholder { addr, size })
415    }
416
417    fn split_front(self, front_size: usize) -> Result<(Placeholder, Placeholder)> {
418        debug_assert!(front_size > 0 && front_size < self.size);
419        debug_assert!(front_size.is_multiple_of(page_size::get()));
420        // SAFETY: `self` owns the placeholder reservation at
421        // `[self.addr, self.addr + self.size)`. `MEM_RELEASE |
422        // MEM_PRESERVE_PLACEHOLDER` is the Win32 idiom for splitting
423        // a placeholder in two: no memory is released, the
424        // reservation is just carved at `front_size`.
425        if let Err(e) = unsafe {
426            VirtualFree(
427                self.addr,
428                front_size,
429                VIRTUAL_FREE_TYPE(MEM_RELEASE.0 | MEM_PRESERVE_PLACEHOLDER.0),
430            )
431        } {
432            // `self` drops here, releasing the unsplit reservation.
433            log_then_return!(SharedMemoryError::WindowsAPIError(e.clone()));
434        }
435        let addr = self.addr;
436        let total = self.size;
437        // Forget the parent so its `Drop` does not release the two
438        // child slots as one.
439        std::mem::forget(self);
440        let front = Placeholder {
441            addr,
442            size: front_size,
443        };
444        let back = Placeholder {
445            // SAFETY: `front_size < total`, so `addr + front_size`
446            // is in-bounds of the original reservation.
447            addr: unsafe { (addr as *mut u8).add(front_size) as *mut c_void },
448            size: total - front_size,
449        };
450        Ok((front, back))
451    }
452
453    fn split_into_three(
454        self,
455        front_size: usize,
456        middle_size: usize,
457    ) -> Result<(Placeholder, Placeholder, Placeholder)> {
458        let (front, rest) = self.split_front(front_size)?;
459        let (middle, back) = rest.split_front(middle_size)?;
460        Ok((front, middle, back))
461    }
462
463    fn map_file_view(self, file_mapping: HANDLE) -> Result<MappedView> {
464        // SAFETY: `self` owns the placeholder slot at
465        // `[self.addr, self.addr + self.size)`.
466        // `MEM_REPLACE_PLACEHOLDER` requires the target range to be
467        // an existing placeholder of exactly that size, which the
468        // type system guarantees here. On success, ownership of the
469        // range transfers to the returned `MappedView`; the caller
470        // releases the file mapping handle via `FileMapping`.
471        let mapped = unsafe {
472            MapViewOfFile3(
473                file_mapping,
474                None,
475                Some(self.addr),
476                0,
477                self.size,
478                MEM_REPLACE_PLACEHOLDER,
479                PAGE_READONLY.0,
480                None,
481            )
482        };
483        if mapped.Value.is_null() {
484            // `self` drops here, releasing the placeholder.
485            log_then_return!(SharedMemoryError::MemoryAllocationFailed(
486                Error::last_os_error().raw_os_error()
487            ));
488        }
489        let addr = self.addr;
490        let len = self.size;
491        std::mem::forget(self);
492        Ok(MappedView { addr, len })
493    }
494}
495
496#[cfg(target_os = "windows")]
497impl Drop for Placeholder {
498    fn drop(&mut self) {
499        // SAFETY: `self.addr` is the base of a placeholder
500        // reservation we own. `MEM_RELEASE` with size 0 releases the
501        // entire reservation.
502        if let Err(e) = unsafe { VirtualFree(self.addr, 0, VIRTUAL_FREE_TYPE(MEM_RELEASE.0)) } {
503            tracing::error!(
504                "Placeholder::drop(addr={:?}, size={}) VirtualFree failed: {:?}",
505                self.addr,
506                self.size,
507                e
508            );
509        }
510    }
511}
512
513/// A trait that abstracts over the particular kind of SharedMemory,
514/// used when invoking operations from Rust that absolutely must have
515/// exclusive control over the shared memory for correctness +
516/// performance, like snapshotting.
517pub trait SharedMemory {
518    /// Return a readonly reference to the host mapping backing this SharedMemory
519    fn region(&self) -> &HostMapping;
520
521    /// Return the base address of the host mapping of this
522    /// region. Following the general Rust philosophy, this does not
523    /// need to be marked as `unsafe` because doing anything with this
524    /// pointer itself requires `unsafe`.
525    fn base_addr(&self) -> usize {
526        self.region().ptr() as usize + page_size::get()
527    }
528
529    /// Return the base address of the host mapping of this region as
530    /// a pointer. Following the general Rust philosophy, this does
531    /// not need to be marked as `unsafe` because doing anything with
532    /// this pointer itself requires `unsafe`.
533    fn base_ptr(&self) -> *mut u8 {
534        self.region().ptr().wrapping_add(page_size::get())
535    }
536
537    /// Return the length of usable memory contained in `self`.
538    /// The returned size does not include the size of the surrounding
539    /// guard pages.
540    fn mem_size(&self) -> usize {
541        self.region().size() - 2 * page_size::get()
542    }
543
544    /// Return the raw base address of the host mapping, including the
545    /// guard pages.
546    fn raw_ptr(&self) -> *mut u8 {
547        self.region().ptr()
548    }
549
550    /// Return the raw size of the host mapping, including the guard
551    /// pages.
552    fn raw_mem_size(&self) -> usize {
553        self.region().size()
554    }
555
556    /// Extract a base address that can be mapped into a VM for this
557    /// SharedMemory.
558    ///
559    /// On Linux this returns a raw `usize` pointer. On Windows it
560    /// returns a [`HostRegionBase`](super::memory_region::HostRegionBase)
561    /// that carries the file-mapping handle metadata needed by WHP.
562    fn host_region_base(&self) -> <HostGuestMemoryRegion as MemoryRegionKind>::HostBaseType {
563        #[cfg(not(windows))]
564        {
565            self.base_addr()
566        }
567        #[cfg(windows)]
568        {
569            super::memory_region::HostRegionBase {
570                from_handle: self.region().file_mapping_handle().into(),
571                handle_base: self.region().ptr() as usize,
572                handle_size: self.region().size(),
573                offset: page_size::get(),
574            }
575        }
576    }
577
578    /// Return the end address of the host region (base + usable size).
579    fn host_region_end(&self) -> <HostGuestMemoryRegion as MemoryRegionKind>::HostBaseType {
580        <HostGuestMemoryRegion as MemoryRegionKind>::add(self.host_region_base(), self.mem_size())
581    }
582
583    /// Run some code with exclusive access to the SharedMemory
584    /// underlying this.  If the SharedMemory is not an
585    /// ExclusiveSharedMemory, any concurrent accesses to the relevant
586    /// HostSharedMemory/GuestSharedMemory may make this fail, or be
587    /// made to fail by this, and should be avoided.
588    fn with_exclusivity<T, F: FnOnce(&mut ExclusiveSharedMemory) -> T>(
589        &mut self,
590        f: F,
591    ) -> Result<T>;
592
593    /// Run some code that is allowed to access the contents of the
594    /// SharedMemory as if it is a normal slice.  By default, this is
595    /// implemented via [`SharedMemory::with_exclusivity`], which is
596    /// the correct implementation for a memory that can be mutated,
597    /// but a [`ReadonlySharedMemory`], can support this.
598    fn with_contents<T, F: FnOnce(&[u8]) -> T>(&mut self, f: F) -> Result<T> {
599        self.with_exclusivity(|m| f(m.as_slice()))
600    }
601}
602
603fn mapping_at(
604    s: &impl SharedMemory,
605    gpa: u64,
606    size: usize,
607    region_type: MemoryRegionType,
608    flags: MemoryRegionFlags,
609) -> MemoryRegion {
610    let guest_base = gpa as usize;
611
612    MemoryRegion {
613        guest_region: guest_base..(guest_base + size),
614        host_region: s.host_region_base()
615            ..<HostGuestMemoryRegion as MemoryRegionKind>::add(s.host_region_base(), size),
616        region_type,
617        flags,
618    }
619}
620
621/// These three structures represent various phases of the lifecycle of
622/// a memory buffer that is shared with the guest. An
623/// ExclusiveSharedMemory is used for certain operations that
624/// unrestrictedly write to the shared memory, including setting it up
625/// and taking snapshots.
626#[derive(Debug)]
627pub struct ExclusiveSharedMemory {
628    region: Arc<HostMapping>,
629}
630unsafe impl Send for ExclusiveSharedMemory {}
631
632impl ExclusiveSharedMemory {
633    /// Helper function used to abstract common checks from Windows
634    /// and Linux implementations of [`ExclusiveSharedMemory::new()`]
635    fn total_size(min_size_bytes: usize) -> Result<usize> {
636        if min_size_bytes > 0 &&
637            // guard page around the memory
638            let Some(total_size) = min_size_bytes.checked_add(2 * page_size::get()) &&
639            total_size % page_size::get() == 0 &&
640            // usize and isize are guaranteed to be the same size, and
641            // isize::MAX should be positive, so this cast should be
642            // safe.
643            total_size <= isize::MAX as usize
644        {
645            Ok(total_size)
646        } else {
647            Err(SharedMemoryError::MemoryRequest(
648                min_size_bytes,
649                2,
650                isize::MAX as usize - 2 * page_size::get(),
651                page_size::get(),
652            ))
653        }
654    }
655
656    /// Create a new region of shared memory with the given minimum
657    /// size in bytes. The region will be surrounded by guard pages.
658    ///
659    /// Return `Err` if shared memory could not be allocated.
660    #[cfg(unix)]
661    #[instrument(skip_all, parent = Span::current(), level= "Trace")]
662    pub fn new(min_size_bytes: usize) -> Result<Self> {
663        use libc::{
664            MAP_ANONYMOUS, MAP_FAILED, MAP_PRIVATE, PROT_READ, PROT_WRITE, c_int, mmap, off_t,
665            size_t,
666        };
667
668        let total_size = Self::total_size(min_size_bytes)?;
669
670        #[cfg(not(miri))]
671        use libc::{MAP_NORESERVE, PROT_NONE, mprotect};
672
673        // allocate the memory
674        #[cfg(not(miri))]
675        let flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE;
676        #[cfg(miri)]
677        let flags = MAP_ANONYMOUS | MAP_PRIVATE;
678
679        let addr = unsafe {
680            mmap(
681                null_mut(),
682                total_size as size_t,
683                PROT_READ | PROT_WRITE,
684                flags,
685                -1 as c_int,
686                0 as off_t,
687            )
688        };
689        if addr == MAP_FAILED {
690            log_then_return!(SharedMemoryError::MmapFailed(
691                Error::last_os_error().raw_os_error()
692            ));
693        }
694        let mmap = Mmap {
695            base: addr,
696            len: total_size,
697        };
698
699        // protect the guard pages
700        #[cfg(not(miri))]
701        {
702            let res = unsafe { mprotect(mmap.base, page_size::get(), PROT_NONE) };
703            if res != 0 {
704                return Err(SharedMemoryError::MprotectFailed(
705                    Error::last_os_error().raw_os_error(),
706                ));
707            }
708            let res = unsafe {
709                mprotect(
710                    (mmap.base as *const u8).add(total_size - page_size::get()) as *mut c_void,
711                    page_size::get(),
712                    PROT_NONE,
713                )
714            };
715            if res != 0 {
716                return Err(SharedMemoryError::MprotectFailed(
717                    Error::last_os_error().raw_os_error(),
718                ));
719            }
720        }
721
722        Ok(Self {
723            // HostMapping is only non-Send/Sync because raw pointers
724            // are not ("as a lint", as the Rust docs say). We don't
725            // want to mark HostMapping Send/Sync immediately, because
726            // that could socially imply that it's "safe" to use
727            // unsafe accesses from multiple threads at once. Instead, we
728            // directly impl Send and Sync on this type. Since this
729            // type does have Send and Sync manually impl'd, the Arc
730            // is not pointless as the lint suggests.
731            #[allow(clippy::arc_with_non_send_sync)]
732            region: Arc::new(HostMapping { mmap }),
733        })
734    }
735
736    /// Create a new region of shared memory with the given minimum
737    /// size in bytes. The region will be surrounded by guard pages.
738    ///
739    /// Return `Err` if shared memory could not be allocated.
740    #[cfg(target_os = "windows")]
741    #[instrument(skip_all, parent = Span::current(), level= "Trace")]
742    pub fn new(min_size_bytes: usize) -> Result<Self> {
743        let total_size = Self::total_size(min_size_bytes)?;
744
745        let mut dwmaximumsizehigh = 0;
746        let mut dwmaximumsizelow = 0;
747
748        if std::mem::size_of::<usize>() == 8 {
749            dwmaximumsizehigh = (total_size >> 32) as u32;
750            dwmaximumsizelow = (total_size & 0xFFFFFFFF) as u32;
751        }
752
753        // Allocate the memory use CreateFileMapping instead of VirtualAlloc
754        // This allows us to map the memory into the surrogate process using MapViewOfFile2
755
756        let flags = PAGE_READWRITE;
757
758        let handle = unsafe {
759            CreateFileMappingA(
760                INVALID_HANDLE_VALUE,
761                None,
762                flags,
763                dwmaximumsizehigh,
764                dwmaximumsizelow,
765                PCSTR::null(),
766            )?
767        };
768
769        if handle.is_invalid() {
770            log_then_return!(SharedMemoryError::MemoryAllocationFailed(
771                Error::last_os_error().raw_os_error()
772            ));
773        }
774        let file_mapping = FileMapping(handle);
775
776        let file_map = FILE_MAP_ALL_ACCESS;
777        let addr = unsafe { MapViewOfFile(file_mapping.0, file_map, 0, 0, 0) };
778
779        if addr.Value.is_null() {
780            log_then_return!(SharedMemoryError::MemoryAllocationFailed(
781                Error::last_os_error().raw_os_error()
782            ));
783        }
784        let view = MappedView {
785            addr: addr.Value,
786            len: total_size,
787        };
788
789        // Set the first and last pages to be guard pages
790
791        let mut unused_out_old_prot_flags = PAGE_PROTECTION_FLAGS(0);
792
793        // If the following calls to VirtualProtect are changed make sure to update the calls to VirtualProtectEx in surrogate_process_manager.rs
794
795        let first_guard_page_start = view.addr;
796        if let Err(e) = unsafe {
797            VirtualProtect(
798                first_guard_page_start,
799                page_size::get(),
800                PAGE_NOACCESS,
801                &mut unused_out_old_prot_flags,
802            )
803        } {
804            log_then_return!(SharedMemoryError::WindowsAPIError(e.clone()));
805        }
806
807        let last_guard_page_start = unsafe { view.addr.add(total_size - page_size::get()) };
808        if let Err(e) = unsafe {
809            VirtualProtect(
810                last_guard_page_start,
811                page_size::get(),
812                PAGE_NOACCESS,
813                &mut unused_out_old_prot_flags,
814            )
815        } {
816            log_then_return!(SharedMemoryError::WindowsAPIError(e.clone()));
817        }
818
819        Ok(Self {
820            // HostMapping is only non-Send/Sync because raw pointers
821            // are not ("as a lint", as the Rust docs say). We don't
822            // want to mark HostMapping Send/Sync immediately, because
823            // that could socially imply that it's "safe" to use
824            // unsafe accesses from multiple threads at once. Instead, we
825            // directly impl Send and Sync on this type. Since this
826            // type does have Send and Sync manually impl'd, the Arc
827            // is not pointless as the lint suggests.
828            #[allow(clippy::arc_with_non_send_sync)]
829            region: Arc::new(HostMapping {
830                mapping: WindowsMapping::Anonymous { view, file_mapping },
831            }),
832        })
833    }
834
835    /// Internal helper method to get the backing memory as a mutable slice.
836    ///
837    /// # Safety
838    /// As per std::slice::from_raw_parts_mut:
839    /// - self.base_addr() must be valid for both reads and writes for
840    ///   self.mem_size() * mem::size_of::<u8>() many bytes, and it
841    ///   must be properly aligned.
842    ///
843    ///   The rules on validity are still somewhat unspecified, but we
844    ///   assume that the result of our calls to mmap/CreateFileMappings may
845    ///   be considered a single "allocated object". The use of
846    ///   non-atomic accesses is alright from a Safe Rust standpoint,
847    ///   because SharedMemoryBuilder is  not Sync.
848    /// - self.base_addr() must point to self.mem_size() consecutive
849    ///   properly initialized values of type u8
850    ///
851    ///   Again, the exact provenance restrictions on what is
852    ///   considered to be initialized values are unclear, but we make
853    ///   sure to use mmap(MAP_ANONYMOUS) and
854    ///   CreateFileMapping(SEC_COMMIT), so the pages in question are
855    ///   zero-initialized, which we hope counts for u8.
856    /// - The memory referenced by the returned slice must not be
857    ///   accessed through any other pointer (not derived from the
858    ///   return value) for the duration of the lifetime 'a. Both read
859    ///   and write accesses are forbidden.
860    ///
861    ///   Accesses from Safe Rust necessarily follow this rule,
862    ///   because the returned slice's lifetime is the same as that of
863    ///   a mutable borrow of self.
864    /// - The total size self.mem_size() * mem::size_of::<u8>() of the
865    ///   slice must be no larger than isize::MAX, and adding that
866    ///   size to data must not "wrap around" the address space. See
867    ///   the safety documentation of pointer::offset.
868    ///
869    ///   This is ensured by a check in ::new()
870    pub(super) fn as_mut_slice(&mut self) -> &mut [u8] {
871        unsafe { std::slice::from_raw_parts_mut(self.base_ptr(), self.mem_size()) }
872    }
873
874    /// Internal helper method to get the backing memory as a slice.
875    ///
876    /// # Safety
877    /// See the discussion on as_mut_slice, with the third point
878    /// replaced by:
879    /// - The memory referenced by the returned slice must not be
880    ///   mutated for the duration of lifetime 'a, except inside an
881    ///   UnsafeCell.
882    ///
883    ///   Host accesses from Safe Rust necessarily follow this rule,
884    ///   because the returned slice's lifetime is the same as that of
885    ///   a borrow of self, preventing mutations via other methods.
886    #[instrument(skip_all, parent = Span::current(), level= "Trace")]
887    pub fn as_slice<'a>(&'a self) -> &'a [u8] {
888        unsafe { std::slice::from_raw_parts(self.base_ptr(), self.mem_size()) }
889    }
890
891    /// Copy the entire contents of `self` into a `Vec<u8>`, then return it
892    #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
893    #[cfg(test)]
894    pub(crate) fn copy_all_to_vec(&self) -> Result<Vec<u8>> {
895        let data = self.as_slice();
896        Ok(data.to_vec())
897    }
898
899    /// Copies all bytes from `src` to `self` starting at offset
900    #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
901    pub fn copy_from_slice(&mut self, src: &[u8], offset: usize) -> Result<()> {
902        let data = self.as_mut_slice();
903        bounds_check!(offset, src.len(), data.len());
904        data[offset..offset + src.len()].copy_from_slice(src);
905        Ok(())
906    }
907
908    generate_reader!(read_u8, u8);
909    generate_reader!(read_i8, i8);
910    generate_reader!(read_u16, u16);
911    generate_reader!(read_i16, i16);
912    generate_reader!(read_u32, u32);
913    generate_reader!(read_i32, i32);
914    generate_reader!(read_u64, u64);
915    generate_reader!(read_i64, i64);
916    generate_reader!(read_usize, usize);
917    generate_reader!(read_isize, isize);
918
919    generate_writer!(write_u8, u8);
920    generate_writer!(write_i8, i8);
921    generate_writer!(write_u16, u16);
922    generate_writer!(write_i16, i16);
923    generate_writer!(write_u32, u32);
924    generate_writer!(write_i32, i32);
925    generate_writer!(write_u64, u64);
926    generate_writer!(write_i64, i64);
927    generate_writer!(write_usize, usize);
928    generate_writer!(write_isize, isize);
929
930    /// Convert the ExclusiveSharedMemory, which may be freely
931    /// modified, into a GuestSharedMemory, which may be somewhat
932    /// freely modified (mostly by the guest), and a HostSharedMemory,
933    /// which may only make certain kinds of accesses that do not race
934    /// in the presence of malicious code inside the guest mutating
935    /// the GuestSharedMemory.
936    pub fn build(self) -> (HostSharedMemory, GuestSharedMemory) {
937        let lock = Arc::new(RwLock::new(()));
938        let hshm = HostSharedMemory {
939            region: self.region.clone(),
940            lock: lock.clone(),
941        };
942        (
943            hshm,
944            GuestSharedMemory {
945                region: self.region.clone(),
946                lock,
947            },
948        )
949    }
950
951    /// Gets the file handle of the shared memory region for this Sandbox
952    #[cfg(target_os = "windows")]
953    pub fn get_mmap_file_handle(&self) -> HANDLE {
954        self.region.file_mapping_handle()
955    }
956}
957
958impl SharedMemory for ExclusiveSharedMemory {
959    fn region(&self) -> &HostMapping {
960        &self.region
961    }
962    fn with_exclusivity<T, F: FnOnce(&mut ExclusiveSharedMemory) -> T>(
963        &mut self,
964        f: F,
965    ) -> Result<T> {
966        Ok(f(self))
967    }
968}
969
970/// A GuestSharedMemory is used to represent
971/// the reference to all-of-memory that is taken by the virtual cpu.
972/// Because of the memory model limitations that affect
973/// HostSharedMemory, it is likely fairly important (to ensure that
974/// our UB remains limited to interaction with an external compilation
975/// unit that likely can't be discovered by the compiler) that _rust_
976/// users do not perform racy accesses to the guest communication
977/// buffers that are also accessed by HostSharedMemory.
978#[derive(Debug)]
979pub struct GuestSharedMemory {
980    region: Arc<HostMapping>,
981    /// The lock that indicates this shared memory is being used by non-Rust code
982    ///
983    /// This lock _must_ be held whenever the guest is executing,
984    /// because it prevents the host from converting its
985    /// HostSharedMemory to an ExclusiveSharedMemory. Since the guest
986    /// may arbitrarily mutate the shared memory, only synchronized
987    /// accesses from Rust should be allowed!
988    ///
989    /// We cannot enforce this in the type system, because the memory
990    /// is mapped in to the VM at VM creation time.
991    pub lock: Arc<RwLock<()>>,
992}
993unsafe impl Send for GuestSharedMemory {}
994
995impl GuestSharedMemory {
996    /// Create a [`super::memory_region::MemoryRegion`] structure
997    /// suitable for mapping this region into a VM
998    pub(crate) fn mapping_at(
999        &self,
1000        guest_base: u64,
1001        region_type: MemoryRegionType,
1002    ) -> MemoryRegion {
1003        let flags = match region_type {
1004            MemoryRegionType::Scratch => {
1005                MemoryRegionFlags::READ | MemoryRegionFlags::WRITE | MemoryRegionFlags::EXECUTE
1006            }
1007            #[cfg(unshared_snapshot_mem)]
1008            MemoryRegionType::Snapshot => {
1009                MemoryRegionFlags::READ | MemoryRegionFlags::WRITE | MemoryRegionFlags::EXECUTE
1010            }
1011            #[allow(clippy::panic)]
1012            // This will not ever actually panic: the only places this
1013            // is called are HyperlightVm::update_snapshot_mapping and
1014            // HyperlightVm::update_scratch_mapping. The latter
1015            // statically uses the Scratch region type, and the former
1016            // does not use this at all when the unshared_snapshot_mem
1017            // feature is not set, since in that case the scratch
1018            // mapping type is ReadonlySharedMemory, not
1019            // GuestSharedMemory.
1020            _ => panic!(
1021                "GuestSharedMemory::mapping_at should only be used for Scratch or Snapshot regions"
1022            ),
1023        };
1024        mapping_at(self, guest_base, self.mem_size(), region_type, flags)
1025    }
1026}
1027
1028impl SharedMemory for GuestSharedMemory {
1029    fn region(&self) -> &HostMapping {
1030        &self.region
1031    }
1032    fn with_exclusivity<T, F: FnOnce(&mut ExclusiveSharedMemory) -> T>(
1033        &mut self,
1034        f: F,
1035    ) -> Result<T> {
1036        let guard = self.lock.try_write()?;
1037        let mut excl = ExclusiveSharedMemory {
1038            region: self.region.clone(),
1039        };
1040        let ret = f(&mut excl);
1041        drop(excl);
1042        drop(guard);
1043        Ok(ret)
1044    }
1045}
1046
1047/// A HostSharedMemory allows synchronized accesses to guest
1048/// communication buffers, allowing it to be used concurrently with a
1049/// GuestSharedMemory.
1050///
1051/// # Concurrency model
1052///
1053/// Given future requirements for asynchronous I/O with a minimum
1054/// amount of copying (e.g. WASIp3 streams), we would like it to be
1055/// possible to safely access these buffers concurrently with the
1056/// guest, ensuring that (1) data is read appropriately if the guest
1057/// is well-behaved; and (2) the host's behaviour is defined
1058/// regardless of whether or not the guest is well-behaved.
1059///
1060/// The ideal (future) flow for a guest->host message is something like
1061///   - Guest writes (unordered) bytes describing a work item into a buffer
1062///   - Guest reveals buffer via a release-store of a pointer into an
1063///     MMIO ring-buffer
1064///   - Host acquire-loads the buffer pointer from the "MMIO" ring
1065///     buffer
1066///   - Host (unordered) reads the bytes from the buffer
1067///   - Host performs validation of those bytes and uses them
1068///
1069/// Unfortunately, there appears to be no way to do this with defined
1070/// behaviour in present Rust (see
1071/// e.g. <https://github.com/rust-lang/unsafe-code-guidelines/issues/152>).
1072/// Rust does not yet have its own defined memory model, but in the
1073/// interim, it is widely treated as inheriting the current C/C++
1074/// memory models.  The most immediate problem is that regardless of
1075/// anything else, under those memory models \[1, p. 17-18; 2, p. 88\],
1076///
1077///   > The execution of a program contains a _data race_ if it
1078///   > contains two [C++23: "potentially concurrent"] conflicting
1079///   > actions [C23: "in different threads"], at least one of which
1080///   > is not atomic, and neither happens before the other [C++23: ",
1081///   > except for the special case for signal handlers described
1082///   > below"].  Any such data race results in undefined behavior.
1083///
1084/// Consequently, if a misbehaving guest fails to correctly
1085/// synchronize its stores with the host, the host's innocent loads
1086/// will trigger undefined behaviour for the entire program, including
1087/// the host.  Note that this also applies if the guest makes an
1088/// unsynchronized read of a location that the host is writing!
1089///
1090/// Despite Rust's de jure inheritance of the C memory model at the
1091/// present time, the compiler in many cases de facto adheres to LLVM
1092/// semantics, so it is worthwhile to consider what LLVM does in this
1093/// case as well.  According to the the LangRef \[3\] memory model,
1094/// loads which are involved in a race that includes at least one
1095/// non-atomic access (whether the load or a store) return `undef`,
1096/// making them roughly equivalent to reading uninitialized
1097/// memory. While this is much better, it is still bad.
1098///
1099/// Considering a different direction, recent C++ papers have seemed
1100/// to lean towards using `volatile` for similar use cases. For
1101/// example, in P1152R0 \[4\], JF Bastien notes that
1102///
1103///   > We’ve shown that volatile is purposely defined to denote
1104///   > external modifications. This happens for:
1105///   >   - Shared memory with untrusted code, where volatile is the
1106///   >     right way to avoid time-of-check time-of-use (ToCToU)
1107///   >     races which lead to security bugs such as \[PWN2OWN\] and
1108///   >     \[XENXSA155\].
1109///
1110/// Unfortunately, although this paper was adopted for C++20 (and,
1111/// sadly, mostly un-adopted for C++23, although that does not concern
1112/// us), the paper did not actually redefine volatile accesses or data
1113/// races to prevent volatile accesses from racing with other accesses
1114/// and causing undefined behaviour.  P1382R1 \[5\] would have amended
1115/// the wording of the data race definition to specifically exclude
1116/// volatile, but, unfortunately, despite receiving a
1117/// generally-positive reception at its first WG21 meeting more than
1118/// five years ago, it has not progressed.
1119///
1120/// Separately from the data race issue, there is also a concern that
1121/// according to the various memory models in use, there may be ways
1122/// in which the guest can semantically obtain uninitialized memory
1123/// and write it into the shared buffer, which may also result in
1124/// undefined behaviour on reads.  The degree to which this is a
1125/// concern is unclear, however, since it is unclear to what degree
1126/// the Rust abstract machine's conception of uninitialized memory
1127/// applies to the sandbox.  Returning briefly to the LLVM level,
1128/// rather than the Rust level, this, combined with the fact that
1129/// racing loads in LLVM return `undef`, as discussed above, we would
1130/// ideally `llvm.freeze` the result of any load out of the sandbox.
1131///
1132/// It would furthermore be ideal if we could run the flatbuffers
1133/// parsing code directly on the guest memory, in order to avoid
1134/// unnecessary copies.  That is unfortunately probably not viable at
1135/// the present time: because the generated flatbuffers parsing code
1136/// doesn't use atomic or volatile accesses, it is likely to introduce
1137/// double-read vulnerabilities.
1138///
1139/// In short, none of the Rust-level operations available to us do the
1140/// right thing, at the Rust spec level or the LLVM spec level. Our
1141/// major remaining options are therefore:
1142///   - Choose one of the options that is available to us, and accept
1143///     that we are doing something unsound according to the spec, but
1144///     hope that no reasonable compiler could possibly notice.
1145///   - Use inline assembly per architecture, for which we would only
1146///     need to worry about the _architecture_'s memory model (which
1147///     is far less demanding).
1148///
1149/// The leading candidate for the first option would seem to be to
1150/// simply use volatile accesses; there seems to be wide agreement
1151/// that this _should_ be a valid use case for them (even if it isn't
1152/// now), and projects like Linux and rust-vmm already use C11
1153/// `volatile` for this purpose.  It is also worth noting that because
1154/// we still do need to synchronize with the guest when it _is_ being
1155/// well-behaved, we would ideally use volatile acquire loads and
1156/// volatile release stores for interacting with the stack pointer in
1157/// the guest in this case.  Unfortunately, while those operations are
1158/// defined in LLVM, they are not presently exposed to Rust. While
1159/// atomic fences that are not associated with memory accesses
1160/// ([`std::sync::atomic::fence`]) might at first glance seem to help with
1161/// this problem, they unfortunately do not \[6\]:
1162///
1163///    > A fence ‘A’ which has (at least) Release ordering semantics,
1164///    > synchronizes with a fence ‘B’ with (at least) Acquire
1165///    > semantics, if and only if there exist operations X and Y,
1166///    > both operating on some atomic object ‘M’ such that A is
1167///    > sequenced before X, Y is sequenced before B and Y observes
1168///    > the change to M. This provides a happens-before dependence
1169///    > between A and B.
1170///
1171/// Note that the X and Y must be to an _atomic_ object.
1172///
1173/// We consequently assume that there has been a strong architectural
1174/// fence on a vmenter/vmexit between data being read and written.
1175/// This is unsafe (not guaranteed in the type system)!
1176///
1177/// \[1\] N3047 C23 Working Draft. <https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3047.pdf>
1178/// \[2\] N4950 C++23 Working Draft. <https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2023/n4950.pdf>
1179/// \[3\] LLVM Language Reference Manual, Memory Model for Concurrent Operations. <https://llvm.org/docs/LangRef.html#memmodel>
1180/// \[4\] P1152R0: Deprecating `volatile`. JF Bastien. <https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2018/p1152r0.html>
1181/// \[5\] P1382R1: `volatile_load<T>` and `volatile_store<T>`. JF Bastien, Paul McKenney, Jeffrey Yasskin, and the indefatigable TBD. <https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2019/p1382r1.pdf>
1182/// \[6\] Documentation for std::sync::atomic::fence. <https://doc.rust-lang.org/std/sync/atomic/fn.fence.html>
1183///
1184/// # Note \[Keeping mappings in sync between userspace and the guest\]
1185///
1186/// When using this structure with mshv on Linux, it is necessary to
1187/// be a little bit careful: since the hypervisor is not directly
1188/// integrated with the host kernel virtual memory subsystem, it is
1189/// easy for the memory region in userspace to get out of sync with
1190/// the memory region mapped into the guest.  Generally speaking, when
1191/// the [`SharedMemory`] is mapped into a partition, the MSHV kernel
1192/// module will call `pin_user_pages(FOLL_PIN|FOLL_WRITE)` on it,
1193/// which will eagerly do any CoW, etc needing to obtain backing pages
1194/// pinned in memory, and then map precisely those backing pages into
1195/// the virtual machine. After that, the backing pages mapped into the
1196/// VM will not change until the region is unmapped or remapped.  This
1197/// means that code in this module needs to be very careful to avoid
1198/// changing the backing pages of the region in the host userspace,
1199/// since that would result in hyperlight-host's view of the memory
1200/// becoming completely divorced from the view of the VM.
1201#[derive(Clone, Debug)]
1202pub struct HostSharedMemory {
1203    region: Arc<HostMapping>,
1204    lock: Arc<RwLock<()>>,
1205}
1206unsafe impl Send for HostSharedMemory {}
1207
1208impl HostSharedMemory {
1209    /// Read a [`Pod`] value of type `T`, whose representation is the same
1210    /// between the sandbox and the host.
1211    pub fn read<T: Pod>(&self, offset: usize) -> Result<T> {
1212        bounds_check!(offset, std::mem::size_of::<T>(), self.mem_size());
1213        let mut ret = T::zeroed();
1214        self.copy_to_slice(bytemuck::bytes_of_mut(&mut ret), offset)?;
1215        Ok(ret)
1216    }
1217
1218    /// Write a [`Pod`] value of type `T`, whose representation is the same
1219    /// between the sandbox and the host.
1220    pub fn write<T: Pod>(&self, offset: usize, data: T) -> Result<()> {
1221        bounds_check!(offset, std::mem::size_of::<T>(), self.mem_size());
1222        self.copy_from_slice(bytemuck::bytes_of(&data), offset)
1223    }
1224
1225    /// Load an integer atomic at `offset` with `ordering`.
1226    pub fn load_atomic<A: AtomicAccess>(
1227        &self,
1228        offset: usize,
1229        ordering: Ordering,
1230    ) -> Result<A::Value> {
1231        if matches!(ordering, Ordering::Release | Ordering::AcqRel) {
1232            return Err(SharedMemoryError::InvalidAtomicLoadOrdering(ordering));
1233        }
1234
1235        bounds_check!(offset, size_of::<A>(), self.mem_size());
1236        let ptr = self.base_ptr().wrapping_add(offset);
1237        if !(ptr as usize).is_multiple_of(align_of::<A>()) {
1238            return Err(SharedMemoryError::AtomicAccessUnaligned(
1239                offset,
1240                align_of::<A>(),
1241            ));
1242        }
1243
1244        let _guard = self.lock.try_read()?;
1245
1246        // SAFETY: The bounds and alignment checks cover an A within the mapping.
1247        // AtomicAccess is sealed to integer atomics, whose bit patterns are valid.
1248        let atomic = unsafe { &*ptr.cast::<A>() };
1249        Ok(atomic.load(ordering))
1250    }
1251
1252    /// Store an integer atomic at `offset` with `ordering`.
1253    pub fn store_atomic<A: AtomicAccess>(
1254        &self,
1255        offset: usize,
1256        value: A::Value,
1257        ordering: Ordering,
1258    ) -> Result<()> {
1259        if matches!(ordering, Ordering::Acquire | Ordering::AcqRel) {
1260            return Err(SharedMemoryError::InvalidAtomicStoreOrdering(ordering));
1261        }
1262
1263        bounds_check!(offset, size_of::<A>(), self.mem_size());
1264        let ptr = self.base_ptr().wrapping_add(offset);
1265        if !(ptr as usize).is_multiple_of(align_of::<A>()) {
1266            return Err(SharedMemoryError::AtomicAccessUnaligned(
1267                offset,
1268                align_of::<A>(),
1269            ));
1270        }
1271
1272        let _guard = self.lock.try_read()?;
1273
1274        // SAFETY: The bounds and alignment checks cover an A within the mapping.
1275        // AtomicAccess is sealed to integer atomics, whose bit patterns are valid.
1276        let atomic = unsafe { &*ptr.cast::<A>() };
1277        atomic.store(value, ordering);
1278        Ok(())
1279    }
1280
1281    /// Copy the contents of the slice into the sandbox at the
1282    /// specified offset
1283    pub fn copy_to_slice(&self, slice: &mut [u8], offset: usize) -> Result<()> {
1284        bounds_check!(offset, slice.len(), self.mem_size());
1285        let base = self.base_ptr().wrapping_add(offset);
1286        let guard = self.lock.try_read()?;
1287
1288        const CHUNK: usize = size_of::<u128>();
1289        let len = slice.len();
1290        let mut i = 0;
1291
1292        // Handle unaligned head bytes until we reach u128 alignment.
1293        // Note: align_offset can return usize::MAX if alignment is impossible.
1294        // In that case, head_len = len via .min(), so we fall back to byte-by-byte
1295        // operations for the entire slice.
1296        let align_offset = base.align_offset(align_of::<u128>());
1297        let head_len = align_offset.min(len);
1298        while i < head_len {
1299            unsafe {
1300                slice[i] = base.add(i).read_volatile();
1301            }
1302            i += 1;
1303        }
1304
1305        // Read aligned u128 chunks
1306        // SAFETY: After processing head_len bytes, base.add(i) is u128-aligned.
1307        // We use write_unaligned for the destination since the slice may not be u128-aligned.
1308        let dst = slice.as_mut_ptr();
1309        while i + CHUNK <= len {
1310            unsafe {
1311                let value = (base.add(i) as *const u128).read_volatile();
1312                std::ptr::write_unaligned(dst.add(i) as *mut u128, value);
1313            }
1314            i += CHUNK;
1315        }
1316
1317        // Handle remaining tail bytes
1318        while i < len {
1319            unsafe {
1320                slice[i] = base.add(i).read_volatile();
1321            }
1322            i += 1;
1323        }
1324
1325        drop(guard);
1326        Ok(())
1327    }
1328
1329    /// Copy the contents of the sandbox at the specified offset into
1330    /// the slice
1331    pub fn copy_from_slice(&self, slice: &[u8], offset: usize) -> Result<()> {
1332        bounds_check!(offset, slice.len(), self.mem_size());
1333        let base = self.base_ptr().wrapping_add(offset);
1334        let guard = self.lock.try_read()?;
1335
1336        const CHUNK: usize = size_of::<u128>();
1337        let len = slice.len();
1338        let mut i = 0;
1339
1340        // Handle unaligned head bytes until we reach u128 alignment.
1341        // Note: align_offset can return usize::MAX if alignment is impossible.
1342        // In that case, head_len = len via .min(), so we fall back to byte-by-byte
1343        // operations for the entire slice.
1344        let align_offset = base.align_offset(align_of::<u128>());
1345        let head_len = align_offset.min(len);
1346        while i < head_len {
1347            unsafe {
1348                base.add(i).write_volatile(slice[i]);
1349            }
1350            i += 1;
1351        }
1352
1353        // Write aligned u128 chunks
1354        // SAFETY: After processing head_len bytes, base.add(i) is u128-aligned.
1355        // We use read_unaligned for the source since the slice may not be u128-aligned.
1356        let src = slice.as_ptr();
1357        while i + CHUNK <= len {
1358            unsafe {
1359                let value = std::ptr::read_unaligned(src.add(i) as *const u128);
1360                (base.add(i) as *mut u128).write_volatile(value);
1361            }
1362            i += CHUNK;
1363        }
1364
1365        // Handle remaining tail bytes
1366        while i < len {
1367            unsafe {
1368                base.add(i).write_volatile(slice[i]);
1369            }
1370            i += 1;
1371        }
1372
1373        drop(guard);
1374        Ok(())
1375    }
1376
1377    /// Fill the memory in the range `[offset, offset + len)` with `value`
1378    #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
1379    pub fn fill(&mut self, value: u8, offset: usize, len: usize) -> Result<()> {
1380        bounds_check!(offset, len, self.mem_size());
1381        let base = self.base_ptr().wrapping_add(offset);
1382        let guard = self.lock.try_read()?;
1383
1384        const CHUNK: usize = size_of::<u128>();
1385        let value_u128 = u128::from_ne_bytes([value; CHUNK]);
1386        let mut i = 0;
1387
1388        // Handle unaligned head bytes until we reach u128 alignment.
1389        // Note: align_offset can return usize::MAX if alignment is impossible.
1390        // In that case, head_len = len via .min(), so we fall back to byte-by-byte
1391        // operations for the entire slice.
1392        let align_offset = base.align_offset(align_of::<u128>());
1393        let head_len = align_offset.min(len);
1394        while i < head_len {
1395            unsafe {
1396                base.add(i).write_volatile(value);
1397            }
1398            i += 1;
1399        }
1400
1401        // Write aligned u128 chunks
1402        // SAFETY: After processing head_len bytes, base.add(i) is u128-aligned
1403        while i + CHUNK <= len {
1404            unsafe {
1405                (base.add(i) as *mut u128).write_volatile(value_u128);
1406            }
1407            i += CHUNK;
1408        }
1409
1410        // Handle remaining tail bytes
1411        while i < len {
1412            unsafe {
1413                base.add(i).write_volatile(value);
1414            }
1415            i += 1;
1416        }
1417
1418        drop(guard);
1419        Ok(())
1420    }
1421}
1422
1423impl HostSharedMemory {
1424    /// Reset this memory region to all-zeros, choosing the fastest
1425    /// strategy for the current platform and hypervisor configuration.
1426    ///
1427    /// On Linux/KVM (without mshv3), uses `MADV_DONTNEED` for lazy
1428    /// zeroing.  On Linux/mshv3, falls through to `fill(0)`.
1429    ///
1430    /// On Windows, zeroing via `fill(0)` is prohibitively expensive
1431    /// for large regions (e.g. 448 MiB scratch).  Instead, the
1432    /// mapping is replaced with a fresh demand-zero allocation.
1433    /// Returns `Some(GuestSharedMemory)` when the mapping was
1434    /// replaced (the caller must update the VM mapping), or `None`
1435    /// when zeroed in place.
1436    ///
1437    // TODO: Find the break-even point between zero-in-place and
1438    // replace for each hypervisor and use a size-based heuristic
1439    // instead of a compile-time platform check.
1440    pub(crate) fn zero_or_replace(&mut self) -> Result<Option<GuestSharedMemory>> {
1441        #[cfg(target_os = "windows")]
1442        {
1443            let new_mem = ExclusiveSharedMemory::new(self.mem_size())?;
1444            let (hscratch, gscratch) = new_mem.build();
1445            *self = hscratch;
1446            Ok(Some(gscratch))
1447        }
1448        #[cfg(not(target_os = "windows"))]
1449        {
1450            self.with_exclusivity(|e| {
1451                #[allow(unused_mut)]
1452                let mut do_copy = true;
1453                // TODO: Find a similar lazy zeroing approach that works on MSHV.
1454                //       (See Note [Keeping mappings in sync between userspace and the guest])
1455                #[cfg(all(feature = "kvm", not(any(feature = "mshv3"))))]
1456                unsafe {
1457                    let ret = libc::madvise(
1458                        e.region.ptr() as *mut libc::c_void,
1459                        e.region.size(),
1460                        libc::MADV_DONTNEED,
1461                    );
1462                    if ret == 0 {
1463                        do_copy = false;
1464                    }
1465                }
1466                if do_copy {
1467                    e.as_mut_slice().fill(0);
1468                }
1469            })?;
1470            Ok(None)
1471        }
1472    }
1473}
1474
1475impl SharedMemory for HostSharedMemory {
1476    fn region(&self) -> &HostMapping {
1477        &self.region
1478    }
1479    fn with_exclusivity<T, F: FnOnce(&mut ExclusiveSharedMemory) -> T>(
1480        &mut self,
1481        f: F,
1482    ) -> Result<T> {
1483        let guard = self.lock.try_write()?;
1484        let mut excl = ExclusiveSharedMemory {
1485            region: self.region.clone(),
1486        };
1487        let ret = f(&mut excl);
1488        drop(excl);
1489        drop(guard);
1490        Ok(ret)
1491    }
1492}
1493
1494/// A ReadonlySharedMemory is a different kind of shared memory,
1495/// separate from the exclusive/host/guest lifecycle, used to
1496/// represent read-only mappings of snapshot pages into the guest
1497/// efficiently.
1498#[derive(Clone, Debug)]
1499pub struct ReadonlySharedMemory {
1500    region: Arc<HostMapping>,
1501    /// Number of bytes from the start of the blob that `mapping_at`
1502    /// exposes to the guest. Production callers pass the size of the
1503    /// guest-visible prefix of the snapshot blob; the remainder of
1504    /// the blob is the page-table tail that lives only host-side.
1505    #[cfg_attr(unshared_snapshot_mem, allow(dead_code))]
1506    guest_mapped_size: usize,
1507}
1508// Safety: HostMapping is only non-Send/Sync (causing
1509// ReadonlySharedMemory to not be automatically Send/Sync) because raw
1510// pointers are not ("as a lint", as the Rust docs say). We don't want
1511// to mark HostMapping Send/Sync immediately, because that could
1512// socially imply that it's "safe" to use unsafe accesses from
1513// multiple threads at once in more cases, including ones that don't
1514// actually ensure immutability/synchronisation. Since
1515// ReadonlySharedMemory can only be accessed by reading, and reading
1516// concurrently from multiple threads is not racy,
1517// ReadonlySharedMemory can be Send and Sync.
1518unsafe impl Send for ReadonlySharedMemory {}
1519unsafe impl Sync for ReadonlySharedMemory {}
1520
1521impl ReadonlySharedMemory {
1522    pub(crate) fn from_bytes(contents: &[u8], guest_mapped_size: usize) -> Result<Self> {
1523        if guest_mapped_size == 0
1524            || guest_mapped_size > contents.len()
1525            || !guest_mapped_size.is_multiple_of(page_size::get())
1526        {
1527            return Err(SharedMemoryError::MemoryRequest(
1528                guest_mapped_size,
1529                0,
1530                contents.len(),
1531                page_size::get(),
1532            ));
1533        }
1534        let mut anon =
1535            ExclusiveSharedMemory::new(contents.len().next_multiple_of(page_size::get()))?;
1536        anon.copy_from_slice(contents, 0)?;
1537        Ok(ReadonlySharedMemory {
1538            region: anon.region,
1539            guest_mapped_size,
1540        })
1541    }
1542
1543    /// The number of bytes that should be mapped into guest PA space.
1544    #[cfg(not(unshared_snapshot_mem))]
1545    pub(crate) fn guest_mapped_size(&self) -> usize {
1546        self.guest_mapped_size
1547    }
1548
1549    /// Create a `ReadonlySharedMemory` backed by a file on disk.
1550    ///
1551    /// The file's length must be a non-zero multiple of `PAGE_SIZE`.
1552    /// `guest_mapped_size` must be a non-zero multiple of `PAGE_SIZE`
1553    /// no greater than the file's length.
1554    pub(crate) fn from_file(file: &std::fs::File, guest_mapped_size: usize) -> Result<Self> {
1555        let len: usize = file
1556            .metadata()
1557            .map_err(SharedMemoryError::FileMetadata)?
1558            .len()
1559            .try_into()
1560            .map_err(SharedMemoryError::FileTooLarge)?;
1561
1562        if len == 0 || !len.is_multiple_of(page_size::get()) {
1563            return Err(SharedMemoryError::MemoryRequest(
1564                len,
1565                0,
1566                usize::MAX,
1567                page_size::get(),
1568            ));
1569        }
1570
1571        if guest_mapped_size == 0
1572            || guest_mapped_size > len
1573            || !guest_mapped_size.is_multiple_of(page_size::get())
1574        {
1575            return Err(SharedMemoryError::MemoryRequest(
1576                guest_mapped_size,
1577                0,
1578                len,
1579                page_size::get(),
1580            ));
1581        }
1582
1583        let region = Self::map_file(file, len)?;
1584        Ok(ReadonlySharedMemory {
1585            region,
1586            guest_mapped_size,
1587        })
1588    }
1589
1590    /// Linux: reserve `[guard][blob][guard]` as one anonymous
1591    /// `PROT_NONE` mapping, then `MAP_FIXED` the file over the
1592    /// middle slot.
1593    #[cfg(unix)]
1594    fn map_file(file: &std::fs::File, len: usize) -> Result<Arc<HostMapping>> {
1595        use std::os::unix::io::AsRawFd;
1596
1597        #[cfg(mshv3)]
1598        use libc::PROT_WRITE;
1599        use libc::{
1600            MAP_ANONYMOUS, MAP_FAILED, MAP_FIXED, MAP_NORESERVE, MAP_PRIVATE, PROT_NONE, PROT_READ,
1601            mmap, off_t, size_t,
1602        };
1603
1604        let total_size =
1605            len.checked_add(2 * page_size::get())
1606                .ok_or(SharedMemoryError::MemoryRequest(
1607                    len,
1608                    0,
1609                    usize::MAX - 2 * page_size::get(),
1610                    1,
1611                ))?;
1612
1613        let fd = file.as_raw_fd();
1614
1615        // 1. Reserve the full `[guard][blob][guard]` address range as
1616        //    one anonymous `PROT_NONE` mapping. This pins the layout
1617        //    and gives us a single RAII owner for the whole region.
1618        // SAFETY: anonymous `mmap` with a null address has no
1619        // preconditions; the kernel picks the address.
1620        let base = unsafe {
1621            mmap(
1622                null_mut(),
1623                total_size as size_t,
1624                PROT_NONE,
1625                MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE,
1626                -1,
1627                0 as off_t,
1628            )
1629        };
1630        if base == MAP_FAILED {
1631            return Err(SharedMemoryError::MmapFailed(
1632                std::io::Error::last_os_error().raw_os_error(),
1633            ));
1634        }
1635        let reservation = Mmap {
1636            base,
1637            len: total_size,
1638        };
1639
1640        // 2. Overlay the file content on the middle slot with
1641        //    `MAP_FIXED`. `MAP_PRIVATE` keeps the mapping detached
1642        //    from the underlying file.
1643        //
1644        //    MSHV's map_user_memory requires host-writable pages
1645        //    (the kernel module calls `pin_user_pages(FOLL_PIN|FOLL_WRITE)`
1646        //    on the region when it is mapped into the partition).
1647        //    KVM accepts read-only host pages for read-only guest slots.
1648        #[cfg(mshv3)]
1649        let file_prot = PROT_READ | PROT_WRITE;
1650        #[cfg(not(mshv3))]
1651        let file_prot = PROT_READ;
1652        // SAFETY: `total_size = len + 2 * PAGE_SIZE_USIZE`, so
1653        // `base + PAGE_SIZE_USIZE` is in-bounds of the reservation.
1654        let usable_ptr = unsafe { (base as *mut u8).add(page_size::get()) };
1655        // SAFETY: `usable_ptr..usable_ptr + len` lies entirely within
1656        // the reservation owned by `reservation`. `MAP_FIXED`
1657        // replaces that sub-range in place; on failure the
1658        // surrounding anonymous mapping is unaffected and
1659        // `reservation` releases it on drop.
1660        let mapped = unsafe {
1661            mmap(
1662                usable_ptr as *mut c_void,
1663                len as size_t,
1664                file_prot,
1665                MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE,
1666                fd,
1667                0 as off_t,
1668            )
1669        };
1670        if mapped == MAP_FAILED {
1671            return Err(SharedMemoryError::MmapFailed(
1672                std::io::Error::last_os_error().raw_os_error(),
1673            ));
1674        }
1675
1676        // 3. The first and last pages keep their `PROT_NONE` from the
1677        //    anonymous reservation, so no extra `mprotect` is needed.
1678
1679        #[allow(clippy::arc_with_non_send_sync)]
1680        Ok(Arc::new(HostMapping { mmap: reservation }))
1681    }
1682
1683    /// Windows: reserve `[guard][blob][guard]` as one
1684    /// `VirtualAlloc2` placeholder, split the middle slot out, and
1685    /// `MapViewOfFile3` the file over the middle slot.
1686    #[cfg(target_os = "windows")]
1687    fn map_file(file: &std::fs::File, len: usize) -> Result<Arc<HostMapping>> {
1688        use std::os::windows::io::AsRawHandle;
1689
1690        let total_size =
1691            len.checked_add(2 * page_size::get())
1692                .ok_or(SharedMemoryError::MemoryRequest(
1693                    len,
1694                    0,
1695                    usize::MAX - 2 * page_size::get(),
1696                    1,
1697                ))?;
1698
1699        let file_handle = HANDLE(file.as_raw_handle());
1700
1701        // 1. Reserve the full `[guard][blob][guard]` address range
1702        //    as one `VirtualAlloc2` placeholder.
1703        let whole = Placeholder::reserve(total_size)?;
1704
1705        // 2. Split the placeholder into three adjacent slots. The
1706        //    leading and trailing slots stay unmapped and act as
1707        //    guard pages. The middle slot will receive the file view.
1708        let (leading, middle, trailing) = whole.split_into_three(page_size::get(), len)?;
1709
1710        // 3. Create a read-only file mapping section over the file.
1711        // SAFETY: `file_handle` is a valid file HANDLE borrowed from
1712        // `file` for the duration of this call. `CreateFileMappingA`
1713        // returns a new handle that we wrap in `FileMapping` below.
1714        let raw_handle =
1715            unsafe { CreateFileMappingA(file_handle, None, PAGE_READONLY, 0, 0, PCSTR::null()) }?;
1716        if raw_handle.is_invalid() {
1717            log_then_return!(SharedMemoryError::MemoryAllocationFailed(
1718                Error::last_os_error().raw_os_error()
1719            ));
1720        }
1721        let file_mapping = FileMapping(raw_handle);
1722
1723        // 4. Replace the middle placeholder slot with a view of the
1724        //    file mapping via `MapViewOfFile3(MEM_REPLACE_PLACEHOLDER)`.
1725        let view = middle.map_file_view(raw_handle)?;
1726
1727        #[allow(clippy::arc_with_non_send_sync)]
1728        Ok(Arc::new(HostMapping {
1729            mapping: WindowsMapping::FileBacked {
1730                leading,
1731                view,
1732                trailing,
1733                file_mapping,
1734            },
1735        }))
1736    }
1737
1738    pub(crate) fn as_slice(&self) -> &[u8] {
1739        unsafe { std::slice::from_raw_parts(self.base_ptr(), self.mem_size()) }
1740    }
1741
1742    #[cfg(unshared_snapshot_mem)]
1743    pub(crate) fn copy_to_writable(&self) -> Result<ExclusiveSharedMemory> {
1744        let mut writable = ExclusiveSharedMemory::new(self.mem_size())?;
1745        writable.copy_from_slice(self.as_slice(), 0)?;
1746        Ok(writable)
1747    }
1748
1749    #[cfg(not(unshared_snapshot_mem))]
1750    pub(crate) fn build(self) -> (Self, Self) {
1751        (self.clone(), self)
1752    }
1753
1754    #[cfg(not(unshared_snapshot_mem))]
1755    pub(crate) fn mapping_at(
1756        &self,
1757        guest_base: u64,
1758        region_type: MemoryRegionType,
1759    ) -> MemoryRegion {
1760        #[allow(clippy::panic)]
1761        // This will not ever actually panic: the only place this is
1762        // called is HyperlightVm::update_snapshot_mapping, which
1763        // always calls it with the Snapshot region type.
1764        if region_type != MemoryRegionType::Snapshot {
1765            panic!("ReadonlySharedMemory::mapping_at should only be used for Snapshot regions");
1766        }
1767        mapping_at(
1768            self,
1769            guest_base,
1770            self.guest_mapped_size(),
1771            region_type,
1772            MemoryRegionFlags::READ | MemoryRegionFlags::EXECUTE,
1773        )
1774    }
1775}
1776
1777impl SharedMemory for ReadonlySharedMemory {
1778    fn region(&self) -> &HostMapping {
1779        &self.region
1780    }
1781    // Trait defaults work for `base_addr`, `base_ptr`, and
1782    // `mem_size`: each `ReadonlySharedMemory` has the
1783    // `[guard][blob][guard]` layout.
1784    //
1785    // `host_region_base` differs per Windows mapping flavour:
1786    //  * `WindowsMapping::Anonymous` (`from_bytes`): file mapping
1787    //    spans the whole region. Same as the trait default.
1788    //  * `WindowsMapping::FileBacked` (`from_file`): file mapping
1789    //    covers only the blob. Expose only the blob to the surrogate.
1790    #[cfg(windows)]
1791    fn host_region_base(&self) -> <HostGuestMemoryRegion as MemoryRegionKind>::HostBaseType {
1792        match &self.region().mapping {
1793            WindowsMapping::Anonymous { .. } => super::memory_region::HostRegionBase {
1794                from_handle: self.region().file_mapping_handle().into(),
1795                handle_base: self.region().ptr() as usize,
1796                handle_size: self.region().size(),
1797                offset: page_size::get(),
1798            },
1799            WindowsMapping::FileBacked { .. } => super::memory_region::HostRegionBase {
1800                from_handle: self.region().file_mapping_handle().into(),
1801                handle_base: self.base_ptr() as usize,
1802                handle_size: self.mem_size(),
1803                offset: 0,
1804            },
1805        }
1806    }
1807    // There's no way to get exclusive (and therefore writable) access
1808    // to a ReadonlySharedMemory.
1809    fn with_exclusivity<T, F: FnOnce(&mut ExclusiveSharedMemory) -> T>(
1810        &mut self,
1811        _: F,
1812    ) -> Result<T> {
1813        Err(SharedMemoryError::ReadonlySharedMemoryExclusiveRequest)
1814    }
1815    // However, just access to the contents as a slice is doable
1816    fn with_contents<T, F: FnOnce(&[u8]) -> T>(&mut self, f: F) -> Result<T> {
1817        Ok(f(self.as_slice()))
1818    }
1819}
1820
1821impl<S: SharedMemory> PartialEq<S> for ReadonlySharedMemory {
1822    fn eq(&self, other: &S) -> bool {
1823        self.raw_ptr() == other.raw_ptr()
1824    }
1825}
1826
1827#[cfg(test)]
1828mod tests {
1829    use std::sync::atomic::{AtomicI32, AtomicU16, Ordering};
1830
1831    #[cfg(not(miri))]
1832    use proptest::prelude::*;
1833
1834    #[cfg(not(miri))]
1835    use super::HostSharedMemory;
1836    use super::{ExclusiveSharedMemory, Result, SharedMemory};
1837    #[cfg(not(miri))]
1838    use crate::mem::shared_mem_tests::read_write_test_suite;
1839
1840    #[test]
1841    fn fill() {
1842        let mem_size: usize = page_size::get();
1843        let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
1844        let (mut hshm, _) = eshm.build();
1845
1846        hshm.fill(1, 0, 1024).unwrap();
1847        hshm.fill(2, 1024, 1024).unwrap();
1848        hshm.fill(3, 2048, 1024).unwrap();
1849        hshm.fill(4, 3072, 1024).unwrap();
1850
1851        let vec = hshm
1852            .with_exclusivity(|e| e.copy_all_to_vec().unwrap())
1853            .unwrap();
1854
1855        assert!(vec[0..1024].iter().all(|&x| x == 1));
1856        assert!(vec[1024..2048].iter().all(|&x| x == 2));
1857        assert!(vec[2048..3072].iter().all(|&x| x == 3));
1858        assert!(vec[3072..4096].iter().all(|&x| x == 4));
1859
1860        hshm.fill(5, 0, mem_size).unwrap();
1861
1862        let vec2 = hshm
1863            .with_exclusivity(|e| e.copy_all_to_vec().unwrap())
1864            .unwrap();
1865        assert!(vec2.iter().all(|&x| x == 5));
1866
1867        assert!(hshm.fill(0, 0, mem_size + 1).is_err());
1868        assert!(hshm.fill(0, mem_size, 1).is_err());
1869    }
1870
1871    /// Verify that `bounds_check!` rejects offset + size combinations that
1872    /// would overflow `usize`.
1873    #[test]
1874    fn bounds_check_overflow() {
1875        let mem_size: usize = page_size::get();
1876        let mut eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
1877
1878        // ExclusiveSharedMemory methods
1879        assert!(eshm.read_i32(usize::MAX).is_err());
1880        assert!(eshm.write_i32(usize::MAX, 0).is_err());
1881        assert!(eshm.copy_from_slice(&[0u8; 1], usize::MAX).is_err());
1882
1883        // HostSharedMemory methods
1884        let (mut hshm, _) = eshm.build();
1885
1886        assert!(hshm.read::<u8>(usize::MAX).is_err());
1887        assert!(hshm.read::<u64>(usize::MAX - 3).is_err());
1888        assert!(hshm.write::<u8>(usize::MAX, 0).is_err());
1889        assert!(hshm.write::<u64>(usize::MAX - 3, 0).is_err());
1890
1891        let mut buf = [0u8; 1];
1892        assert!(hshm.copy_to_slice(&mut buf, usize::MAX).is_err());
1893        assert!(hshm.copy_from_slice(&[0u8; 1], usize::MAX).is_err());
1894
1895        assert!(hshm.fill(0, usize::MAX, 1).is_err());
1896        assert!(hshm.fill(0, 1, usize::MAX).is_err());
1897    }
1898
1899    #[test]
1900    fn atomic_access() {
1901        let mem_size = page_size::get();
1902        let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
1903        let (hshm, _) = eshm.build();
1904
1905        hshm.store_atomic::<AtomicU16>(0, 0x1234, Ordering::Release)
1906            .unwrap();
1907        assert_eq!(
1908            hshm.load_atomic::<AtomicU16>(0, Ordering::Acquire).unwrap(),
1909            0x1234
1910        );
1911
1912        hshm.store_atomic::<AtomicI32>(4, -42, Ordering::SeqCst)
1913            .unwrap();
1914        assert_eq!(
1915            hshm.load_atomic::<AtomicI32>(4, Ordering::SeqCst).unwrap(),
1916            -42
1917        );
1918
1919        assert!(hshm.load_atomic::<AtomicU16>(1, Ordering::Relaxed).is_err());
1920        assert!(
1921            hshm.load_atomic::<AtomicU16>(mem_size - 1, Ordering::Relaxed)
1922                .is_err()
1923        );
1924        assert!(hshm.load_atomic::<AtomicU16>(0, Ordering::Release).is_err());
1925        assert!(
1926            hshm.store_atomic::<AtomicU16>(0, 0, Ordering::Acquire)
1927                .is_err()
1928        );
1929    }
1930
1931    #[test]
1932    fn atomic_access_observes_exclusivity() {
1933        let eshm = ExclusiveSharedMemory::new(page_size::get()).unwrap();
1934        let (mut hshm, _) = eshm.build();
1935        let other = hshm.clone();
1936
1937        hshm.with_exclusivity(|_| {
1938            assert!(
1939                other
1940                    .load_atomic::<AtomicU16>(0, Ordering::Relaxed)
1941                    .is_err()
1942            );
1943        })
1944        .unwrap();
1945    }
1946
1947    #[test]
1948    fn copy_into_from() -> Result<()> {
1949        let mem_size: usize = page_size::get();
1950        let vec_len = 10;
1951        let eshm = ExclusiveSharedMemory::new(mem_size)?;
1952        let (hshm, _) = eshm.build();
1953        let vec = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
1954        // write the value to the memory at the beginning.
1955        hshm.copy_from_slice(&vec, 0)?;
1956
1957        let mut vec2 = vec![0; vec_len];
1958        // read the value back from the memory at the beginning.
1959        hshm.copy_to_slice(vec2.as_mut_slice(), 0)?;
1960        assert_eq!(vec, vec2);
1961
1962        let offset = mem_size - vec.len();
1963        // write the value to the memory at the end.
1964        hshm.copy_from_slice(&vec, offset)?;
1965
1966        let mut vec3 = vec![0; vec_len];
1967        // read the value back from the memory at the end.
1968        hshm.copy_to_slice(&mut vec3, offset)?;
1969        assert_eq!(vec, vec3);
1970
1971        let offset = mem_size / 2;
1972        // write the value to the memory at the middle.
1973        hshm.copy_from_slice(&vec, offset)?;
1974
1975        let mut vec4 = vec![0; vec_len];
1976        // read the value back from the memory at the middle.
1977        hshm.copy_to_slice(&mut vec4, offset)?;
1978        assert_eq!(vec, vec4);
1979
1980        // try and read a value from an offset that is beyond the end of the memory.
1981        let mut vec5 = vec![0; vec_len];
1982        assert!(hshm.copy_to_slice(&mut vec5, mem_size).is_err());
1983
1984        // try and write a value to an offset that is beyond the end of the memory.
1985        assert!(hshm.copy_from_slice(&vec5, mem_size).is_err());
1986
1987        // try and read a value from an offset that is too large.
1988        let mut vec6 = vec![0; vec_len];
1989        assert!(hshm.copy_to_slice(&mut vec6, mem_size * 2).is_err());
1990
1991        // try and write a value to an offset that is too large.
1992        assert!(hshm.copy_from_slice(&vec6, mem_size * 2).is_err());
1993
1994        // try and read a value that is too large.
1995        let mut vec7 = vec![0; mem_size * 2];
1996        assert!(hshm.copy_to_slice(&mut vec7, 0).is_err());
1997
1998        // try and write a value that is too large.
1999        assert!(hshm.copy_from_slice(&vec7, 0).is_err());
2000
2001        Ok(())
2002    }
2003
2004    // proptest uses file I/O (getcwd, open) which miri doesn't support
2005    #[cfg(not(miri))]
2006    proptest! {
2007        #[test]
2008        fn read_write_i32(val in -0x1000_i32..0x1000_i32) {
2009            read_write_test_suite(
2010                val,
2011                ExclusiveSharedMemory::new,
2012                Box::new(ExclusiveSharedMemory::read_i32),
2013                Box::new(ExclusiveSharedMemory::write_i32),
2014            )
2015            .unwrap();
2016            read_write_test_suite(
2017                val,
2018                |s| {
2019                    let e = ExclusiveSharedMemory::new(s)?;
2020                    let (h, _) = e.build();
2021                    Ok(h)
2022                },
2023                Box::new(HostSharedMemory::read::<i32>),
2024                Box::new(|h, o, v| h.write::<i32>(o, v)),
2025            )
2026            .unwrap();
2027        }
2028    }
2029
2030    #[test]
2031    fn alloc_fail() {
2032        let gm = ExclusiveSharedMemory::new(0);
2033        assert!(gm.is_err());
2034        let gm = ExclusiveSharedMemory::new(usize::MAX);
2035        assert!(gm.is_err());
2036    }
2037
2038    #[test]
2039    fn clone() {
2040        let eshm = ExclusiveSharedMemory::new(page_size::get()).unwrap();
2041        let (hshm1, _) = eshm.build();
2042        let hshm2 = hshm1.clone();
2043
2044        // after hshm1 is cloned, hshm1 and hshm2 should have identical
2045        // memory sizes and pointers.
2046        assert_eq!(hshm1.mem_size(), hshm2.mem_size());
2047        assert_eq!(hshm1.base_addr(), hshm2.base_addr());
2048
2049        // we should be able to copy a byte array into both hshm1 and hshm2,
2050        // and have both changes be reflected in all clones
2051        hshm1.copy_from_slice(b"a", 0).unwrap();
2052        hshm2.copy_from_slice(b"b", 1).unwrap();
2053
2054        // at this point, both hshm1 and hshm2 should have
2055        // offset 0 = 'a', offset 1 = 'b'
2056        for (raw_offset, expected) in &[(0, b'a'), (1, b'b')] {
2057            assert_eq!(hshm1.read::<u8>(*raw_offset).unwrap(), *expected);
2058            assert_eq!(hshm2.read::<u8>(*raw_offset).unwrap(), *expected);
2059        }
2060
2061        // after we drop hshm1, hshm2 should still exist, be valid,
2062        // and have all contents from before hshm1 was dropped
2063        drop(hshm1);
2064
2065        // at this point, hshm2 should still have offset 0 = 'a', offset 1 = 'b'
2066        for (raw_offset, expected) in &[(0, b'a'), (1, b'b')] {
2067            assert_eq!(hshm2.read::<u8>(*raw_offset).unwrap(), *expected);
2068        }
2069        hshm2.copy_from_slice(b"c", 2).unwrap();
2070        assert_eq!(hshm2.read::<u8>(2).unwrap(), b'c');
2071        drop(hshm2);
2072    }
2073
2074    #[test]
2075    fn copy_all_to_vec() {
2076        let mut data = vec![b'a', b'b', b'c'];
2077        data.resize(page_size::get(), 0);
2078        let mut eshm = ExclusiveSharedMemory::new(data.len()).unwrap();
2079        eshm.copy_from_slice(data.as_slice(), 0).unwrap();
2080        let ret_vec = eshm.copy_all_to_vec().unwrap();
2081        assert_eq!(data, ret_vec);
2082    }
2083
2084    /// Test that verifies memory is properly unmapped when all SharedMemory
2085    /// references are dropped.
2086    #[test]
2087    #[cfg(all(target_os = "linux", not(miri)))]
2088    fn test_drop() {
2089        use proc_maps::get_process_maps;
2090
2091        // Use a unique size that no other test uses to avoid false positives
2092        // from concurrent tests allocating at the same address.
2093        // The mprotect calls split the mapping into 3 regions (guard, usable, guard),
2094        // so we check for the usable region which has this exact size.
2095        //
2096        // NOTE: If this test fails intermittently, there may be a race condition
2097        // where another test allocates memory at the same address between our
2098        // drop and the mapping check. Ensure UNIQUE_SIZE is not used by any
2099        // other test in the codebase to avoid this.
2100        let unique_size: usize = page_size::get() * 17;
2101
2102        let pid = std::process::id();
2103
2104        let eshm = ExclusiveSharedMemory::new(unique_size).unwrap();
2105        let (hshm1, gshm) = eshm.build();
2106        let hshm2 = hshm1.clone();
2107
2108        // Use the usable memory region (not raw), since mprotect splits the mapping
2109        let base_ptr = hshm1.base_ptr() as usize;
2110        let mem_size = hshm1.mem_size();
2111
2112        // Helper to check if exact mapping exists (matching both address and size)
2113        let has_exact_mapping = |ptr: usize, size: usize| -> bool {
2114            get_process_maps(pid.try_into().unwrap())
2115                .unwrap()
2116                .iter()
2117                .any(|m| m.start() == ptr && m.size() == size)
2118        };
2119
2120        // Verify mapping exists before drop
2121        assert!(
2122            has_exact_mapping(base_ptr, mem_size),
2123            "shared memory mapping not found at {:#x} with size {}",
2124            base_ptr,
2125            mem_size
2126        );
2127
2128        // Drop all references
2129        drop(hshm1);
2130        drop(hshm2);
2131        drop(gshm);
2132
2133        // Verify exact mapping is gone
2134        assert!(
2135            !has_exact_mapping(base_ptr, mem_size),
2136            "shared memory mapping still exists at {:#x} with size {} after drop",
2137            base_ptr,
2138            mem_size
2139        );
2140    }
2141
2142    /// Tests for the optimized aligned memory operations.
2143    /// These tests verify that the u128 chunk optimization works correctly
2144    /// for various alignment scenarios and buffer sizes.
2145    mod alignment_tests {
2146        use super::*;
2147
2148        const CHUNK_SIZE: usize = size_of::<u128>();
2149
2150        /// Test copy operations with all possible starting alignment offsets (0-15)
2151        #[test]
2152        fn copy_with_various_alignments() {
2153            // Use a buffer large enough to test all alignment cases
2154            let mem_size: usize = page_size::get();
2155            let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2156            let (hshm, _) = eshm.build();
2157
2158            // Test all 16 possible alignment offsets (0 through 15)
2159            for start_offset in 0..CHUNK_SIZE {
2160                let test_len = 64; // Enough to cover head, aligned chunks, and tail
2161                let test_data: Vec<u8> = (0..test_len).map(|i| (i + start_offset) as u8).collect();
2162
2163                // Write data at the given offset
2164                hshm.copy_from_slice(&test_data, start_offset).unwrap();
2165
2166                // Read it back
2167                let mut read_buf = vec![0u8; test_len];
2168                hshm.copy_to_slice(&mut read_buf, start_offset).unwrap();
2169
2170                assert_eq!(
2171                    test_data, read_buf,
2172                    "Mismatch at alignment offset {}",
2173                    start_offset
2174                );
2175            }
2176        }
2177
2178        /// Test copy operations with lengths smaller than chunk size (< 16 bytes)
2179        #[test]
2180        fn copy_small_lengths() {
2181            let mem_size: usize = page_size::get();
2182            let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2183            let (hshm, _) = eshm.build();
2184
2185            for len in 0..CHUNK_SIZE {
2186                let test_data: Vec<u8> = (0..len).map(|i| i as u8).collect();
2187
2188                hshm.copy_from_slice(&test_data, 0).unwrap();
2189
2190                let mut read_buf = vec![0u8; len];
2191                hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2192
2193                assert_eq!(test_data, read_buf, "Mismatch for length {}", len);
2194            }
2195        }
2196
2197        /// Test copy operations with lengths that don't align to chunk boundaries
2198        #[test]
2199        fn copy_non_aligned_lengths() {
2200            let mem_size: usize = page_size::get();
2201            let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2202            let (hshm, _) = eshm.build();
2203
2204            // Test lengths like 17, 31, 33, 47, 63, 65, etc.
2205            let test_lengths = [17, 31, 33, 47, 63, 65, 100, 127, 129, 255, 257];
2206
2207            for &len in &test_lengths {
2208                let test_data: Vec<u8> = (0..len).map(|i| (i % 256) as u8).collect();
2209
2210                hshm.copy_from_slice(&test_data, 0).unwrap();
2211
2212                let mut read_buf = vec![0u8; len];
2213                hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2214
2215                assert_eq!(test_data, read_buf, "Mismatch for length {}", len);
2216            }
2217        }
2218
2219        /// Test copy with exactly one chunk (16 bytes)
2220        #[test]
2221        fn copy_exact_chunk_size() {
2222            let mem_size: usize = page_size::get();
2223            let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2224            let (hshm, _) = eshm.build();
2225
2226            let test_data: Vec<u8> = (0..CHUNK_SIZE).map(|i| i as u8).collect();
2227
2228            hshm.copy_from_slice(&test_data, 0).unwrap();
2229
2230            let mut read_buf = vec![0u8; CHUNK_SIZE];
2231            hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2232
2233            assert_eq!(test_data, read_buf);
2234        }
2235
2236        /// Test fill with various alignment offsets
2237        #[test]
2238        fn fill_with_various_alignments() {
2239            let mem_size: usize = page_size::get();
2240            let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2241            let (mut hshm, _) = eshm.build();
2242
2243            for start_offset in 0..CHUNK_SIZE {
2244                let fill_len = 64;
2245                let fill_value = (start_offset % 256) as u8;
2246
2247                // Clear memory first
2248                hshm.fill(0, 0, mem_size).unwrap();
2249
2250                // Fill at the given offset
2251                hshm.fill(fill_value, start_offset, fill_len).unwrap();
2252
2253                // Read it back and verify
2254                let mut read_buf = vec![0u8; fill_len];
2255                hshm.copy_to_slice(&mut read_buf, start_offset).unwrap();
2256
2257                assert!(
2258                    read_buf.iter().all(|&b| b == fill_value),
2259                    "Fill mismatch at alignment offset {}",
2260                    start_offset
2261                );
2262            }
2263        }
2264
2265        /// Test fill with lengths smaller than chunk size
2266        #[test]
2267        fn fill_small_lengths() {
2268            let mem_size: usize = page_size::get();
2269            let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2270            let (mut hshm, _) = eshm.build();
2271
2272            for len in 0..CHUNK_SIZE {
2273                let fill_value = 0xAB;
2274
2275                hshm.fill(0, 0, mem_size).unwrap(); // Clear
2276                hshm.fill(fill_value, 0, len).unwrap();
2277
2278                let mut read_buf = vec![0u8; len];
2279                hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2280
2281                assert!(
2282                    read_buf.iter().all(|&b| b == fill_value),
2283                    "Fill mismatch for length {}",
2284                    len
2285                );
2286            }
2287        }
2288
2289        /// Test fill with non-aligned lengths
2290        #[test]
2291        fn fill_non_aligned_lengths() {
2292            let mem_size: usize = page_size::get();
2293            let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2294            let (mut hshm, _) = eshm.build();
2295
2296            let test_lengths = [17, 31, 33, 47, 63, 65, 100, 127, 129, 255, 257];
2297
2298            for &len in &test_lengths {
2299                let fill_value = 0xCD;
2300
2301                hshm.fill(0, 0, mem_size).unwrap(); // Clear
2302                hshm.fill(fill_value, 0, len).unwrap();
2303
2304                let mut read_buf = vec![0u8; len];
2305                hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2306
2307                assert!(
2308                    read_buf.iter().all(|&b| b == fill_value),
2309                    "Fill mismatch for length {}",
2310                    len
2311                );
2312            }
2313        }
2314
2315        /// Test edge cases: length 0 and length 1
2316        #[test]
2317        fn copy_edge_cases() {
2318            let mem_size: usize = page_size::get();
2319            let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2320            let (hshm, _) = eshm.build();
2321
2322            // Length 0
2323            let empty: Vec<u8> = vec![];
2324            hshm.copy_from_slice(&empty, 0).unwrap();
2325            let mut read_buf: Vec<u8> = vec![];
2326            hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2327            assert!(read_buf.is_empty());
2328
2329            // Length 1
2330            let single = vec![0x42u8];
2331            hshm.copy_from_slice(&single, 0).unwrap();
2332            let mut read_buf = vec![0u8; 1];
2333            hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2334            assert_eq!(single, read_buf);
2335        }
2336
2337        /// Test combined: unaligned start + non-aligned length
2338        #[test]
2339        fn copy_unaligned_start_and_length() {
2340            let mem_size: usize = page_size::get();
2341            let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2342            let (hshm, _) = eshm.build();
2343
2344            // Start at offset 7 (unaligned), length 37 (not a multiple of 16)
2345            let start_offset = 7;
2346            let len = 37;
2347            let test_data: Vec<u8> = (0..len).map(|i| (i * 3) as u8).collect();
2348
2349            hshm.copy_from_slice(&test_data, start_offset).unwrap();
2350
2351            let mut read_buf = vec![0u8; len];
2352            hshm.copy_to_slice(&mut read_buf, start_offset).unwrap();
2353
2354            assert_eq!(test_data, read_buf);
2355        }
2356    }
2357
2358    #[cfg(target_os = "linux")]
2359    mod guard_page_crash_test {
2360        use crate::mem::shared_mem::{ExclusiveSharedMemory, SharedMemory};
2361
2362        const TEST_EXIT_CODE: u8 = 211; // an uncommon exit code, used for testing purposes
2363
2364        /// hook sigsegv to exit with status code, to make it testable, rather than have it exit from a signal
2365        /// NOTE: We CANNOT panic!() in the handler, and make the tests #[should_panic], because
2366        ///     the test harness process will crash anyway after the test passes
2367        fn setup_signal_handler() {
2368            unsafe {
2369                signal_hook_registry::register_signal_unchecked(libc::SIGSEGV, || {
2370                    std::process::exit(TEST_EXIT_CODE.into());
2371                })
2372                .unwrap();
2373            }
2374        }
2375
2376        #[test]
2377        #[ignore] // this test is ignored because it will crash the running process
2378        fn read() {
2379            setup_signal_handler();
2380
2381            let eshm = ExclusiveSharedMemory::new(page_size::get()).unwrap();
2382            let (hshm, _) = eshm.build();
2383            let guard_page_ptr = hshm.raw_ptr();
2384            unsafe { std::ptr::read_volatile(guard_page_ptr) };
2385        }
2386
2387        #[test]
2388        #[ignore] // this test is ignored because it will crash the running process
2389        fn write() {
2390            setup_signal_handler();
2391
2392            let eshm = ExclusiveSharedMemory::new(page_size::get()).unwrap();
2393            let (hshm, _) = eshm.build();
2394            let guard_page_ptr = hshm.raw_ptr();
2395            unsafe { std::ptr::write_volatile(guard_page_ptr, 0u8) };
2396        }
2397
2398        #[test]
2399        #[ignore] // this test is ignored because it will crash the running process
2400        fn exec() {
2401            setup_signal_handler();
2402
2403            let eshm = ExclusiveSharedMemory::new(page_size::get()).unwrap();
2404            let (hshm, _) = eshm.build();
2405            let guard_page_ptr = hshm.raw_ptr();
2406            let func: fn() = unsafe { std::mem::transmute(guard_page_ptr) };
2407            func();
2408        }
2409
2410        // provides a way for running the above tests in a separate process since they expect to crash
2411        #[test]
2412        #[cfg_attr(miri, ignore)] // miri can't spawn subprocesses
2413        fn guard_page_testing_shim() {
2414            let tests = vec!["read", "write", "exec"];
2415            for test in tests {
2416                let triple = std::env::var("TARGET_TRIPLE").ok();
2417                let target_args = if let Some(triple) = triple.filter(|t| !t.is_empty()) {
2418                    vec!["--target".to_string(), triple.to_string()]
2419                } else {
2420                    vec![]
2421                };
2422                let output = std::process::Command::new("cargo")
2423                    .args(["test", "-p", "hyperlight-host", "--lib"])
2424                    .args(target_args)
2425                    .args(["--", "--ignored", test])
2426                    .stdin(std::process::Stdio::null())
2427                    .output()
2428                    .expect("Unable to launch tests");
2429                let exit_code = output.status.code();
2430                if exit_code != Some(TEST_EXIT_CODE.into()) {
2431                    eprintln!("=== Guard Page test '{}' failed ===", test);
2432                    eprintln!("Exit code: {:?} (expected {})", exit_code, TEST_EXIT_CODE);
2433                    eprintln!("=== STDOUT ===");
2434                    eprintln!("{}", String::from_utf8_lossy(&output.stdout));
2435                    eprintln!("=== STDERR ===");
2436                    eprintln!("{}", String::from_utf8_lossy(&output.stderr));
2437                    panic!(
2438                        "Guard Page test failed: {} (exit code {:?}, expected {})",
2439                        test, exit_code, TEST_EXIT_CODE
2440                    );
2441                }
2442            }
2443        }
2444    }
2445
2446    #[cfg(not(miri))]
2447    mod from_file_tests {
2448        use std::io::Write;
2449
2450        use tempfile::NamedTempFile;
2451
2452        use crate::mem::shared_mem::{ReadonlySharedMemory, SharedMemory};
2453
2454        pub(super) fn make_temp_file(len: usize) -> NamedTempFile {
2455            let mut f = NamedTempFile::new().expect("create temp file");
2456            if len > 0 {
2457                let mut buf = vec![0u8; len];
2458                for (i, b) in buf.iter_mut().enumerate() {
2459                    *b = (i & 0xff) as u8;
2460                }
2461                f.write_all(&buf).expect("write temp file");
2462                f.flush().expect("flush temp file");
2463            }
2464            f
2465        }
2466
2467        #[test]
2468        fn from_file_success_single_page() {
2469            let tmp = make_temp_file(page_size::get());
2470            let mut rsm = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get())
2471                .expect("from_file should succeed");
2472            assert_eq!(rsm.mem_size(), page_size::get());
2473            rsm.with_contents(|slice| {
2474                for (i, b) in slice.iter().enumerate() {
2475                    assert_eq!(*b, (i & 0xff) as u8);
2476                }
2477            })
2478            .expect("with_contents should succeed");
2479        }
2480
2481        #[test]
2482        fn from_file_success_smaller_guest_mapped_size() {
2483            let tmp = make_temp_file(2 * page_size::get());
2484            let rsm = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get())
2485                .expect("from_file should succeed");
2486            assert_eq!(rsm.mem_size(), 2 * page_size::get());
2487        }
2488
2489        #[test]
2490        fn from_file_rejects_empty_file() {
2491            let tmp = make_temp_file(0);
2492            let err = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get())
2493                .expect_err("empty file should be rejected");
2494            assert!(format!("{}", err).contains("0x0 < 0x0"));
2495        }
2496
2497        #[test]
2498        fn from_file_rejects_unaligned_file_length() {
2499            let tmp = make_temp_file(page_size::get() + 1);
2500            let err = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get())
2501                .expect_err("unaligned file length should be rejected");
2502            assert!(format!("{}", err).contains(&format!(
2503                "0x{:x} % 0x{:x} = 0",
2504                page_size::get() + 1,
2505                page_size::get()
2506            )));
2507        }
2508
2509        #[test]
2510        fn from_file_rejects_zero_guest_mapped_size() {
2511            let tmp = make_temp_file(page_size::get());
2512            let err = ReadonlySharedMemory::from_file(tmp.as_file(), 0)
2513                .expect_err("zero guest_mapped_size should be rejected");
2514            assert!(format!("{}", err).contains("0x0 < 0x0"));
2515        }
2516
2517        #[test]
2518        fn from_file_rejects_unaligned_guest_mapped_size() {
2519            let tmp = make_temp_file(2 * page_size::get());
2520            let err = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get() + 1)
2521                .expect_err("unaligned guest_mapped_size should be rejected");
2522            assert!(format!("{}", err).contains(&format!(
2523                "0x{:x} % 0x{:x} = ",
2524                page_size::get() + 1,
2525                page_size::get()
2526            )));
2527        }
2528
2529        #[test]
2530        fn from_file_rejects_guest_mapped_size_exceeding_file() {
2531            let tmp = make_temp_file(page_size::get());
2532            let err = ReadonlySharedMemory::from_file(tmp.as_file(), 2 * page_size::get())
2533                .expect_err("guest_mapped_size > file length should be rejected");
2534            assert!(format!("{}", err).contains(&format!(
2535                "0x{:x} <= 0x{:x}",
2536                2 * page_size::get(),
2537                page_size::get()
2538            )));
2539        }
2540
2541        /// Tests in this submodule are `#[ignore]`'d because each one
2542        /// is expected to die from a memory access violation. They are
2543        /// not run by a plain `cargo test`. The `from_file_guard_page_shim`
2544        /// test in the parent module re-executes each one in a subprocess
2545        /// and asserts the trap occurred.
2546        mod guard_page_crash_tests {
2547            use super::make_temp_file;
2548            use crate::mem::shared_mem::{ReadonlySharedMemory, SharedMemory};
2549
2550            /// Loads from the byte immediately before the mapping.
2551            #[test]
2552            #[ignore]
2553            pub(super) fn leading_guard_page_traps() {
2554                let tmp = make_temp_file(page_size::get());
2555                let rsm = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get())
2556                    .expect("from_file should succeed");
2557                let guard_ptr = unsafe { rsm.base_ptr().sub(page_size::get()) };
2558                println!("reached_guard");
2559                let _ = unsafe { std::ptr::read_volatile(guard_ptr) };
2560                println!("survived_guard");
2561            }
2562
2563            /// Loads from the byte immediately after the mapping.
2564            #[test]
2565            #[ignore]
2566            pub(super) fn trailing_guard_page_traps() {
2567                let tmp = make_temp_file(page_size::get());
2568                let rsm = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get())
2569                    .expect("from_file should succeed");
2570                let guard_ptr = unsafe { rsm.base_ptr().add(rsm.mem_size()) };
2571                println!("reached_guard");
2572                let _ = unsafe { std::ptr::read_volatile(guard_ptr) };
2573                println!("survived_guard");
2574            }
2575        }
2576
2577        /// Spawn each ignored guard-page test as a subprocess and assert
2578        /// it terminated by an OS access violation. Re-executes the
2579        /// current test binary so no rebuild or `cargo` invocation is
2580        /// needed.
2581        #[test]
2582        #[cfg_attr(miri, ignore)] // miri can't spawn subprocesses
2583        fn from_file_guard_page_shim() {
2584            use guard_page_crash_tests::{leading_guard_page_traps, trailing_guard_page_traps};
2585            let ignored_test_paths = [
2586                test_path(leading_guard_page_traps),
2587                test_path(trailing_guard_page_traps),
2588            ];
2589
2590            let exe = std::env::current_exe().expect("current_exe");
2591            for path in &ignored_test_paths {
2592                run_guard_page_subprocess(&exe, path);
2593            }
2594        }
2595
2596        /// Derive a libtest filter path for a test function from its
2597        /// type name. Strips the leading crate-name segment that
2598        /// `type_name` includes but libtest does not.
2599        fn test_path<F: Fn()>(_: F) -> &'static str {
2600            let full = std::any::type_name::<F>();
2601            let (_, rest) = full
2602                .split_once("::")
2603                .expect("type_name of a function item is always qualified by the crate name");
2604            rest
2605        }
2606
2607        fn run_guard_page_subprocess(exe: &std::path::Path, ignored_test_path: &str) {
2608            let output = std::process::Command::new(exe)
2609                .args([
2610                    "--ignored",
2611                    "--nocapture",
2612                    "--exact",
2613                    "--test-threads=1",
2614                    ignored_test_path,
2615                ])
2616                .stdin(std::process::Stdio::null())
2617                .output()
2618                .expect("Unable to launch subprocess test");
2619
2620            let stdout = String::from_utf8_lossy(&output.stdout);
2621            let stderr = String::from_utf8_lossy(&output.stderr);
2622
2623            // libtest with no matching filter exits 0, so verify the
2624            // test actually ran via the "running 1 test" banner.
2625            let ran_test = stdout.contains("running 1 test");
2626            let reached = stdout.contains("reached_guard");
2627            let survived = stdout.contains("survived_guard");
2628            let by_access_violation = killed_by_access_violation(&output.status);
2629
2630            let ok = reached && !survived && by_access_violation && ran_test;
2631            if !ok {
2632                eprintln!("=== Guard page shim failed for {} ===", ignored_test_path);
2633                eprintln!(
2634                    "status={:?} ran_test={} reached={} survived={} by_access_violation={}",
2635                    output.status, ran_test, reached, survived, by_access_violation
2636                );
2637                eprintln!("=== STDOUT ===\n{}", stdout);
2638                eprintln!("=== STDERR ===\n{}", stderr);
2639                let hint = if !ran_test {
2640                    format!(
2641                        "\nHINT: ran_test=false (subprocess reported 'running 0 tests'). \
2642                         Most likely cause is a stale test path in the shim. Verify that \
2643                         `{}` still exists and matches the path passed via --exact above.",
2644                        ignored_test_path
2645                    )
2646                } else {
2647                    String::new()
2648                };
2649                panic!(
2650                    "Expected subprocess to run {}, print 'reached_guard', \
2651                     then die from a memory access fault. ran_test={}, reached={}, \
2652                     survived={}, by_access_violation={}, status={:?}{}",
2653                    ignored_test_path,
2654                    ran_test,
2655                    reached,
2656                    survived,
2657                    by_access_violation,
2658                    output.status,
2659                    hint
2660                );
2661            }
2662
2663            println!(
2664                "guard page trap confirmed for {}: subprocess terminated with {:?}",
2665                ignored_test_path, output.status
2666            );
2667        }
2668
2669        /// Returns true if `status` indicates the process died from a
2670        /// memory access fault (SIGBUS on macos, SIGSEGV on linux,
2671        /// STATUS_ACCESS_VIOLATION (or 0xDEAD) on Windows).
2672        fn killed_by_access_violation(status: &std::process::ExitStatus) -> bool {
2673            #[cfg(unix)]
2674            {
2675                use std::os::unix::process::ExitStatusExt;
2676                let expected_signal = if cfg!(target_os = "macos") {
2677                    libc::SIGBUS
2678                } else {
2679                    libc::SIGSEGV
2680                };
2681                status.signal() == Some(expected_signal)
2682            }
2683            #[cfg(windows)]
2684            {
2685                use windows::Win32::Foundation::STATUS_ACCESS_VIOLATION;
2686                // See https://github.com/hyperlight-dev/hyperlight/issues/1507
2687                status.code() == Some(STATUS_ACCESS_VIOLATION.0) || status.code() == Some(0xDEAD)
2688            }
2689        }
2690    }
2691}