Skip to main content

hyperlight_host/mem/
shared_mem.rs

1// SPDX-License-Identifier: Apache-2.0
2// Copyright 2025 The Hyperlight Authors.
3
4use std::ffi::c_void;
5use std::io::Error;
6use std::mem::{align_of, size_of};
7#[cfg(unix)]
8use std::ptr::null_mut;
9use std::sync::atomic::Ordering;
10use std::sync::{Arc, RwLock};
11
12use bytemuck::Pod;
13use thiserror::Error;
14use tracing::{Span, instrument};
15#[cfg(target_os = "windows")]
16use windows::Win32::Foundation::{CloseHandle, HANDLE, INVALID_HANDLE_VALUE};
17#[cfg(target_os = "windows")]
18use windows::Win32::System::Memory::PAGE_READWRITE;
19#[cfg(target_os = "windows")]
20use windows::Win32::System::Memory::{
21    CreateFileMappingA, FILE_MAP_ALL_ACCESS, MEM_PRESERVE_PLACEHOLDER, MEM_RELEASE,
22    MEM_REPLACE_PLACEHOLDER, MEM_RESERVE, MEM_RESERVE_PLACEHOLDER, MEMORY_MAPPED_VIEW_ADDRESS,
23    MapViewOfFile, MapViewOfFile3, PAGE_NOACCESS, PAGE_PROTECTION_FLAGS, PAGE_READONLY,
24    UnmapViewOfFile, VIRTUAL_ALLOCATION_TYPE, VIRTUAL_FREE_TYPE, VirtualAlloc2, VirtualFree,
25    VirtualProtect,
26};
27#[cfg(target_os = "windows")]
28use windows::core::PCSTR;
29
30use super::memory_region::{
31    HostGuestMemoryRegion, MemoryRegion, MemoryRegionFlags, MemoryRegionKind, MemoryRegionType,
32};
33use crate::log_then_return;
34
35type Result<T> = core::result::Result<T, SharedMemoryError>;
36
37/// This is just an alias for std::backtrace::Backtrace that we
38/// introduce to stop thiserror from using its backtrace
39/// functionality, which depends on nightly APIs.
40type ThisErrorHackBacktrace = std::backtrace::Backtrace;
41
42/// An error encountered while setting up or manipulating a shared memory region
43#[derive(Error, Debug)]
44pub enum SharedMemoryError {
45    /// Some operation on the shared memory attempted to read or write
46    /// out of bounds
47    #[error("Cannot access a value with size {0} at offset {1} in memory of size {2}")]
48    Bounds(usize, usize, usize),
49
50    /// An atomic access was not aligned for its atomic type.
51    #[error("Atomic access at offset {0} is not aligned to {1} bytes")]
52    AtomicAccessUnaligned(usize, usize),
53
54    /// An atomic load used an unsupported ordering.
55    #[error("Invalid atomic load ordering: {0:?}")]
56    InvalidAtomicLoadOrdering(Ordering),
57
58    /// An atomic store used an unsupported ordering.
59    #[error("Invalid atomic store ordering: {0:?}")]
60    InvalidAtomicStoreOrdering(Ordering),
61
62    /// When creating a memory with contents from a file, metadata for
63    /// that file could not be read
64    #[error("Could not access metadata for file: {0}")]
65    FileMetadata(std::io::Error),
66
67    /// When creating a memory with contents from a file, that file
68    /// was logically larger than the range of a usize.
69    #[error("File size exceeded usize: {0}")]
70    FileTooLarge(std::num::TryFromIntError),
71
72    /// The locking discipline used to enforce temporary exclusive
73    /// access by the host to a shared memory failed in some way
74    #[error("Could not acquire memory lock: {0} at {1}")]
75    LockError(String, ThisErrorHackBacktrace),
76
77    /// A request to allocate a shared memory could not be fulfilled
78    /// by the host operating system
79    #[error("Memory Allocation Failed with OS Error {0:?}.")]
80    MemoryAllocationFailed(Option<i32>),
81
82    /// A ruqest to allocate a shared shared memory had an invalid
83    /// size, due either to bounds or alignment.
84    #[error(
85        "Memory request does not satisfy constraints: 0x{1:x} < 0x{0:x} <= 0x{2:x} && 0x{0:x} % 0x{3:x} = 0"
86    )]
87    MemoryRequest(usize, usize, usize, usize),
88
89    /// An request to mmap a file or create an anonymous memory could
90    /// not be fulfilled by the host operating system
91    #[error("mmap failed with os error {0:?}")]
92    MmapFailed(Option<i32>),
93
94    /// A request to change the host-side permissions on the guard
95    /// pages of a shared memory region could not be fulfilled by the
96    /// host operating system
97    #[error("mprotect failed with os error {0:?}")]
98    MprotectFailed(Option<i32>),
99
100    /// A Windows virtual memory API call failed
101    #[cfg(target_os = "windows")]
102    #[error("Windows API Error Result {0:?}")]
103    WindowsAPIError(#[from] windows_result::Error),
104
105    /// Calling code attempted to take exclusive (write) access to a
106    /// [`ReadonlySharedMemory`].
107    #[error("Cannot take exclusive access to a ReadonlySharedMemory")]
108    ReadonlySharedMemoryExclusiveRequest,
109
110    /// Calling code attempted to freeze memory that still has writable aliases.
111    #[error("Cannot freeze shared memory that still has other owners")]
112    SharedMemoryNotExclusive,
113
114    /// An error was encountered when trying to convert a slice of raw
115    /// bytes into some logical data
116    #[error("Error reading slice {0}")]
117    TryFromSlice(#[from] std::array::TryFromSliceError),
118
119    /// An error was encountered during a routine error conversion
120    /// that should have been infallible
121    #[error("Error reading int: {0}")]
122    TryFromInt(#[from] std::num::TryFromIntError),
123}
124impl<T> From<std::sync::TryLockError<T>> for SharedMemoryError {
125    fn from(e: std::sync::TryLockError<T>) -> SharedMemoryError {
126        SharedMemoryError::LockError(format!("{:?}", e), std::backtrace::Backtrace::capture())
127    }
128}
129
130/// Makes sure that the given `offset` and `size` are within the bounds of the memory with size `mem_size`.
131macro_rules! bounds_check {
132    ($offset:expr, $size:expr, $mem_size:expr) => {
133        if $offset.checked_add($size).is_none_or(|end| end > $mem_size) {
134            return Err(SharedMemoryError::Bounds($offset, $size, $mem_size));
135        }
136    };
137}
138
139mod atomic_access {
140    pub trait Sealed {}
141}
142
143/// An integer atomic supported by [`HostSharedMemory`] atomic operations.
144///
145/// This trait is sealed and implemented for the standard signed and unsigned
146/// integer atomic types.
147#[allow(private_bounds)]
148pub trait AtomicAccess: atomic_access::Sealed {
149    /// The integer stored by this atomic type.
150    type Value: Copy;
151
152    /// Load the atomic value with `ordering`.
153    #[doc(hidden)]
154    fn load(&self, ordering: Ordering) -> Self::Value;
155
156    /// Store `value` with `ordering`.
157    #[doc(hidden)]
158    fn store(&self, value: Self::Value, ordering: Ordering);
159}
160
161macro_rules! impl_atomic_access {
162    ($atomic:ty, $value:ty) => {
163        impl atomic_access::Sealed for $atomic {}
164
165        impl AtomicAccess for $atomic {
166            type Value = $value;
167
168            fn load(&self, ordering: Ordering) -> Self::Value {
169                <$atomic>::load(self, ordering)
170            }
171
172            fn store(&self, value: Self::Value, ordering: Ordering) {
173                <$atomic>::store(self, value, ordering);
174            }
175        }
176    };
177}
178
179/// generates a reader function for the given type
180macro_rules! generate_reader {
181    ($fname:ident, $ty:ty) => {
182        /// Read a value of type `$ty` from the memory at the given offset.
183        #[allow(dead_code)]
184        #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
185        pub(crate) fn $fname(&self, offset: usize) -> Result<$ty> {
186            let data = self.as_slice();
187            bounds_check!(offset, std::mem::size_of::<$ty>(), data.len());
188            Ok(<$ty>::from_le_bytes(
189                data[offset..offset + std::mem::size_of::<$ty>()].try_into()?,
190            ))
191        }
192    };
193}
194
195/// generates a writer function for the given type
196macro_rules! generate_writer {
197    ($fname:ident, $ty:ty) => {
198        /// Write a value of type `$ty` to the memory at the given offset.
199        #[allow(dead_code)]
200        pub(crate) fn $fname(&mut self, offset: usize, value: $ty) -> Result<()> {
201            let data = self.as_mut_slice();
202            bounds_check!(offset, std::mem::size_of::<$ty>(), data.len());
203            data[offset..offset + std::mem::size_of::<$ty>()].copy_from_slice(&value.to_le_bytes());
204            Ok(())
205        }
206    };
207}
208
209impl_atomic_access!(std::sync::atomic::AtomicI8, i8);
210impl_atomic_access!(std::sync::atomic::AtomicI16, i16);
211impl_atomic_access!(std::sync::atomic::AtomicI32, i32);
212impl_atomic_access!(std::sync::atomic::AtomicI64, i64);
213impl_atomic_access!(std::sync::atomic::AtomicIsize, isize);
214impl_atomic_access!(std::sync::atomic::AtomicU8, u8);
215impl_atomic_access!(std::sync::atomic::AtomicU16, u16);
216impl_atomic_access!(std::sync::atomic::AtomicU32, u32);
217impl_atomic_access!(std::sync::atomic::AtomicU64, u64);
218impl_atomic_access!(std::sync::atomic::AtomicUsize, usize);
219
220/// A representation of a host mapping of a shared memory region,
221/// which will be released when this structure is Drop'd. This is not
222/// individually Clone (since it holds ownership of the mapping), or
223/// Send or Sync, since it doesn't ensure any particular synchronization.
224#[derive(Debug)]
225pub struct HostMapping {
226    #[cfg(not(target_os = "windows"))]
227    mmap: Mmap,
228    #[cfg(target_os = "windows")]
229    mapping: WindowsMapping,
230}
231
232/// Windows-side flavors of a [`HostMapping`].
233#[cfg(target_os = "windows")]
234#[derive(Debug)]
235enum WindowsMapping {
236    /// `[guard][blob][guard]` carved from a single anonymous file
237    /// mapping created via `CreateFileMappingA(INVALID_HANDLE_VALUE)`
238    /// and mapped with `MapViewOfFile`.
239    ///
240    /// ```text
241    /// |<------------------ view ------------------>|
242    /// [   guard   ][         blob         ][   guard   ]
243    /// ```
244    Anonymous {
245        view: MappedView,
246        file_mapping: FileMapping,
247    },
248    /// File-backed: a `VirtualAlloc2` placeholder is split in three.
249    /// The middle slot is replaced by a `MapViewOfFile3` view of the
250    /// file. The flanking placeholder slots remain unmapped and act
251    /// as the guard pages.
252    ///
253    /// ```text
254    /// [ leading ][         view         ][ trailing ]
255    /// ```
256    FileBacked {
257        leading: Placeholder,
258        view: MappedView,
259        trailing: Placeholder,
260        file_mapping: FileMapping,
261    },
262}
263
264impl HostMapping {
265    /// Base address of the host mapping, including the surrounding guard pages.
266    pub(crate) fn ptr(&self) -> *mut u8 {
267        #[cfg(not(target_os = "windows"))]
268        {
269            self.mmap.base as *mut u8
270        }
271        #[cfg(target_os = "windows")]
272        match &self.mapping {
273            WindowsMapping::Anonymous { view, .. } => view.addr as *mut u8,
274            WindowsMapping::FileBacked { leading, .. } => leading.addr as *mut u8,
275        }
276    }
277
278    /// Total size of the host mapping, including the surrounding guard pages.
279    pub(crate) fn size(&self) -> usize {
280        #[cfg(not(target_os = "windows"))]
281        {
282            self.mmap.len
283        }
284        #[cfg(target_os = "windows")]
285        match &self.mapping {
286            WindowsMapping::Anonymous { view, .. } => view.len,
287            WindowsMapping::FileBacked {
288                leading,
289                view,
290                trailing,
291                ..
292            } => leading.size + view.len + trailing.size,
293        }
294    }
295
296    /// Win32 file-mapping handle backing this mapping.
297    #[cfg(target_os = "windows")]
298    pub(crate) fn file_mapping_handle(&self) -> HANDLE {
299        match &self.mapping {
300            WindowsMapping::Anonymous { file_mapping, .. }
301            | WindowsMapping::FileBacked { file_mapping, .. } => file_mapping.0,
302        }
303    }
304}
305
306/// RAII guard for an `mmap` reservation. Calls `munmap` on drop.
307#[cfg(unix)]
308#[derive(Debug)]
309struct Mmap {
310    base: *mut c_void,
311    len: usize,
312}
313
314#[cfg(unix)]
315impl Drop for Mmap {
316    fn drop(&mut self) {
317        // SAFETY: `self.base` and `self.len` are exactly what was
318        // returned by the `mmap` that produced this `Mmap`, and that
319        // mapping has not been unmapped (we own it).
320        unsafe {
321            if libc::munmap(self.base, self.len) != 0 {
322                tracing::error!(
323                    "Mmap::drop: munmap failed: {:?}",
324                    std::io::Error::last_os_error()
325                );
326            }
327        }
328    }
329}
330
331/// RAII guard for a Win32 mapped view. Calls `UnmapViewOfFile` on drop.
332#[cfg(target_os = "windows")]
333#[derive(Debug)]
334struct MappedView {
335    addr: *mut c_void,
336    len: usize,
337}
338
339#[cfg(target_os = "windows")]
340impl Drop for MappedView {
341    fn drop(&mut self) {
342        let view = MEMORY_MAPPED_VIEW_ADDRESS { Value: self.addr };
343        // Plain `UnmapViewOfFile` fully releases the address range.
344        // `UnmapViewOfFile2(MEM_PRESERVE_PLACEHOLDER)` would convert
345        // it back into a placeholder for remapping, which is not
346        // what we want: the surrounding guard `Placeholder`s release
347        // their slots independently on drop.
348        // SAFETY: `self.addr` is the base address returned by the
349        // `MapViewOfFile` call that produced this `MappedView`, and
350        // the view has not been unmapped (we own it).
351        if let Err(e) = unsafe { UnmapViewOfFile(view) } {
352            tracing::error!(
353                "MappedView::drop(addr={:?}, len={}) UnmapViewOfFile failed: {:?}",
354                self.addr,
355                self.len,
356                e
357            );
358        }
359    }
360}
361
362/// Owns a Win32 file-mapping `HANDLE`. Calls `CloseHandle` on drop.
363#[cfg(target_os = "windows")]
364#[derive(Debug)]
365struct FileMapping(HANDLE);
366
367#[cfg(target_os = "windows")]
368impl Drop for FileMapping {
369    fn drop(&mut self) {
370        // SAFETY: `self.0` is a valid HANDLE returned by
371        // `CreateFileMappingA` that has not been closed (we own it).
372        unsafe {
373            if let Err(e) = CloseHandle(self.0) {
374                tracing::error!(
375                    "FileMapping::drop(handle={:?}) CloseHandle failed: {:?}",
376                    self.0,
377                    e
378                );
379            }
380        }
381    }
382}
383
384/// RAII guard for a `VirtualAlloc2` placeholder reservation. Owns
385/// the `[addr, addr + size)` range until split into smaller
386/// placeholders, replaced by a mapped view, or dropped (which calls
387/// `VirtualFree(MEM_RELEASE)`).
388#[cfg(target_os = "windows")]
389#[derive(Debug)]
390pub(crate) struct Placeholder {
391    addr: *mut c_void,
392    size: usize,
393}
394
395#[cfg(target_os = "windows")]
396impl Placeholder {
397    fn reserve(size: usize) -> Result<Self> {
398        // SAFETY: `VirtualAlloc2` with `MEM_RESERVE |
399        // MEM_RESERVE_PLACEHOLDER` and `PAGE_NOACCESS` only reserves
400        // address space. No pages are committed and no access is
401        // granted, so the call has no preconditions and the returned
402        // reservation cannot be misused from safe code.
403        let addr = unsafe {
404            VirtualAlloc2(
405                None,
406                None,
407                size,
408                VIRTUAL_ALLOCATION_TYPE(MEM_RESERVE.0 | MEM_RESERVE_PLACEHOLDER.0),
409                PAGE_NOACCESS.0,
410                None,
411            )
412        };
413        if addr.is_null() {
414            log_then_return!(SharedMemoryError::MemoryAllocationFailed(
415                Error::last_os_error().raw_os_error()
416            ));
417        }
418        Ok(Placeholder { addr, size })
419    }
420
421    fn split_front(self, front_size: usize) -> Result<(Placeholder, Placeholder)> {
422        debug_assert!(front_size > 0 && front_size < self.size);
423        debug_assert!(front_size.is_multiple_of(page_size::get()));
424        // SAFETY: `self` owns the placeholder reservation at
425        // `[self.addr, self.addr + self.size)`. `MEM_RELEASE |
426        // MEM_PRESERVE_PLACEHOLDER` is the Win32 idiom for splitting
427        // a placeholder in two: no memory is released, the
428        // reservation is just carved at `front_size`.
429        if let Err(e) = unsafe {
430            VirtualFree(
431                self.addr,
432                front_size,
433                VIRTUAL_FREE_TYPE(MEM_RELEASE.0 | MEM_PRESERVE_PLACEHOLDER.0),
434            )
435        } {
436            // `self` drops here, releasing the unsplit reservation.
437            log_then_return!(SharedMemoryError::WindowsAPIError(e.clone()));
438        }
439        let addr = self.addr;
440        let total = self.size;
441        // Forget the parent so its `Drop` does not release the two
442        // child slots as one.
443        std::mem::forget(self);
444        let front = Placeholder {
445            addr,
446            size: front_size,
447        };
448        let back = Placeholder {
449            // SAFETY: `front_size < total`, so `addr + front_size`
450            // is in-bounds of the original reservation.
451            addr: unsafe { (addr as *mut u8).add(front_size) as *mut c_void },
452            size: total - front_size,
453        };
454        Ok((front, back))
455    }
456
457    fn split_into_three(
458        self,
459        front_size: usize,
460        middle_size: usize,
461    ) -> Result<(Placeholder, Placeholder, Placeholder)> {
462        let (front, rest) = self.split_front(front_size)?;
463        let (middle, back) = rest.split_front(middle_size)?;
464        Ok((front, middle, back))
465    }
466
467    fn map_file_view(self, file_mapping: HANDLE) -> Result<MappedView> {
468        // SAFETY: `self` owns the placeholder slot at
469        // `[self.addr, self.addr + self.size)`.
470        // `MEM_REPLACE_PLACEHOLDER` requires the target range to be
471        // an existing placeholder of exactly that size, which the
472        // type system guarantees here. On success, ownership of the
473        // range transfers to the returned `MappedView`; the caller
474        // releases the file mapping handle via `FileMapping`.
475        let mapped = unsafe {
476            MapViewOfFile3(
477                file_mapping,
478                None,
479                Some(self.addr),
480                0,
481                self.size,
482                MEM_REPLACE_PLACEHOLDER,
483                PAGE_READONLY.0,
484                None,
485            )
486        };
487        if mapped.Value.is_null() {
488            // `self` drops here, releasing the placeholder.
489            log_then_return!(SharedMemoryError::MemoryAllocationFailed(
490                Error::last_os_error().raw_os_error()
491            ));
492        }
493        let addr = self.addr;
494        let len = self.size;
495        std::mem::forget(self);
496        Ok(MappedView { addr, len })
497    }
498}
499
500#[cfg(target_os = "windows")]
501impl Drop for Placeholder {
502    fn drop(&mut self) {
503        // SAFETY: `self.addr` is the base of a placeholder
504        // reservation we own. `MEM_RELEASE` with size 0 releases the
505        // entire reservation.
506        if let Err(e) = unsafe { VirtualFree(self.addr, 0, VIRTUAL_FREE_TYPE(MEM_RELEASE.0)) } {
507            tracing::error!(
508                "Placeholder::drop(addr={:?}, size={}) VirtualFree failed: {:?}",
509                self.addr,
510                self.size,
511                e
512            );
513        }
514    }
515}
516
517/// A trait that abstracts over the particular kind of SharedMemory,
518/// used when invoking operations from Rust that absolutely must have
519/// exclusive control over the shared memory for correctness +
520/// performance, like snapshotting.
521pub trait SharedMemory {
522    /// Return a readonly reference to the host mapping backing this SharedMemory
523    fn region(&self) -> &HostMapping;
524
525    /// Return the base address of the host mapping of this
526    /// region. Following the general Rust philosophy, this does not
527    /// need to be marked as `unsafe` because doing anything with this
528    /// pointer itself requires `unsafe`.
529    fn base_addr(&self) -> usize {
530        self.region().ptr() as usize + page_size::get()
531    }
532
533    /// Return the base address of the host mapping of this region as
534    /// a pointer. Following the general Rust philosophy, this does
535    /// not need to be marked as `unsafe` because doing anything with
536    /// this pointer itself requires `unsafe`.
537    fn base_ptr(&self) -> *mut u8 {
538        self.region().ptr().wrapping_add(page_size::get())
539    }
540
541    /// Return the length of usable memory contained in `self`.
542    /// The returned size does not include the size of the surrounding
543    /// guard pages.
544    fn mem_size(&self) -> usize {
545        self.region().size() - 2 * page_size::get()
546    }
547
548    /// Return the raw base address of the host mapping, including the
549    /// guard pages.
550    fn raw_ptr(&self) -> *mut u8 {
551        self.region().ptr()
552    }
553
554    /// Return the raw size of the host mapping, including the guard
555    /// pages.
556    fn raw_mem_size(&self) -> usize {
557        self.region().size()
558    }
559
560    /// Extract a base address that can be mapped into a VM for this
561    /// SharedMemory.
562    ///
563    /// On Linux this returns a raw `usize` pointer. On Windows it
564    /// returns a [`HostRegionBase`](super::memory_region::HostRegionBase)
565    /// that carries the file-mapping handle metadata needed by WHP.
566    fn host_region_base(&self) -> <HostGuestMemoryRegion as MemoryRegionKind>::HostBaseType {
567        #[cfg(not(windows))]
568        {
569            self.base_addr()
570        }
571        #[cfg(windows)]
572        {
573            super::memory_region::HostRegionBase {
574                from_handle: self.region().file_mapping_handle().into(),
575                handle_base: self.region().ptr() as usize,
576                handle_size: self.region().size(),
577                offset: page_size::get(),
578            }
579        }
580    }
581
582    /// Return the end address of the host region (base + usable size).
583    fn host_region_end(&self) -> <HostGuestMemoryRegion as MemoryRegionKind>::HostBaseType {
584        <HostGuestMemoryRegion as MemoryRegionKind>::add(self.host_region_base(), self.mem_size())
585    }
586
587    /// Run some code with exclusive access to the SharedMemory
588    /// underlying this.  If the SharedMemory is not an
589    /// ExclusiveSharedMemory, any concurrent accesses to the relevant
590    /// HostSharedMemory/GuestSharedMemory may make this fail, or be
591    /// made to fail by this, and should be avoided.
592    fn with_exclusivity<T, F: FnOnce(&mut ExclusiveSharedMemory) -> T>(
593        &mut self,
594        f: F,
595    ) -> Result<T>;
596
597    /// Run some code that is allowed to access the contents of the
598    /// SharedMemory as if it is a normal slice.  By default, this is
599    /// implemented via [`SharedMemory::with_exclusivity`], which is
600    /// the correct implementation for a memory that can be mutated,
601    /// but a [`ReadonlySharedMemory`], can support this.
602    fn with_contents<T, F: FnOnce(&[u8]) -> T>(&mut self, f: F) -> Result<T> {
603        self.with_exclusivity(|m| f(m.as_slice()))
604    }
605}
606
607fn mapping_at(
608    s: &impl SharedMemory,
609    gpa: u64,
610    size: usize,
611    region_type: MemoryRegionType,
612    flags: MemoryRegionFlags,
613) -> MemoryRegion {
614    let guest_base = gpa as usize;
615
616    MemoryRegion {
617        guest_region: guest_base..(guest_base + size),
618        host_region: s.host_region_base()
619            ..<HostGuestMemoryRegion as MemoryRegionKind>::add(s.host_region_base(), size),
620        region_type,
621        flags,
622    }
623}
624
625/// These three structures represent various phases of the lifecycle of
626/// a memory buffer that is shared with the guest. An
627/// ExclusiveSharedMemory is used for certain operations that
628/// unrestrictedly write to the shared memory, including setting it up
629/// and taking snapshots.
630#[derive(Debug)]
631pub struct ExclusiveSharedMemory {
632    region: Arc<HostMapping>,
633}
634unsafe impl Send for ExclusiveSharedMemory {}
635
636impl ExclusiveSharedMemory {
637    /// Helper function used to abstract common checks from Windows
638    /// and Linux implementations of [`ExclusiveSharedMemory::new()`]
639    fn total_size(min_size_bytes: usize) -> Result<usize> {
640        if min_size_bytes > 0 &&
641            // guard page around the memory
642            let Some(total_size) = min_size_bytes.checked_add(2 * page_size::get()) &&
643            total_size % page_size::get() == 0 &&
644            // usize and isize are guaranteed to be the same size, and
645            // isize::MAX should be positive, so this cast should be
646            // safe.
647            total_size <= isize::MAX as usize
648        {
649            Ok(total_size)
650        } else {
651            Err(SharedMemoryError::MemoryRequest(
652                min_size_bytes,
653                2,
654                isize::MAX as usize - 2 * page_size::get(),
655                page_size::get(),
656            ))
657        }
658    }
659
660    /// Create a new region of shared memory with the given minimum
661    /// size in bytes. The region will be surrounded by guard pages.
662    ///
663    /// Return `Err` if shared memory could not be allocated.
664    #[cfg(unix)]
665    #[instrument(skip_all, parent = Span::current(), level= "Trace")]
666    pub fn new(min_size_bytes: usize) -> Result<Self> {
667        use libc::{
668            MAP_ANONYMOUS, MAP_FAILED, MAP_PRIVATE, PROT_READ, PROT_WRITE, c_int, mmap, off_t,
669            size_t,
670        };
671
672        let total_size = Self::total_size(min_size_bytes)?;
673
674        #[cfg(not(miri))]
675        use libc::{MAP_NORESERVE, PROT_NONE, mprotect};
676
677        // allocate the memory
678        #[cfg(not(miri))]
679        let flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE;
680        #[cfg(miri)]
681        let flags = MAP_ANONYMOUS | MAP_PRIVATE;
682
683        let addr = unsafe {
684            mmap(
685                null_mut(),
686                total_size as size_t,
687                PROT_READ | PROT_WRITE,
688                flags,
689                -1 as c_int,
690                0 as off_t,
691            )
692        };
693        if addr == MAP_FAILED {
694            log_then_return!(SharedMemoryError::MmapFailed(
695                Error::last_os_error().raw_os_error()
696            ));
697        }
698        let mmap = Mmap {
699            base: addr,
700            len: total_size,
701        };
702
703        // protect the guard pages
704        #[cfg(not(miri))]
705        {
706            let res = unsafe { mprotect(mmap.base, page_size::get(), PROT_NONE) };
707            if res != 0 {
708                return Err(SharedMemoryError::MprotectFailed(
709                    Error::last_os_error().raw_os_error(),
710                ));
711            }
712            let res = unsafe {
713                mprotect(
714                    (mmap.base as *const u8).add(total_size - page_size::get()) as *mut c_void,
715                    page_size::get(),
716                    PROT_NONE,
717                )
718            };
719            if res != 0 {
720                return Err(SharedMemoryError::MprotectFailed(
721                    Error::last_os_error().raw_os_error(),
722                ));
723            }
724        }
725
726        Ok(Self {
727            // HostMapping is only non-Send/Sync because raw pointers
728            // are not ("as a lint", as the Rust docs say). We don't
729            // want to mark HostMapping Send/Sync immediately, because
730            // that could socially imply that it's "safe" to use
731            // unsafe accesses from multiple threads at once. Instead, we
732            // directly impl Send and Sync on this type. Since this
733            // type does have Send and Sync manually impl'd, the Arc
734            // is not pointless as the lint suggests.
735            #[allow(clippy::arc_with_non_send_sync)]
736            region: Arc::new(HostMapping { mmap }),
737        })
738    }
739
740    /// Create a new region of shared memory with the given minimum
741    /// size in bytes. The region will be surrounded by guard pages.
742    ///
743    /// Return `Err` if shared memory could not be allocated.
744    #[cfg(target_os = "windows")]
745    #[instrument(skip_all, parent = Span::current(), level= "Trace")]
746    pub fn new(min_size_bytes: usize) -> Result<Self> {
747        let total_size = Self::total_size(min_size_bytes)?;
748
749        let mut dwmaximumsizehigh = 0;
750        let mut dwmaximumsizelow = 0;
751
752        if std::mem::size_of::<usize>() == 8 {
753            dwmaximumsizehigh = (total_size >> 32) as u32;
754            dwmaximumsizelow = (total_size & 0xFFFFFFFF) as u32;
755        }
756
757        // Allocate the memory use CreateFileMapping instead of VirtualAlloc
758        // This allows us to map the memory into the surrogate process using MapViewOfFile2
759
760        let flags = PAGE_READWRITE;
761
762        let handle = unsafe {
763            CreateFileMappingA(
764                INVALID_HANDLE_VALUE,
765                None,
766                flags,
767                dwmaximumsizehigh,
768                dwmaximumsizelow,
769                PCSTR::null(),
770            )?
771        };
772
773        if handle.is_invalid() {
774            log_then_return!(SharedMemoryError::MemoryAllocationFailed(
775                Error::last_os_error().raw_os_error()
776            ));
777        }
778        let file_mapping = FileMapping(handle);
779
780        let file_map = FILE_MAP_ALL_ACCESS;
781        let addr = unsafe { MapViewOfFile(file_mapping.0, file_map, 0, 0, 0) };
782
783        if addr.Value.is_null() {
784            log_then_return!(SharedMemoryError::MemoryAllocationFailed(
785                Error::last_os_error().raw_os_error()
786            ));
787        }
788        let view = MappedView {
789            addr: addr.Value,
790            len: total_size,
791        };
792
793        // Set the first and last pages to be guard pages
794
795        let mut unused_out_old_prot_flags = PAGE_PROTECTION_FLAGS(0);
796
797        // If the following calls to VirtualProtect are changed make sure to update the calls to VirtualProtectEx in surrogate_process_manager.rs
798
799        let first_guard_page_start = view.addr;
800        if let Err(e) = unsafe {
801            VirtualProtect(
802                first_guard_page_start,
803                page_size::get(),
804                PAGE_NOACCESS,
805                &mut unused_out_old_prot_flags,
806            )
807        } {
808            log_then_return!(SharedMemoryError::WindowsAPIError(e.clone()));
809        }
810
811        let last_guard_page_start = unsafe { view.addr.add(total_size - page_size::get()) };
812        if let Err(e) = unsafe {
813            VirtualProtect(
814                last_guard_page_start,
815                page_size::get(),
816                PAGE_NOACCESS,
817                &mut unused_out_old_prot_flags,
818            )
819        } {
820            log_then_return!(SharedMemoryError::WindowsAPIError(e.clone()));
821        }
822
823        Ok(Self {
824            // HostMapping is only non-Send/Sync because raw pointers
825            // are not ("as a lint", as the Rust docs say). We don't
826            // want to mark HostMapping Send/Sync immediately, because
827            // that could socially imply that it's "safe" to use
828            // unsafe accesses from multiple threads at once. Instead, we
829            // directly impl Send and Sync on this type. Since this
830            // type does have Send and Sync manually impl'd, the Arc
831            // is not pointless as the lint suggests.
832            #[allow(clippy::arc_with_non_send_sync)]
833            region: Arc::new(HostMapping {
834                mapping: WindowsMapping::Anonymous { view, file_mapping },
835            }),
836        })
837    }
838
839    /// Internal helper method to get the backing memory as a mutable slice.
840    ///
841    /// # Safety
842    /// As per std::slice::from_raw_parts_mut:
843    /// - self.base_addr() must be valid for both reads and writes for
844    ///   self.mem_size() * mem::size_of::<u8>() many bytes, and it
845    ///   must be properly aligned.
846    ///
847    ///   The rules on validity are still somewhat unspecified, but we
848    ///   assume that the result of our calls to mmap/CreateFileMappings may
849    ///   be considered a single "allocated object". The use of
850    ///   non-atomic accesses is alright from a Safe Rust standpoint,
851    ///   because SharedMemoryBuilder is  not Sync.
852    /// - self.base_addr() must point to self.mem_size() consecutive
853    ///   properly initialized values of type u8
854    ///
855    ///   Again, the exact provenance restrictions on what is
856    ///   considered to be initialized values are unclear, but we make
857    ///   sure to use mmap(MAP_ANONYMOUS) and
858    ///   CreateFileMapping(SEC_COMMIT), so the pages in question are
859    ///   zero-initialized, which we hope counts for u8.
860    /// - The memory referenced by the returned slice must not be
861    ///   accessed through any other pointer (not derived from the
862    ///   return value) for the duration of the lifetime 'a. Both read
863    ///   and write accesses are forbidden.
864    ///
865    ///   Accesses from Safe Rust necessarily follow this rule,
866    ///   because the returned slice's lifetime is the same as that of
867    ///   a mutable borrow of self.
868    /// - The total size self.mem_size() * mem::size_of::<u8>() of the
869    ///   slice must be no larger than isize::MAX, and adding that
870    ///   size to data must not "wrap around" the address space. See
871    ///   the safety documentation of pointer::offset.
872    ///
873    ///   This is ensured by a check in ::new()
874    pub(crate) fn as_mut_slice(&mut self) -> &mut [u8] {
875        unsafe { std::slice::from_raw_parts_mut(self.base_ptr(), self.mem_size()) }
876    }
877
878    /// Internal helper method to get the backing memory as a slice.
879    ///
880    /// # Safety
881    /// See the discussion on as_mut_slice, with the third point
882    /// replaced by:
883    /// - The memory referenced by the returned slice must not be
884    ///   mutated for the duration of lifetime 'a, except inside an
885    ///   UnsafeCell.
886    ///
887    ///   Host accesses from Safe Rust necessarily follow this rule,
888    ///   because the returned slice's lifetime is the same as that of
889    ///   a borrow of self, preventing mutations via other methods.
890    #[instrument(skip_all, parent = Span::current(), level= "Trace")]
891    pub fn as_slice<'a>(&'a self) -> &'a [u8] {
892        unsafe { std::slice::from_raw_parts(self.base_ptr(), self.mem_size()) }
893    }
894
895    /// Copy the entire contents of `self` into a `Vec<u8>`, then return it
896    #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
897    #[cfg(test)]
898    pub(crate) fn copy_all_to_vec(&self) -> Result<Vec<u8>> {
899        let data = self.as_slice();
900        Ok(data.to_vec())
901    }
902
903    /// Copies all bytes from `src` to `self` starting at offset
904    #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
905    pub fn copy_from_slice(&mut self, src: &[u8], offset: usize) -> Result<()> {
906        let data = self.as_mut_slice();
907        bounds_check!(offset, src.len(), data.len());
908        data[offset..offset + src.len()].copy_from_slice(src);
909        Ok(())
910    }
911
912    generate_reader!(read_u8, u8);
913    generate_reader!(read_i8, i8);
914    generate_reader!(read_u16, u16);
915    generate_reader!(read_i16, i16);
916    generate_reader!(read_u32, u32);
917    generate_reader!(read_i32, i32);
918    generate_reader!(read_u64, u64);
919    generate_reader!(read_i64, i64);
920    generate_reader!(read_usize, usize);
921    generate_reader!(read_isize, isize);
922
923    generate_writer!(write_u8, u8);
924    generate_writer!(write_i8, i8);
925    generate_writer!(write_u16, u16);
926    generate_writer!(write_i16, i16);
927    generate_writer!(write_u32, u32);
928    generate_writer!(write_i32, i32);
929    generate_writer!(write_u64, u64);
930    generate_writer!(write_i64, i64);
931    generate_writer!(write_usize, usize);
932    generate_writer!(write_isize, isize);
933
934    /// Convert the ExclusiveSharedMemory, which may be freely
935    /// modified, into a GuestSharedMemory, which may be somewhat
936    /// freely modified (mostly by the guest), and a HostSharedMemory,
937    /// which may only make certain kinds of accesses that do not race
938    /// in the presence of malicious code inside the guest mutating
939    /// the GuestSharedMemory.
940    pub fn build(self) -> (HostSharedMemory, GuestSharedMemory) {
941        let lock = Arc::new(RwLock::new(()));
942        let hshm = HostSharedMemory {
943            region: self.region.clone(),
944            lock: lock.clone(),
945        };
946        (
947            hshm,
948            GuestSharedMemory {
949                region: self.region.clone(),
950                lock,
951            },
952        )
953    }
954
955    /// Convert this allocation to immutable memory without copying it.
956    pub(crate) fn freeze(mut self, guest_mapped_size: usize) -> Result<ReadonlySharedMemory> {
957        if guest_mapped_size > self.mem_size()
958            || !guest_mapped_size.is_multiple_of(page_size::get())
959        {
960            return Err(SharedMemoryError::MemoryRequest(
961                guest_mapped_size,
962                0,
963                self.mem_size(),
964                page_size::get(),
965            ));
966        }
967        if Arc::get_mut(&mut self.region).is_none() {
968            return Err(SharedMemoryError::SharedMemoryNotExclusive);
969        }
970
971        // This is the only Arc left. To make another one you have to clone this
972        // one, and it lives in `self`, which this function owns.
973        Ok(ReadonlySharedMemory {
974            region: self.region,
975            guest_mapped_size,
976        })
977    }
978
979    /// Gets the file handle of the shared memory region for this Sandbox
980    #[cfg(target_os = "windows")]
981    pub fn get_mmap_file_handle(&self) -> HANDLE {
982        self.region.file_mapping_handle()
983    }
984}
985
986impl SharedMemory for ExclusiveSharedMemory {
987    fn region(&self) -> &HostMapping {
988        &self.region
989    }
990    fn with_exclusivity<T, F: FnOnce(&mut ExclusiveSharedMemory) -> T>(
991        &mut self,
992        f: F,
993    ) -> Result<T> {
994        Ok(f(self))
995    }
996}
997
998/// A GuestSharedMemory is used to represent
999/// the reference to all-of-memory that is taken by the virtual cpu.
1000/// Because of the memory model limitations that affect
1001/// HostSharedMemory, it is likely fairly important (to ensure that
1002/// our UB remains limited to interaction with an external compilation
1003/// unit that likely can't be discovered by the compiler) that _rust_
1004/// users do not perform racy accesses to the guest communication
1005/// buffers that are also accessed by HostSharedMemory.
1006#[derive(Debug)]
1007pub struct GuestSharedMemory {
1008    region: Arc<HostMapping>,
1009    /// The lock that indicates this shared memory is being used by non-Rust code
1010    ///
1011    /// This lock _must_ be held whenever the guest is executing,
1012    /// because it prevents the host from converting its
1013    /// HostSharedMemory to an ExclusiveSharedMemory. Since the guest
1014    /// may arbitrarily mutate the shared memory, only synchronized
1015    /// accesses from Rust should be allowed!
1016    ///
1017    /// We cannot enforce this in the type system, because the memory
1018    /// is mapped in to the VM at VM creation time.
1019    pub lock: Arc<RwLock<()>>,
1020}
1021unsafe impl Send for GuestSharedMemory {}
1022
1023impl GuestSharedMemory {
1024    /// Create a [`super::memory_region::MemoryRegion`] structure
1025    /// suitable for mapping this region into a VM
1026    pub(crate) fn mapping_at(
1027        &self,
1028        guest_base: u64,
1029        region_type: MemoryRegionType,
1030    ) -> MemoryRegion {
1031        let flags = match region_type {
1032            MemoryRegionType::Scratch => {
1033                MemoryRegionFlags::READ | MemoryRegionFlags::WRITE | MemoryRegionFlags::EXECUTE
1034            }
1035            #[cfg(unshared_snapshot_mem)]
1036            MemoryRegionType::Snapshot => {
1037                MemoryRegionFlags::READ | MemoryRegionFlags::WRITE | MemoryRegionFlags::EXECUTE
1038            }
1039            #[allow(clippy::panic)]
1040            // This will not ever actually panic: the only places this
1041            // is called are SnapshotMemoryBacking::mappings and
1042            // HyperlightVm::update_scratch_mapping. The latter
1043            // statically uses the Scratch region type, and the former
1044            // does not use this at all when the unshared_snapshot_mem
1045            // feature is not set, since in that case the scratch
1046            // mapping type is ReadonlySharedMemory, not
1047            // GuestSharedMemory.
1048            _ => panic!(
1049                "GuestSharedMemory::mapping_at should only be used for Scratch or Snapshot regions"
1050            ),
1051        };
1052        mapping_at(self, guest_base, self.mem_size(), region_type, flags)
1053    }
1054}
1055
1056impl SharedMemory for GuestSharedMemory {
1057    fn region(&self) -> &HostMapping {
1058        &self.region
1059    }
1060    fn with_exclusivity<T, F: FnOnce(&mut ExclusiveSharedMemory) -> T>(
1061        &mut self,
1062        f: F,
1063    ) -> Result<T> {
1064        let guard = self.lock.try_write()?;
1065        let mut excl = ExclusiveSharedMemory {
1066            region: self.region.clone(),
1067        };
1068        let ret = f(&mut excl);
1069        drop(excl);
1070        drop(guard);
1071        Ok(ret)
1072    }
1073}
1074
1075/// A HostSharedMemory allows synchronized accesses to guest
1076/// communication buffers, allowing it to be used concurrently with a
1077/// GuestSharedMemory.
1078///
1079/// # Concurrency model
1080///
1081/// Given future requirements for asynchronous I/O with a minimum
1082/// amount of copying (e.g. WASIp3 streams), we would like it to be
1083/// possible to safely access these buffers concurrently with the
1084/// guest, ensuring that (1) data is read appropriately if the guest
1085/// is well-behaved; and (2) the host's behaviour is defined
1086/// regardless of whether or not the guest is well-behaved.
1087///
1088/// The ideal (future) flow for a guest->host message is something like
1089///   - Guest writes (unordered) bytes describing a work item into a buffer
1090///   - Guest reveals buffer via a release-store of a pointer into an
1091///     MMIO ring-buffer
1092///   - Host acquire-loads the buffer pointer from the "MMIO" ring
1093///     buffer
1094///   - Host (unordered) reads the bytes from the buffer
1095///   - Host performs validation of those bytes and uses them
1096///
1097/// Unfortunately, there appears to be no way to do this with defined
1098/// behaviour in present Rust (see
1099/// e.g. <https://github.com/rust-lang/unsafe-code-guidelines/issues/152>).
1100/// Rust does not yet have its own defined memory model, but in the
1101/// interim, it is widely treated as inheriting the current C/C++
1102/// memory models.  The most immediate problem is that regardless of
1103/// anything else, under those memory models \[1, p. 17-18; 2, p. 88\],
1104///
1105///   > The execution of a program contains a _data race_ if it
1106///   > contains two [C++23: "potentially concurrent"] conflicting
1107///   > actions [C23: "in different threads"], at least one of which
1108///   > is not atomic, and neither happens before the other [C++23: ",
1109///   > except for the special case for signal handlers described
1110///   > below"].  Any such data race results in undefined behavior.
1111///
1112/// Consequently, if a misbehaving guest fails to correctly
1113/// synchronize its stores with the host, the host's innocent loads
1114/// will trigger undefined behaviour for the entire program, including
1115/// the host.  Note that this also applies if the guest makes an
1116/// unsynchronized read of a location that the host is writing!
1117///
1118/// Despite Rust's de jure inheritance of the C memory model at the
1119/// present time, the compiler in many cases de facto adheres to LLVM
1120/// semantics, so it is worthwhile to consider what LLVM does in this
1121/// case as well.  According to the the LangRef \[3\] memory model,
1122/// loads which are involved in a race that includes at least one
1123/// non-atomic access (whether the load or a store) return `undef`,
1124/// making them roughly equivalent to reading uninitialized
1125/// memory. While this is much better, it is still bad.
1126///
1127/// Considering a different direction, recent C++ papers have seemed
1128/// to lean towards using `volatile` for similar use cases. For
1129/// example, in P1152R0 \[4\], JF Bastien notes that
1130///
1131///   > We’ve shown that volatile is purposely defined to denote
1132///   > external modifications. This happens for:
1133///   >   - Shared memory with untrusted code, where volatile is the
1134///   >     right way to avoid time-of-check time-of-use (ToCToU)
1135///   >     races which lead to security bugs such as \[PWN2OWN\] and
1136///   >     \[XENXSA155\].
1137///
1138/// Unfortunately, although this paper was adopted for C++20 (and,
1139/// sadly, mostly un-adopted for C++23, although that does not concern
1140/// us), the paper did not actually redefine volatile accesses or data
1141/// races to prevent volatile accesses from racing with other accesses
1142/// and causing undefined behaviour.  P1382R1 \[5\] would have amended
1143/// the wording of the data race definition to specifically exclude
1144/// volatile, but, unfortunately, despite receiving a
1145/// generally-positive reception at its first WG21 meeting more than
1146/// five years ago, it has not progressed.
1147///
1148/// Separately from the data race issue, there is also a concern that
1149/// according to the various memory models in use, there may be ways
1150/// in which the guest can semantically obtain uninitialized memory
1151/// and write it into the shared buffer, which may also result in
1152/// undefined behaviour on reads.  The degree to which this is a
1153/// concern is unclear, however, since it is unclear to what degree
1154/// the Rust abstract machine's conception of uninitialized memory
1155/// applies to the sandbox.  Returning briefly to the LLVM level,
1156/// rather than the Rust level, this, combined with the fact that
1157/// racing loads in LLVM return `undef`, as discussed above, we would
1158/// ideally `llvm.freeze` the result of any load out of the sandbox.
1159///
1160/// It would furthermore be ideal if we could run the flatbuffers
1161/// parsing code directly on the guest memory, in order to avoid
1162/// unnecessary copies.  That is unfortunately probably not viable at
1163/// the present time: because the generated flatbuffers parsing code
1164/// doesn't use atomic or volatile accesses, it is likely to introduce
1165/// double-read vulnerabilities.
1166///
1167/// In short, none of the Rust-level operations available to us do the
1168/// right thing, at the Rust spec level or the LLVM spec level. Our
1169/// major remaining options are therefore:
1170///   - Choose one of the options that is available to us, and accept
1171///     that we are doing something unsound according to the spec, but
1172///     hope that no reasonable compiler could possibly notice.
1173///   - Use inline assembly per architecture, for which we would only
1174///     need to worry about the _architecture_'s memory model (which
1175///     is far less demanding).
1176///
1177/// The leading candidate for the first option would seem to be to
1178/// simply use volatile accesses; there seems to be wide agreement
1179/// that this _should_ be a valid use case for them (even if it isn't
1180/// now), and projects like Linux and rust-vmm already use C11
1181/// `volatile` for this purpose.  It is also worth noting that because
1182/// we still do need to synchronize with the guest when it _is_ being
1183/// well-behaved, we would ideally use volatile acquire loads and
1184/// volatile release stores for interacting with the stack pointer in
1185/// the guest in this case.  Unfortunately, while those operations are
1186/// defined in LLVM, they are not presently exposed to Rust. While
1187/// atomic fences that are not associated with memory accesses
1188/// ([`std::sync::atomic::fence`]) might at first glance seem to help with
1189/// this problem, they unfortunately do not \[6\]:
1190///
1191///    > A fence ‘A’ which has (at least) Release ordering semantics,
1192///    > synchronizes with a fence ‘B’ with (at least) Acquire
1193///    > semantics, if and only if there exist operations X and Y,
1194///    > both operating on some atomic object ‘M’ such that A is
1195///    > sequenced before X, Y is sequenced before B and Y observes
1196///    > the change to M. This provides a happens-before dependence
1197///    > between A and B.
1198///
1199/// Note that the X and Y must be to an _atomic_ object.
1200///
1201/// We consequently assume that there has been a strong architectural
1202/// fence on a vmenter/vmexit between data being read and written.
1203/// This is unsafe (not guaranteed in the type system)!
1204///
1205/// \[1\] N3047 C23 Working Draft. <https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3047.pdf>
1206/// \[2\] N4950 C++23 Working Draft. <https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2023/n4950.pdf>
1207/// \[3\] LLVM Language Reference Manual, Memory Model for Concurrent Operations. <https://llvm.org/docs/LangRef.html#memmodel>
1208/// \[4\] P1152R0: Deprecating `volatile`. JF Bastien. <https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2018/p1152r0.html>
1209/// \[5\] P1382R1: `volatile_load<T>` and `volatile_store<T>`. JF Bastien, Paul McKenney, Jeffrey Yasskin, and the indefatigable TBD. <https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2019/p1382r1.pdf>
1210/// \[6\] Documentation for std::sync::atomic::fence. <https://doc.rust-lang.org/std/sync/atomic/fn.fence.html>
1211///
1212/// # Note \[Keeping mappings in sync between userspace and the guest\]
1213///
1214/// When using this structure with mshv on Linux, it is necessary to
1215/// be a little bit careful: since the hypervisor is not directly
1216/// integrated with the host kernel virtual memory subsystem, it is
1217/// easy for the memory region in userspace to get out of sync with
1218/// the memory region mapped into the guest.  Generally speaking, when
1219/// the [`SharedMemory`] is mapped into a partition, the MSHV kernel
1220/// module will call `pin_user_pages(FOLL_PIN|FOLL_WRITE)` on it,
1221/// which will eagerly do any CoW, etc needing to obtain backing pages
1222/// pinned in memory, and then map precisely those backing pages into
1223/// the virtual machine. After that, the backing pages mapped into the
1224/// VM will not change until the region is unmapped or remapped.  This
1225/// means that code in this module needs to be very careful to avoid
1226/// changing the backing pages of the region in the host userspace,
1227/// since that would result in hyperlight-host's view of the memory
1228/// becoming completely divorced from the view of the VM.
1229#[derive(Clone, Debug)]
1230pub struct HostSharedMemory {
1231    region: Arc<HostMapping>,
1232    lock: Arc<RwLock<()>>,
1233}
1234unsafe impl Send for HostSharedMemory {}
1235
1236impl HostSharedMemory {
1237    /// Read a [`Pod`] value of type `T`, whose representation is the same
1238    /// between the sandbox and the host.
1239    pub fn read<T: Pod>(&self, offset: usize) -> Result<T> {
1240        bounds_check!(offset, std::mem::size_of::<T>(), self.mem_size());
1241        let mut ret = T::zeroed();
1242        self.copy_to_slice(bytemuck::bytes_of_mut(&mut ret), offset)?;
1243        Ok(ret)
1244    }
1245
1246    /// Write a [`Pod`] value of type `T`, whose representation is the same
1247    /// between the sandbox and the host.
1248    pub fn write<T: Pod>(&self, offset: usize, data: T) -> Result<()> {
1249        bounds_check!(offset, std::mem::size_of::<T>(), self.mem_size());
1250        self.copy_from_slice(bytemuck::bytes_of(&data), offset)
1251    }
1252
1253    /// Load an integer atomic at `offset` with `ordering`.
1254    pub fn load_atomic<A: AtomicAccess>(
1255        &self,
1256        offset: usize,
1257        ordering: Ordering,
1258    ) -> Result<A::Value> {
1259        if matches!(ordering, Ordering::Release | Ordering::AcqRel) {
1260            return Err(SharedMemoryError::InvalidAtomicLoadOrdering(ordering));
1261        }
1262
1263        bounds_check!(offset, size_of::<A>(), self.mem_size());
1264        let ptr = self.base_ptr().wrapping_add(offset);
1265        if !(ptr as usize).is_multiple_of(align_of::<A>()) {
1266            return Err(SharedMemoryError::AtomicAccessUnaligned(
1267                offset,
1268                align_of::<A>(),
1269            ));
1270        }
1271
1272        let _guard = self.lock.try_read()?;
1273
1274        // SAFETY: The bounds and alignment checks cover an A within the mapping.
1275        // AtomicAccess is sealed to integer atomics, whose bit patterns are valid.
1276        let atomic = unsafe { &*ptr.cast::<A>() };
1277        Ok(atomic.load(ordering))
1278    }
1279
1280    /// Store an integer atomic at `offset` with `ordering`.
1281    pub fn store_atomic<A: AtomicAccess>(
1282        &self,
1283        offset: usize,
1284        value: A::Value,
1285        ordering: Ordering,
1286    ) -> Result<()> {
1287        if matches!(ordering, Ordering::Acquire | Ordering::AcqRel) {
1288            return Err(SharedMemoryError::InvalidAtomicStoreOrdering(ordering));
1289        }
1290
1291        bounds_check!(offset, size_of::<A>(), self.mem_size());
1292        let ptr = self.base_ptr().wrapping_add(offset);
1293        if !(ptr as usize).is_multiple_of(align_of::<A>()) {
1294            return Err(SharedMemoryError::AtomicAccessUnaligned(
1295                offset,
1296                align_of::<A>(),
1297            ));
1298        }
1299
1300        let _guard = self.lock.try_read()?;
1301
1302        // SAFETY: The bounds and alignment checks cover an A within the mapping.
1303        // AtomicAccess is sealed to integer atomics, whose bit patterns are valid.
1304        let atomic = unsafe { &*ptr.cast::<A>() };
1305        atomic.store(value, ordering);
1306        Ok(())
1307    }
1308
1309    /// Run `f` with the contents as a plain slice. The write lock keeps every
1310    /// other host access to this memory out until `f` returns.
1311    pub(crate) fn with_exclusive_contents<T>(&self, f: impl FnOnce(&[u8]) -> T) -> Result<T> {
1312        let guard = self.lock.try_write()?;
1313        let exclusive = ExclusiveSharedMemory {
1314            region: self.region.clone(),
1315        };
1316        let ret = f(exclusive.as_slice());
1317        drop(exclusive);
1318        drop(guard);
1319        Ok(ret)
1320    }
1321
1322    /// Copy the contents of the slice into the sandbox at the
1323    /// specified offset
1324    pub fn copy_to_slice(&self, slice: &mut [u8], offset: usize) -> Result<()> {
1325        bounds_check!(offset, slice.len(), self.mem_size());
1326        let base = self.base_ptr().wrapping_add(offset);
1327        let guard = self.lock.try_read()?;
1328
1329        const CHUNK: usize = size_of::<u128>();
1330        let len = slice.len();
1331        let mut i = 0;
1332
1333        // Handle unaligned head bytes until we reach u128 alignment.
1334        // Note: align_offset can return usize::MAX if alignment is impossible.
1335        // In that case, head_len = len via .min(), so we fall back to byte-by-byte
1336        // operations for the entire slice.
1337        let align_offset = base.align_offset(align_of::<u128>());
1338        let head_len = align_offset.min(len);
1339        while i < head_len {
1340            unsafe {
1341                slice[i] = base.add(i).read_volatile();
1342            }
1343            i += 1;
1344        }
1345
1346        // Read aligned u128 chunks
1347        // SAFETY: After processing head_len bytes, base.add(i) is u128-aligned.
1348        // We use write_unaligned for the destination since the slice may not be u128-aligned.
1349        let dst = slice.as_mut_ptr();
1350        while i + CHUNK <= len {
1351            unsafe {
1352                let value = (base.add(i) as *const u128).read_volatile();
1353                std::ptr::write_unaligned(dst.add(i) as *mut u128, value);
1354            }
1355            i += CHUNK;
1356        }
1357
1358        // Handle remaining tail bytes
1359        while i < len {
1360            unsafe {
1361                slice[i] = base.add(i).read_volatile();
1362            }
1363            i += 1;
1364        }
1365
1366        drop(guard);
1367        Ok(())
1368    }
1369
1370    /// Copy the contents of the sandbox at the specified offset into
1371    /// the slice
1372    pub fn copy_from_slice(&self, slice: &[u8], offset: usize) -> Result<()> {
1373        bounds_check!(offset, slice.len(), self.mem_size());
1374        let base = self.base_ptr().wrapping_add(offset);
1375        let guard = self.lock.try_read()?;
1376
1377        const CHUNK: usize = size_of::<u128>();
1378        let len = slice.len();
1379        let mut i = 0;
1380
1381        // Handle unaligned head bytes until we reach u128 alignment.
1382        // Note: align_offset can return usize::MAX if alignment is impossible.
1383        // In that case, head_len = len via .min(), so we fall back to byte-by-byte
1384        // operations for the entire slice.
1385        let align_offset = base.align_offset(align_of::<u128>());
1386        let head_len = align_offset.min(len);
1387        while i < head_len {
1388            unsafe {
1389                base.add(i).write_volatile(slice[i]);
1390            }
1391            i += 1;
1392        }
1393
1394        // Write aligned u128 chunks
1395        // SAFETY: After processing head_len bytes, base.add(i) is u128-aligned.
1396        // We use read_unaligned for the source since the slice may not be u128-aligned.
1397        let src = slice.as_ptr();
1398        while i + CHUNK <= len {
1399            unsafe {
1400                let value = std::ptr::read_unaligned(src.add(i) as *const u128);
1401                (base.add(i) as *mut u128).write_volatile(value);
1402            }
1403            i += CHUNK;
1404        }
1405
1406        // Handle remaining tail bytes
1407        while i < len {
1408            unsafe {
1409                base.add(i).write_volatile(slice[i]);
1410            }
1411            i += 1;
1412        }
1413
1414        drop(guard);
1415        Ok(())
1416    }
1417
1418    /// Fill the memory in the range `[offset, offset + len)` with `value`
1419    #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
1420    pub fn fill(&mut self, value: u8, offset: usize, len: usize) -> Result<()> {
1421        bounds_check!(offset, len, self.mem_size());
1422        let base = self.base_ptr().wrapping_add(offset);
1423        let guard = self.lock.try_read()?;
1424
1425        const CHUNK: usize = size_of::<u128>();
1426        let value_u128 = u128::from_ne_bytes([value; CHUNK]);
1427        let mut i = 0;
1428
1429        // Handle unaligned head bytes until we reach u128 alignment.
1430        // Note: align_offset can return usize::MAX if alignment is impossible.
1431        // In that case, head_len = len via .min(), so we fall back to byte-by-byte
1432        // operations for the entire slice.
1433        let align_offset = base.align_offset(align_of::<u128>());
1434        let head_len = align_offset.min(len);
1435        while i < head_len {
1436            unsafe {
1437                base.add(i).write_volatile(value);
1438            }
1439            i += 1;
1440        }
1441
1442        // Write aligned u128 chunks
1443        // SAFETY: After processing head_len bytes, base.add(i) is u128-aligned
1444        while i + CHUNK <= len {
1445            unsafe {
1446                (base.add(i) as *mut u128).write_volatile(value_u128);
1447            }
1448            i += CHUNK;
1449        }
1450
1451        // Handle remaining tail bytes
1452        while i < len {
1453            unsafe {
1454                base.add(i).write_volatile(value);
1455            }
1456            i += 1;
1457        }
1458
1459        drop(guard);
1460        Ok(())
1461    }
1462}
1463
1464impl HostSharedMemory {
1465    /// Reset this memory region to all-zeros, choosing the fastest
1466    /// strategy for the current platform and hypervisor configuration.
1467    ///
1468    /// On Linux/KVM (without mshv3), uses `MADV_DONTNEED` for lazy
1469    /// zeroing.  On Linux/mshv3, falls through to `fill(0)`.
1470    ///
1471    /// On Windows, zeroing via `fill(0)` is prohibitively expensive
1472    /// for large regions (e.g. 448 MiB scratch).  Instead, the
1473    /// mapping is replaced with a fresh demand-zero allocation.
1474    /// Returns `Some(GuestSharedMemory)` when the mapping was
1475    /// replaced (the caller must update the VM mapping), or `None`
1476    /// when zeroed in place.
1477    ///
1478    // TODO: Find the break-even point between zero-in-place and
1479    // replace for each hypervisor and use a size-based heuristic
1480    // instead of a compile-time platform check.
1481    pub(crate) fn zero_or_replace(&mut self) -> Result<Option<GuestSharedMemory>> {
1482        #[cfg(target_os = "windows")]
1483        {
1484            let new_mem = ExclusiveSharedMemory::new(self.mem_size())?;
1485            let (hscratch, gscratch) = new_mem.build();
1486            *self = hscratch;
1487            Ok(Some(gscratch))
1488        }
1489        #[cfg(not(target_os = "windows"))]
1490        {
1491            self.with_exclusivity(|e| {
1492                #[allow(unused_mut)]
1493                let mut do_copy = true;
1494                // TODO: Find a similar lazy zeroing approach that works on MSHV.
1495                //       (See Note [Keeping mappings in sync between userspace and the guest])
1496                #[cfg(all(feature = "kvm", not(any(feature = "mshv3"))))]
1497                unsafe {
1498                    let ret = libc::madvise(
1499                        e.region.ptr() as *mut libc::c_void,
1500                        e.region.size(),
1501                        libc::MADV_DONTNEED,
1502                    );
1503                    if ret == 0 {
1504                        do_copy = false;
1505                    }
1506                }
1507                if do_copy {
1508                    e.as_mut_slice().fill(0);
1509                }
1510            })?;
1511            Ok(None)
1512        }
1513    }
1514}
1515
1516impl SharedMemory for HostSharedMemory {
1517    fn region(&self) -> &HostMapping {
1518        &self.region
1519    }
1520    fn with_exclusivity<T, F: FnOnce(&mut ExclusiveSharedMemory) -> T>(
1521        &mut self,
1522        f: F,
1523    ) -> Result<T> {
1524        let guard = self.lock.try_write()?;
1525        let mut excl = ExclusiveSharedMemory {
1526            region: self.region.clone(),
1527        };
1528        let ret = f(&mut excl);
1529        drop(excl);
1530        drop(guard);
1531        Ok(ret)
1532    }
1533}
1534
1535/// A ReadonlySharedMemory is a different kind of shared memory,
1536/// separate from the exclusive/host/guest lifecycle, used to
1537/// represent read-only mappings of snapshot pages into the guest
1538/// efficiently.
1539#[derive(Clone, Debug)]
1540pub struct ReadonlySharedMemory {
1541    region: Arc<HostMapping>,
1542    /// Number of bytes from the start of the blob that `mapping_at`
1543    /// exposes to the guest. Production callers pass the size of the
1544    /// guest-visible prefix of the snapshot blob; the remainder of
1545    /// the blob is the page-table tail that lives only host-side.
1546    guest_mapped_size: usize,
1547}
1548// Safety: HostMapping is only non-Send/Sync (causing
1549// ReadonlySharedMemory to not be automatically Send/Sync) because raw
1550// pointers are not ("as a lint", as the Rust docs say). We don't want
1551// to mark HostMapping Send/Sync immediately, because that could
1552// socially imply that it's "safe" to use unsafe accesses from
1553// multiple threads at once in more cases, including ones that don't
1554// actually ensure immutability/synchronisation. Since
1555// ReadonlySharedMemory can only be accessed by reading, and reading
1556// concurrently from multiple threads is not racy,
1557// ReadonlySharedMemory can be Send and Sync.
1558unsafe impl Send for ReadonlySharedMemory {}
1559unsafe impl Sync for ReadonlySharedMemory {}
1560
1561impl ReadonlySharedMemory {
1562    #[cfg(any(test, hvf))]
1563    pub(crate) fn from_bytes(contents: &[u8], guest_mapped_size: usize) -> Result<Self> {
1564        if guest_mapped_size > contents.len() || !guest_mapped_size.is_multiple_of(page_size::get())
1565        {
1566            return Err(SharedMemoryError::MemoryRequest(
1567                guest_mapped_size,
1568                0,
1569                contents.len(),
1570                page_size::get(),
1571            ));
1572        }
1573        let mut anon =
1574            ExclusiveSharedMemory::new(contents.len().next_multiple_of(page_size::get()))?;
1575        anon.copy_from_slice(contents, 0)?;
1576        Ok(ReadonlySharedMemory {
1577            region: anon.region,
1578            guest_mapped_size,
1579        })
1580    }
1581
1582    /// The number of bytes that should be mapped into guest PA space.
1583    pub(crate) fn guest_mapped_size(&self) -> usize {
1584        self.guest_mapped_size
1585    }
1586
1587    /// Create a `ReadonlySharedMemory` backed by a file on disk.
1588    ///
1589    /// The file's length must be a non-zero multiple of `PAGE_SIZE`.
1590    /// `guest_mapped_size` must be a multiple of `PAGE_SIZE` no greater
1591    /// than the file's length.
1592    pub(crate) fn from_file(file: &std::fs::File, guest_mapped_size: usize) -> Result<Self> {
1593        let len: usize = file
1594            .metadata()
1595            .map_err(SharedMemoryError::FileMetadata)?
1596            .len()
1597            .try_into()
1598            .map_err(SharedMemoryError::FileTooLarge)?;
1599
1600        if len == 0 || !len.is_multiple_of(page_size::get()) {
1601            return Err(SharedMemoryError::MemoryRequest(
1602                len,
1603                0,
1604                usize::MAX,
1605                page_size::get(),
1606            ));
1607        }
1608
1609        if guest_mapped_size > len || !guest_mapped_size.is_multiple_of(page_size::get()) {
1610            return Err(SharedMemoryError::MemoryRequest(
1611                guest_mapped_size,
1612                0,
1613                len,
1614                page_size::get(),
1615            ));
1616        }
1617
1618        let region = Self::map_file(file, len)?;
1619        Ok(ReadonlySharedMemory {
1620            region,
1621            guest_mapped_size,
1622        })
1623    }
1624
1625    /// Linux: reserve `[guard][blob][guard]` as one anonymous
1626    /// `PROT_NONE` mapping, then `MAP_FIXED` the file over the
1627    /// middle slot.
1628    #[cfg(unix)]
1629    fn map_file(file: &std::fs::File, len: usize) -> Result<Arc<HostMapping>> {
1630        use std::os::unix::io::AsRawFd;
1631
1632        #[cfg(mshv3)]
1633        use libc::PROT_WRITE;
1634        use libc::{
1635            MAP_ANONYMOUS, MAP_FAILED, MAP_FIXED, MAP_NORESERVE, MAP_PRIVATE, PROT_NONE, PROT_READ,
1636            mmap, off_t, size_t,
1637        };
1638
1639        let total_size =
1640            len.checked_add(2 * page_size::get())
1641                .ok_or(SharedMemoryError::MemoryRequest(
1642                    len,
1643                    0,
1644                    usize::MAX - 2 * page_size::get(),
1645                    1,
1646                ))?;
1647
1648        let fd = file.as_raw_fd();
1649
1650        // 1. Reserve the full `[guard][blob][guard]` address range as
1651        //    one anonymous `PROT_NONE` mapping. This pins the layout
1652        //    and gives us a single RAII owner for the whole region.
1653        // SAFETY: anonymous `mmap` with a null address has no
1654        // preconditions; the kernel picks the address.
1655        let base = unsafe {
1656            mmap(
1657                null_mut(),
1658                total_size as size_t,
1659                PROT_NONE,
1660                MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE,
1661                -1,
1662                0 as off_t,
1663            )
1664        };
1665        if base == MAP_FAILED {
1666            return Err(SharedMemoryError::MmapFailed(
1667                std::io::Error::last_os_error().raw_os_error(),
1668            ));
1669        }
1670        let reservation = Mmap {
1671            base,
1672            len: total_size,
1673        };
1674
1675        // 2. Overlay the file content on the middle slot with
1676        //    `MAP_FIXED`. `MAP_PRIVATE` keeps the mapping detached
1677        //    from the underlying file.
1678        //
1679        //    MSHV's map_user_memory requires host-writable pages
1680        //    (the kernel module calls `pin_user_pages(FOLL_PIN|FOLL_WRITE)`
1681        //    on the region when it is mapped into the partition).
1682        //    KVM accepts read-only host pages for read-only guest slots.
1683        #[cfg(mshv3)]
1684        let file_prot = PROT_READ | PROT_WRITE;
1685        #[cfg(not(mshv3))]
1686        let file_prot = PROT_READ;
1687        // SAFETY: `total_size = len + 2 * PAGE_SIZE_USIZE`, so
1688        // `base + PAGE_SIZE_USIZE` is in-bounds of the reservation.
1689        let usable_ptr = unsafe { (base as *mut u8).add(page_size::get()) };
1690        // SAFETY: `usable_ptr..usable_ptr + len` lies entirely within
1691        // the reservation owned by `reservation`. `MAP_FIXED`
1692        // replaces that sub-range in place; on failure the
1693        // surrounding anonymous mapping is unaffected and
1694        // `reservation` releases it on drop.
1695        let mapped = unsafe {
1696            mmap(
1697                usable_ptr as *mut c_void,
1698                len as size_t,
1699                file_prot,
1700                MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE,
1701                fd,
1702                0 as off_t,
1703            )
1704        };
1705        if mapped == MAP_FAILED {
1706            return Err(SharedMemoryError::MmapFailed(
1707                std::io::Error::last_os_error().raw_os_error(),
1708            ));
1709        }
1710
1711        // 3. The first and last pages keep their `PROT_NONE` from the
1712        //    anonymous reservation, so no extra `mprotect` is needed.
1713
1714        #[allow(clippy::arc_with_non_send_sync)]
1715        Ok(Arc::new(HostMapping { mmap: reservation }))
1716    }
1717
1718    /// Windows: reserve `[guard][blob][guard]` as one
1719    /// `VirtualAlloc2` placeholder, split the middle slot out, and
1720    /// `MapViewOfFile3` the file over the middle slot.
1721    #[cfg(target_os = "windows")]
1722    fn map_file(file: &std::fs::File, len: usize) -> Result<Arc<HostMapping>> {
1723        use std::os::windows::io::AsRawHandle;
1724
1725        let total_size =
1726            len.checked_add(2 * page_size::get())
1727                .ok_or(SharedMemoryError::MemoryRequest(
1728                    len,
1729                    0,
1730                    usize::MAX - 2 * page_size::get(),
1731                    1,
1732                ))?;
1733
1734        let file_handle = HANDLE(file.as_raw_handle());
1735
1736        // 1. Reserve the full `[guard][blob][guard]` address range
1737        //    as one `VirtualAlloc2` placeholder.
1738        let whole = Placeholder::reserve(total_size)?;
1739
1740        // 2. Split the placeholder into three adjacent slots. The
1741        //    leading and trailing slots stay unmapped and act as
1742        //    guard pages. The middle slot will receive the file view.
1743        let (leading, middle, trailing) = whole.split_into_three(page_size::get(), len)?;
1744
1745        // 3. Create a read-only file mapping section over the file.
1746        // SAFETY: `file_handle` is a valid file HANDLE borrowed from
1747        // `file` for the duration of this call. `CreateFileMappingA`
1748        // returns a new handle that we wrap in `FileMapping` below.
1749        let raw_handle =
1750            unsafe { CreateFileMappingA(file_handle, None, PAGE_READONLY, 0, 0, PCSTR::null()) }?;
1751        if raw_handle.is_invalid() {
1752            log_then_return!(SharedMemoryError::MemoryAllocationFailed(
1753                Error::last_os_error().raw_os_error()
1754            ));
1755        }
1756        let file_mapping = FileMapping(raw_handle);
1757
1758        // 4. Replace the middle placeholder slot with a view of the
1759        //    file mapping via `MapViewOfFile3(MEM_REPLACE_PLACEHOLDER)`.
1760        let view = middle.map_file_view(raw_handle)?;
1761
1762        #[allow(clippy::arc_with_non_send_sync)]
1763        Ok(Arc::new(HostMapping {
1764            mapping: WindowsMapping::FileBacked {
1765                leading,
1766                view,
1767                trailing,
1768                file_mapping,
1769            },
1770        }))
1771    }
1772
1773    pub(crate) fn as_slice(&self) -> &[u8] {
1774        unsafe { std::slice::from_raw_parts(self.base_ptr(), self.mem_size()) }
1775    }
1776
1777    #[cfg(unshared_snapshot_mem)]
1778    pub(crate) fn copy_to_writable(&self) -> Result<ExclusiveSharedMemory> {
1779        let mut writable = ExclusiveSharedMemory::new(self.guest_mapped_size)?;
1780        writable.copy_from_slice(&self.as_slice()[..self.guest_mapped_size], 0)?;
1781        Ok(writable)
1782    }
1783
1784    #[cfg(not(unshared_snapshot_mem))]
1785    pub(crate) fn copy_to_slice(&self, slice: &mut [u8], offset: usize) -> Result<()> {
1786        bounds_check!(offset, slice.len(), self.mem_size());
1787        slice.copy_from_slice(&self.as_slice()[offset..offset + slice.len()]);
1788        Ok(())
1789    }
1790
1791    #[cfg(not(unshared_snapshot_mem))]
1792    pub(crate) fn mapping_at(
1793        &self,
1794        guest_base: u64,
1795        region_type: MemoryRegionType,
1796    ) -> MemoryRegion {
1797        #[allow(clippy::panic)]
1798        // This will not ever actually panic: the only place this is
1799        // called is SnapshotMemoryBacking::mappings, which
1800        // always calls it with the Snapshot region type.
1801        if region_type != MemoryRegionType::Snapshot {
1802            panic!("ReadonlySharedMemory::mapping_at should only be used for Snapshot regions");
1803        }
1804        mapping_at(
1805            self,
1806            guest_base,
1807            self.guest_mapped_size(),
1808            region_type,
1809            MemoryRegionFlags::READ | MemoryRegionFlags::EXECUTE,
1810        )
1811    }
1812}
1813
1814impl SharedMemory for ReadonlySharedMemory {
1815    fn region(&self) -> &HostMapping {
1816        &self.region
1817    }
1818    // Trait defaults work for `base_addr`, `base_ptr`, and
1819    // `mem_size`: each `ReadonlySharedMemory` has the
1820    // `[guard][blob][guard]` layout.
1821    //
1822    // `host_region_base` differs per Windows mapping flavour:
1823    //  * `WindowsMapping::Anonymous` (`from_bytes`): file mapping
1824    //    spans the whole region. Same as the trait default.
1825    //  * `WindowsMapping::FileBacked` (`from_file`): file mapping
1826    //    covers only the blob. Expose only the blob to the surrogate.
1827    #[cfg(windows)]
1828    fn host_region_base(&self) -> <HostGuestMemoryRegion as MemoryRegionKind>::HostBaseType {
1829        match &self.region().mapping {
1830            WindowsMapping::Anonymous { .. } => super::memory_region::HostRegionBase {
1831                from_handle: self.region().file_mapping_handle().into(),
1832                handle_base: self.region().ptr() as usize,
1833                handle_size: self.region().size(),
1834                offset: page_size::get(),
1835            },
1836            WindowsMapping::FileBacked { .. } => super::memory_region::HostRegionBase {
1837                from_handle: self.region().file_mapping_handle().into(),
1838                handle_base: self.base_ptr() as usize,
1839                handle_size: self.mem_size(),
1840                offset: 0,
1841            },
1842        }
1843    }
1844    // There's no way to get exclusive (and therefore writable) access
1845    // to a ReadonlySharedMemory.
1846    fn with_exclusivity<T, F: FnOnce(&mut ExclusiveSharedMemory) -> T>(
1847        &mut self,
1848        _: F,
1849    ) -> Result<T> {
1850        Err(SharedMemoryError::ReadonlySharedMemoryExclusiveRequest)
1851    }
1852    // However, just access to the contents as a slice is doable
1853    fn with_contents<T, F: FnOnce(&[u8]) -> T>(&mut self, f: F) -> Result<T> {
1854        Ok(f(self.as_slice()))
1855    }
1856}
1857
1858impl<S: SharedMemory> PartialEq<S> for ReadonlySharedMemory {
1859    fn eq(&self, other: &S) -> bool {
1860        self.raw_ptr() == other.raw_ptr()
1861    }
1862}
1863
1864#[cfg(test)]
1865mod tests {
1866    use std::sync::atomic::{AtomicI32, AtomicU16, Ordering};
1867
1868    #[cfg(not(miri))]
1869    use proptest::prelude::*;
1870
1871    #[cfg(not(miri))]
1872    use super::HostSharedMemory;
1873    use super::{ExclusiveSharedMemory, Result, SharedMemory, SharedMemoryError};
1874    #[cfg(not(miri))]
1875    use crate::mem::shared_mem_tests::read_write_test_suite;
1876
1877    #[test]
1878    fn fill() {
1879        let mem_size: usize = page_size::get();
1880        let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
1881        let (mut hshm, _) = eshm.build();
1882
1883        hshm.fill(1, 0, 1024).unwrap();
1884        hshm.fill(2, 1024, 1024).unwrap();
1885        hshm.fill(3, 2048, 1024).unwrap();
1886        hshm.fill(4, 3072, 1024).unwrap();
1887
1888        let vec = hshm
1889            .with_exclusivity(|e| e.copy_all_to_vec().unwrap())
1890            .unwrap();
1891
1892        assert!(vec[0..1024].iter().all(|&x| x == 1));
1893        assert!(vec[1024..2048].iter().all(|&x| x == 2));
1894        assert!(vec[2048..3072].iter().all(|&x| x == 3));
1895        assert!(vec[3072..4096].iter().all(|&x| x == 4));
1896
1897        hshm.fill(5, 0, mem_size).unwrap();
1898
1899        let vec2 = hshm
1900            .with_exclusivity(|e| e.copy_all_to_vec().unwrap())
1901            .unwrap();
1902        assert!(vec2.iter().all(|&x| x == 5));
1903
1904        assert!(hshm.fill(0, 0, mem_size + 1).is_err());
1905        assert!(hshm.fill(0, mem_size, 1).is_err());
1906    }
1907
1908    /// Verify that `bounds_check!` rejects offset + size combinations that
1909    /// would overflow `usize`.
1910    #[test]
1911    fn bounds_check_overflow() {
1912        let mem_size: usize = page_size::get();
1913        let mut eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
1914
1915        // ExclusiveSharedMemory methods
1916        assert!(eshm.read_i32(usize::MAX).is_err());
1917        assert!(eshm.write_i32(usize::MAX, 0).is_err());
1918        assert!(eshm.copy_from_slice(&[0u8; 1], usize::MAX).is_err());
1919
1920        // HostSharedMemory methods
1921        let (mut hshm, _) = eshm.build();
1922
1923        assert!(hshm.read::<u8>(usize::MAX).is_err());
1924        assert!(hshm.read::<u64>(usize::MAX - 3).is_err());
1925        assert!(hshm.write::<u8>(usize::MAX, 0).is_err());
1926        assert!(hshm.write::<u64>(usize::MAX - 3, 0).is_err());
1927
1928        let mut buf = [0u8; 1];
1929        assert!(hshm.copy_to_slice(&mut buf, usize::MAX).is_err());
1930        assert!(hshm.copy_from_slice(&[0u8; 1], usize::MAX).is_err());
1931
1932        assert!(hshm.fill(0, usize::MAX, 1).is_err());
1933        assert!(hshm.fill(0, 1, usize::MAX).is_err());
1934    }
1935
1936    #[test]
1937    fn atomic_access() {
1938        let mem_size = page_size::get();
1939        let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
1940        let (hshm, _) = eshm.build();
1941
1942        hshm.store_atomic::<AtomicU16>(0, 0x1234, Ordering::Release)
1943            .unwrap();
1944        assert_eq!(
1945            hshm.load_atomic::<AtomicU16>(0, Ordering::Acquire).unwrap(),
1946            0x1234
1947        );
1948
1949        hshm.store_atomic::<AtomicI32>(4, -42, Ordering::SeqCst)
1950            .unwrap();
1951        assert_eq!(
1952            hshm.load_atomic::<AtomicI32>(4, Ordering::SeqCst).unwrap(),
1953            -42
1954        );
1955
1956        assert!(hshm.load_atomic::<AtomicU16>(1, Ordering::Relaxed).is_err());
1957        assert!(
1958            hshm.load_atomic::<AtomicU16>(mem_size - 1, Ordering::Relaxed)
1959                .is_err()
1960        );
1961        assert!(hshm.load_atomic::<AtomicU16>(0, Ordering::Release).is_err());
1962        assert!(
1963            hshm.store_atomic::<AtomicU16>(0, 0, Ordering::Acquire)
1964                .is_err()
1965        );
1966    }
1967
1968    #[test]
1969    fn atomic_access_observes_exclusivity() {
1970        let eshm = ExclusiveSharedMemory::new(page_size::get()).unwrap();
1971        let (mut hshm, _) = eshm.build();
1972        let other = hshm.clone();
1973
1974        hshm.with_exclusivity(|_| {
1975            assert!(
1976                other
1977                    .load_atomic::<AtomicU16>(0, Ordering::Relaxed)
1978                    .is_err()
1979            );
1980        })
1981        .unwrap();
1982    }
1983
1984    #[test]
1985    fn freeze_transfers_mapping_without_copying() {
1986        let page_size = page_size::get();
1987        let mut memory = ExclusiveSharedMemory::new(2 * page_size).unwrap();
1988        memory.copy_from_slice(&[1, 2, 3, 4], page_size).unwrap();
1989        let base_addr = memory.base_addr();
1990
1991        let frozen = memory.freeze(page_size).unwrap();
1992
1993        assert_eq!(frozen.base_addr(), base_addr);
1994        assert_eq!(frozen.guest_mapped_size(), page_size);
1995        assert_eq!(&frozen.as_slice()[page_size..page_size + 4], &[1, 2, 3, 4]);
1996    }
1997
1998    #[test]
1999    fn freeze_rejects_writable_aliases() {
2000        let memory = ExclusiveSharedMemory::new(page_size::get()).unwrap();
2001        let alias = ExclusiveSharedMemory {
2002            region: memory.region.clone(),
2003        };
2004
2005        assert!(matches!(
2006            memory.freeze(0),
2007            Err(SharedMemoryError::SharedMemoryNotExclusive)
2008        ));
2009        drop(alias);
2010    }
2011
2012    #[test]
2013    fn freeze_rejects_invalid_guest_mapped_size() {
2014        let page_size = page_size::get();
2015        let freeze = |size| ExclusiveSharedMemory::new(page_size).unwrap().freeze(size);
2016
2017        assert!(freeze(page_size + 1).is_err());
2018        assert!(freeze(2 * page_size).is_err());
2019    }
2020
2021    #[test]
2022    fn copy_into_from() -> Result<()> {
2023        let mem_size: usize = page_size::get();
2024        let vec_len = 10;
2025        let eshm = ExclusiveSharedMemory::new(mem_size)?;
2026        let (hshm, _) = eshm.build();
2027        let vec = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
2028        // write the value to the memory at the beginning.
2029        hshm.copy_from_slice(&vec, 0)?;
2030
2031        let mut vec2 = vec![0; vec_len];
2032        // read the value back from the memory at the beginning.
2033        hshm.copy_to_slice(vec2.as_mut_slice(), 0)?;
2034        assert_eq!(vec, vec2);
2035
2036        let offset = mem_size - vec.len();
2037        // write the value to the memory at the end.
2038        hshm.copy_from_slice(&vec, offset)?;
2039
2040        let mut vec3 = vec![0; vec_len];
2041        // read the value back from the memory at the end.
2042        hshm.copy_to_slice(&mut vec3, offset)?;
2043        assert_eq!(vec, vec3);
2044
2045        let offset = mem_size / 2;
2046        // write the value to the memory at the middle.
2047        hshm.copy_from_slice(&vec, offset)?;
2048
2049        let mut vec4 = vec![0; vec_len];
2050        // read the value back from the memory at the middle.
2051        hshm.copy_to_slice(&mut vec4, offset)?;
2052        assert_eq!(vec, vec4);
2053
2054        // try and read a value from an offset that is beyond the end of the memory.
2055        let mut vec5 = vec![0; vec_len];
2056        assert!(hshm.copy_to_slice(&mut vec5, mem_size).is_err());
2057
2058        // try and write a value to an offset that is beyond the end of the memory.
2059        assert!(hshm.copy_from_slice(&vec5, mem_size).is_err());
2060
2061        // try and read a value from an offset that is too large.
2062        let mut vec6 = vec![0; vec_len];
2063        assert!(hshm.copy_to_slice(&mut vec6, mem_size * 2).is_err());
2064
2065        // try and write a value to an offset that is too large.
2066        assert!(hshm.copy_from_slice(&vec6, mem_size * 2).is_err());
2067
2068        // try and read a value that is too large.
2069        let mut vec7 = vec![0; mem_size * 2];
2070        assert!(hshm.copy_to_slice(&mut vec7, 0).is_err());
2071
2072        // try and write a value that is too large.
2073        assert!(hshm.copy_from_slice(&vec7, 0).is_err());
2074
2075        Ok(())
2076    }
2077
2078    // proptest uses file I/O (getcwd, open) which miri doesn't support
2079    #[cfg(not(miri))]
2080    proptest! {
2081        #[test]
2082        fn read_write_i32(val in -0x1000_i32..0x1000_i32) {
2083            read_write_test_suite(
2084                val,
2085                ExclusiveSharedMemory::new,
2086                Box::new(ExclusiveSharedMemory::read_i32),
2087                Box::new(ExclusiveSharedMemory::write_i32),
2088            )
2089            .unwrap();
2090            read_write_test_suite(
2091                val,
2092                |s| {
2093                    let e = ExclusiveSharedMemory::new(s)?;
2094                    let (h, _) = e.build();
2095                    Ok(h)
2096                },
2097                Box::new(HostSharedMemory::read::<i32>),
2098                Box::new(|h, o, v| h.write::<i32>(o, v)),
2099            )
2100            .unwrap();
2101        }
2102    }
2103
2104    #[test]
2105    fn alloc_fail() {
2106        let gm = ExclusiveSharedMemory::new(0);
2107        assert!(gm.is_err());
2108        let gm = ExclusiveSharedMemory::new(usize::MAX);
2109        assert!(gm.is_err());
2110    }
2111
2112    #[test]
2113    fn clone() {
2114        let eshm = ExclusiveSharedMemory::new(page_size::get()).unwrap();
2115        let (hshm1, _) = eshm.build();
2116        let hshm2 = hshm1.clone();
2117
2118        // after hshm1 is cloned, hshm1 and hshm2 should have identical
2119        // memory sizes and pointers.
2120        assert_eq!(hshm1.mem_size(), hshm2.mem_size());
2121        assert_eq!(hshm1.base_addr(), hshm2.base_addr());
2122
2123        // we should be able to copy a byte array into both hshm1 and hshm2,
2124        // and have both changes be reflected in all clones
2125        hshm1.copy_from_slice(b"a", 0).unwrap();
2126        hshm2.copy_from_slice(b"b", 1).unwrap();
2127
2128        // at this point, both hshm1 and hshm2 should have
2129        // offset 0 = 'a', offset 1 = 'b'
2130        for (raw_offset, expected) in &[(0, b'a'), (1, b'b')] {
2131            assert_eq!(hshm1.read::<u8>(*raw_offset).unwrap(), *expected);
2132            assert_eq!(hshm2.read::<u8>(*raw_offset).unwrap(), *expected);
2133        }
2134
2135        // after we drop hshm1, hshm2 should still exist, be valid,
2136        // and have all contents from before hshm1 was dropped
2137        drop(hshm1);
2138
2139        // at this point, hshm2 should still have offset 0 = 'a', offset 1 = 'b'
2140        for (raw_offset, expected) in &[(0, b'a'), (1, b'b')] {
2141            assert_eq!(hshm2.read::<u8>(*raw_offset).unwrap(), *expected);
2142        }
2143        hshm2.copy_from_slice(b"c", 2).unwrap();
2144        assert_eq!(hshm2.read::<u8>(2).unwrap(), b'c');
2145        drop(hshm2);
2146    }
2147
2148    #[test]
2149    fn copy_all_to_vec() {
2150        let mut data = vec![b'a', b'b', b'c'];
2151        data.resize(page_size::get(), 0);
2152        let mut eshm = ExclusiveSharedMemory::new(data.len()).unwrap();
2153        eshm.copy_from_slice(data.as_slice(), 0).unwrap();
2154        let ret_vec = eshm.copy_all_to_vec().unwrap();
2155        assert_eq!(data, ret_vec);
2156    }
2157
2158    /// Test that verifies memory is properly unmapped when all SharedMemory
2159    /// references are dropped.
2160    #[test]
2161    #[cfg(all(target_os = "linux", not(miri)))]
2162    fn test_drop() {
2163        use proc_maps::get_process_maps;
2164
2165        // Use a unique size that no other test uses to avoid false positives
2166        // from concurrent tests allocating at the same address.
2167        // The mprotect calls split the mapping into 3 regions (guard, usable, guard),
2168        // so we check for the usable region which has this exact size.
2169        //
2170        // NOTE: If this test fails intermittently, there may be a race condition
2171        // where another test allocates memory at the same address between our
2172        // drop and the mapping check. Ensure UNIQUE_SIZE is not used by any
2173        // other test in the codebase to avoid this.
2174        let unique_size: usize = page_size::get() * 17;
2175
2176        let pid = std::process::id();
2177
2178        let eshm = ExclusiveSharedMemory::new(unique_size).unwrap();
2179        let (hshm1, gshm) = eshm.build();
2180        let hshm2 = hshm1.clone();
2181
2182        // Use the usable memory region (not raw), since mprotect splits the mapping
2183        let base_ptr = hshm1.base_ptr() as usize;
2184        let mem_size = hshm1.mem_size();
2185
2186        // Helper to check if exact mapping exists (matching both address and size)
2187        let has_exact_mapping = |ptr: usize, size: usize| -> bool {
2188            get_process_maps(pid.try_into().unwrap())
2189                .unwrap()
2190                .iter()
2191                .any(|m| m.start() == ptr && m.size() == size)
2192        };
2193
2194        // Verify mapping exists before drop
2195        assert!(
2196            has_exact_mapping(base_ptr, mem_size),
2197            "shared memory mapping not found at {:#x} with size {}",
2198            base_ptr,
2199            mem_size
2200        );
2201
2202        // Drop all references
2203        drop(hshm1);
2204        drop(hshm2);
2205        drop(gshm);
2206
2207        // Verify exact mapping is gone
2208        assert!(
2209            !has_exact_mapping(base_ptr, mem_size),
2210            "shared memory mapping still exists at {:#x} with size {} after drop",
2211            base_ptr,
2212            mem_size
2213        );
2214    }
2215
2216    /// Tests for the optimized aligned memory operations.
2217    /// These tests verify that the u128 chunk optimization works correctly
2218    /// for various alignment scenarios and buffer sizes.
2219    mod alignment_tests {
2220        use super::*;
2221
2222        const CHUNK_SIZE: usize = size_of::<u128>();
2223
2224        /// Test copy operations with all possible starting alignment offsets (0-15)
2225        #[test]
2226        fn copy_with_various_alignments() {
2227            // Use a buffer large enough to test all alignment cases
2228            let mem_size: usize = page_size::get();
2229            let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2230            let (hshm, _) = eshm.build();
2231
2232            // Test all 16 possible alignment offsets (0 through 15)
2233            for start_offset in 0..CHUNK_SIZE {
2234                let test_len = 64; // Enough to cover head, aligned chunks, and tail
2235                let test_data: Vec<u8> = (0..test_len).map(|i| (i + start_offset) as u8).collect();
2236
2237                // Write data at the given offset
2238                hshm.copy_from_slice(&test_data, start_offset).unwrap();
2239
2240                // Read it back
2241                let mut read_buf = vec![0u8; test_len];
2242                hshm.copy_to_slice(&mut read_buf, start_offset).unwrap();
2243
2244                assert_eq!(
2245                    test_data, read_buf,
2246                    "Mismatch at alignment offset {}",
2247                    start_offset
2248                );
2249            }
2250        }
2251
2252        /// Test copy operations with lengths smaller than chunk size (< 16 bytes)
2253        #[test]
2254        fn copy_small_lengths() {
2255            let mem_size: usize = page_size::get();
2256            let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2257            let (hshm, _) = eshm.build();
2258
2259            for len in 0..CHUNK_SIZE {
2260                let test_data: Vec<u8> = (0..len).map(|i| i as u8).collect();
2261
2262                hshm.copy_from_slice(&test_data, 0).unwrap();
2263
2264                let mut read_buf = vec![0u8; len];
2265                hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2266
2267                assert_eq!(test_data, read_buf, "Mismatch for length {}", len);
2268            }
2269        }
2270
2271        /// Test copy operations with lengths that don't align to chunk boundaries
2272        #[test]
2273        fn copy_non_aligned_lengths() {
2274            let mem_size: usize = page_size::get();
2275            let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2276            let (hshm, _) = eshm.build();
2277
2278            // Test lengths like 17, 31, 33, 47, 63, 65, etc.
2279            let test_lengths = [17, 31, 33, 47, 63, 65, 100, 127, 129, 255, 257];
2280
2281            for &len in &test_lengths {
2282                let test_data: Vec<u8> = (0..len).map(|i| (i % 256) as u8).collect();
2283
2284                hshm.copy_from_slice(&test_data, 0).unwrap();
2285
2286                let mut read_buf = vec![0u8; len];
2287                hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2288
2289                assert_eq!(test_data, read_buf, "Mismatch for length {}", len);
2290            }
2291        }
2292
2293        /// Test copy with exactly one chunk (16 bytes)
2294        #[test]
2295        fn copy_exact_chunk_size() {
2296            let mem_size: usize = page_size::get();
2297            let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2298            let (hshm, _) = eshm.build();
2299
2300            let test_data: Vec<u8> = (0..CHUNK_SIZE).map(|i| i as u8).collect();
2301
2302            hshm.copy_from_slice(&test_data, 0).unwrap();
2303
2304            let mut read_buf = vec![0u8; CHUNK_SIZE];
2305            hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2306
2307            assert_eq!(test_data, read_buf);
2308        }
2309
2310        /// Test fill with various alignment offsets
2311        #[test]
2312        fn fill_with_various_alignments() {
2313            let mem_size: usize = page_size::get();
2314            let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2315            let (mut hshm, _) = eshm.build();
2316
2317            for start_offset in 0..CHUNK_SIZE {
2318                let fill_len = 64;
2319                let fill_value = (start_offset % 256) as u8;
2320
2321                // Clear memory first
2322                hshm.fill(0, 0, mem_size).unwrap();
2323
2324                // Fill at the given offset
2325                hshm.fill(fill_value, start_offset, fill_len).unwrap();
2326
2327                // Read it back and verify
2328                let mut read_buf = vec![0u8; fill_len];
2329                hshm.copy_to_slice(&mut read_buf, start_offset).unwrap();
2330
2331                assert!(
2332                    read_buf.iter().all(|&b| b == fill_value),
2333                    "Fill mismatch at alignment offset {}",
2334                    start_offset
2335                );
2336            }
2337        }
2338
2339        /// Test fill with lengths smaller than chunk size
2340        #[test]
2341        fn fill_small_lengths() {
2342            let mem_size: usize = page_size::get();
2343            let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2344            let (mut hshm, _) = eshm.build();
2345
2346            for len in 0..CHUNK_SIZE {
2347                let fill_value = 0xAB;
2348
2349                hshm.fill(0, 0, mem_size).unwrap(); // Clear
2350                hshm.fill(fill_value, 0, len).unwrap();
2351
2352                let mut read_buf = vec![0u8; len];
2353                hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2354
2355                assert!(
2356                    read_buf.iter().all(|&b| b == fill_value),
2357                    "Fill mismatch for length {}",
2358                    len
2359                );
2360            }
2361        }
2362
2363        /// Test fill with non-aligned lengths
2364        #[test]
2365        fn fill_non_aligned_lengths() {
2366            let mem_size: usize = page_size::get();
2367            let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2368            let (mut hshm, _) = eshm.build();
2369
2370            let test_lengths = [17, 31, 33, 47, 63, 65, 100, 127, 129, 255, 257];
2371
2372            for &len in &test_lengths {
2373                let fill_value = 0xCD;
2374
2375                hshm.fill(0, 0, mem_size).unwrap(); // Clear
2376                hshm.fill(fill_value, 0, len).unwrap();
2377
2378                let mut read_buf = vec![0u8; len];
2379                hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2380
2381                assert!(
2382                    read_buf.iter().all(|&b| b == fill_value),
2383                    "Fill mismatch for length {}",
2384                    len
2385                );
2386            }
2387        }
2388
2389        /// Test edge cases: length 0 and length 1
2390        #[test]
2391        fn copy_edge_cases() {
2392            let mem_size: usize = page_size::get();
2393            let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2394            let (hshm, _) = eshm.build();
2395
2396            // Length 0
2397            let empty: Vec<u8> = vec![];
2398            hshm.copy_from_slice(&empty, 0).unwrap();
2399            let mut read_buf: Vec<u8> = vec![];
2400            hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2401            assert!(read_buf.is_empty());
2402
2403            // Length 1
2404            let single = vec![0x42u8];
2405            hshm.copy_from_slice(&single, 0).unwrap();
2406            let mut read_buf = vec![0u8; 1];
2407            hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2408            assert_eq!(single, read_buf);
2409        }
2410
2411        /// Test combined: unaligned start + non-aligned length
2412        #[test]
2413        fn copy_unaligned_start_and_length() {
2414            let mem_size: usize = page_size::get();
2415            let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2416            let (hshm, _) = eshm.build();
2417
2418            // Start at offset 7 (unaligned), length 37 (not a multiple of 16)
2419            let start_offset = 7;
2420            let len = 37;
2421            let test_data: Vec<u8> = (0..len).map(|i| (i * 3) as u8).collect();
2422
2423            hshm.copy_from_slice(&test_data, start_offset).unwrap();
2424
2425            let mut read_buf = vec![0u8; len];
2426            hshm.copy_to_slice(&mut read_buf, start_offset).unwrap();
2427
2428            assert_eq!(test_data, read_buf);
2429        }
2430    }
2431
2432    /// Shared by the guard page shims. The ignored tests crash on
2433    /// purpose, so each one runs in its own process.
2434    #[cfg(not(miri))]
2435    mod guard_page_subprocess {
2436        /// Derive a libtest filter path for a test function from its
2437        /// type name. Strips the leading crate-name segment that
2438        /// `type_name` includes but libtest does not.
2439        pub(super) fn test_path<F: Fn()>(_: F) -> &'static str {
2440            let full = std::any::type_name::<F>();
2441            let (_, rest) = full
2442                .split_once("::")
2443                .expect("type_name of a function item is always qualified by the crate name");
2444            rest
2445        }
2446
2447        pub(super) fn run_guard_page_subprocess(exe: &std::path::Path, ignored_test_path: &str) {
2448            let output = std::process::Command::new(exe)
2449                .args([
2450                    "--ignored",
2451                    "--nocapture",
2452                    "--exact",
2453                    "--test-threads=1",
2454                    ignored_test_path,
2455                ])
2456                .stdin(std::process::Stdio::null())
2457                .output()
2458                .expect("Unable to launch subprocess test");
2459
2460            let stdout = String::from_utf8_lossy(&output.stdout);
2461            let stderr = String::from_utf8_lossy(&output.stderr);
2462
2463            // libtest with no matching filter exits 0, so verify the
2464            // test actually ran via the "running 1 test" banner.
2465            let ran_test = stdout.contains("running 1 test");
2466            let reached = stdout.contains("reached_guard");
2467            let survived = stdout.contains("survived_guard");
2468            let by_access_violation = killed_by_access_violation(&output.status);
2469
2470            let ok = reached && !survived && by_access_violation && ran_test;
2471            if !ok {
2472                eprintln!("=== Guard page shim failed for {} ===", ignored_test_path);
2473                eprintln!(
2474                    "status={:?} ran_test={} reached={} survived={} by_access_violation={}",
2475                    output.status, ran_test, reached, survived, by_access_violation
2476                );
2477                eprintln!("=== STDOUT ===\n{}", stdout);
2478                eprintln!("=== STDERR ===\n{}", stderr);
2479                let hint = if !ran_test {
2480                    format!(
2481                        "\nHINT: ran_test=false (subprocess reported 'running 0 tests'). \
2482                         Most likely cause is a stale test path in the shim. Verify that \
2483                         `{}` still exists and matches the path passed via --exact above.",
2484                        ignored_test_path
2485                    )
2486                } else {
2487                    String::new()
2488                };
2489                panic!(
2490                    "Expected subprocess to run {}, print 'reached_guard', \
2491                     then die from a memory access fault. ran_test={}, reached={}, \
2492                     survived={}, by_access_violation={}, status={:?}{}",
2493                    ignored_test_path,
2494                    ran_test,
2495                    reached,
2496                    survived,
2497                    by_access_violation,
2498                    output.status,
2499                    hint
2500                );
2501            }
2502
2503            println!(
2504                "guard page trap confirmed for {}: subprocess terminated with {:?}",
2505                ignored_test_path, output.status
2506            );
2507        }
2508
2509        /// Returns true if `status` indicates the process died from a
2510        /// memory access fault (SIGBUS on macos, SIGSEGV on linux,
2511        /// STATUS_ACCESS_VIOLATION (or 0xDEAD) on Windows).
2512        fn killed_by_access_violation(status: &std::process::ExitStatus) -> bool {
2513            #[cfg(unix)]
2514            {
2515                use std::os::unix::process::ExitStatusExt;
2516                let expected_signal = if cfg!(target_os = "macos") {
2517                    libc::SIGBUS
2518                } else {
2519                    libc::SIGSEGV
2520                };
2521                status.signal() == Some(expected_signal)
2522            }
2523            #[cfg(windows)]
2524            {
2525                use windows::Win32::Foundation::STATUS_ACCESS_VIOLATION;
2526                // See https://github.com/hyperlight-dev/hyperlight/issues/1507
2527                status.code() == Some(STATUS_ACCESS_VIOLATION.0) || status.code() == Some(0xDEAD)
2528            }
2529        }
2530    }
2531
2532    #[cfg(all(not(miri), any(target_os = "linux", windows)))]
2533    mod guard_page_crash_test {
2534        use super::guard_page_subprocess::{run_guard_page_subprocess, test_path};
2535        use crate::mem::shared_mem::{ExclusiveSharedMemory, SharedMemory};
2536
2537        #[test]
2538        #[ignore] // this test is ignored because it will crash the running process
2539        fn read() {
2540            let eshm = ExclusiveSharedMemory::new(page_size::get()).unwrap();
2541            let (hshm, _) = eshm.build();
2542            let guard_page_ptr = hshm.raw_ptr();
2543            println!("reached_guard");
2544            unsafe { std::ptr::read_volatile(guard_page_ptr) };
2545            println!("survived_guard");
2546        }
2547
2548        #[test]
2549        #[ignore] // this test is ignored because it will crash the running process
2550        fn write() {
2551            let eshm = ExclusiveSharedMemory::new(page_size::get()).unwrap();
2552            let (hshm, _) = eshm.build();
2553            let guard_page_ptr = hshm.raw_ptr();
2554            println!("reached_guard");
2555            unsafe { std::ptr::write_volatile(guard_page_ptr, 0u8) };
2556            println!("survived_guard");
2557        }
2558
2559        #[test]
2560        #[ignore] // this test is ignored because it will crash the running process
2561        fn exec() {
2562            let eshm = ExclusiveSharedMemory::new(page_size::get()).unwrap();
2563            let (hshm, _) = eshm.build();
2564            let guard_page_ptr = hshm.raw_ptr();
2565            let func: fn() = unsafe { std::mem::transmute(guard_page_ptr) };
2566            println!("reached_guard");
2567            func();
2568            println!("survived_guard");
2569        }
2570
2571        // provides a way for running the above tests in a separate process since they expect to crash
2572        #[test]
2573        fn guard_page_testing_shim() {
2574            let exe = std::env::current_exe().expect("current_exe");
2575            for path in [test_path(read), test_path(write), test_path(exec)] {
2576                run_guard_page_subprocess(&exe, path);
2577            }
2578        }
2579    }
2580
2581    #[cfg(not(miri))]
2582    mod from_file_tests {
2583        use std::io::Write;
2584
2585        use tempfile::NamedTempFile;
2586
2587        use crate::mem::shared_mem::{ReadonlySharedMemory, SharedMemory};
2588
2589        pub(super) fn make_temp_file(len: usize) -> NamedTempFile {
2590            let mut f = NamedTempFile::new().expect("create temp file");
2591            if len > 0 {
2592                let mut buf = vec![0u8; len];
2593                for (i, b) in buf.iter_mut().enumerate() {
2594                    *b = (i & 0xff) as u8;
2595                }
2596                f.write_all(&buf).expect("write temp file");
2597                f.flush().expect("flush temp file");
2598            }
2599            f
2600        }
2601
2602        #[test]
2603        fn from_file_success_single_page() {
2604            let tmp = make_temp_file(page_size::get());
2605            let mut rsm = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get())
2606                .expect("from_file should succeed");
2607            assert_eq!(rsm.mem_size(), page_size::get());
2608            rsm.with_contents(|slice| {
2609                for (i, b) in slice.iter().enumerate() {
2610                    assert_eq!(*b, (i & 0xff) as u8);
2611                }
2612            })
2613            .expect("with_contents should succeed");
2614        }
2615
2616        #[test]
2617        fn from_file_success_smaller_guest_mapped_size() {
2618            let tmp = make_temp_file(2 * page_size::get());
2619            let rsm = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get())
2620                .expect("from_file should succeed");
2621            assert_eq!(rsm.mem_size(), 2 * page_size::get());
2622        }
2623
2624        #[test]
2625        fn from_file_rejects_empty_file() {
2626            let tmp = make_temp_file(0);
2627            let err = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get())
2628                .expect_err("empty file should be rejected");
2629            assert!(format!("{}", err).contains("0x0 < 0x0"));
2630        }
2631
2632        #[test]
2633        fn from_file_rejects_unaligned_file_length() {
2634            let tmp = make_temp_file(page_size::get() + 1);
2635            let err = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get())
2636                .expect_err("unaligned file length should be rejected");
2637            assert!(format!("{}", err).contains(&format!(
2638                "0x{:x} % 0x{:x} = 0",
2639                page_size::get() + 1,
2640                page_size::get()
2641            )));
2642        }
2643
2644        #[test]
2645        fn from_file_rejects_unaligned_guest_mapped_size() {
2646            let tmp = make_temp_file(2 * page_size::get());
2647            let err = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get() + 1)
2648                .expect_err("unaligned guest_mapped_size should be rejected");
2649            assert!(format!("{}", err).contains(&format!(
2650                "0x{:x} % 0x{:x} = ",
2651                page_size::get() + 1,
2652                page_size::get()
2653            )));
2654        }
2655
2656        #[test]
2657        fn from_file_rejects_guest_mapped_size_exceeding_file() {
2658            let tmp = make_temp_file(page_size::get());
2659            let err = ReadonlySharedMemory::from_file(tmp.as_file(), 2 * page_size::get())
2660                .expect_err("guest_mapped_size > file length should be rejected");
2661            assert!(format!("{}", err).contains(&format!(
2662                "0x{:x} <= 0x{:x}",
2663                2 * page_size::get(),
2664                page_size::get()
2665            )));
2666        }
2667
2668        /// Tests in this submodule are `#[ignore]`'d because each one
2669        /// is expected to die from a memory access violation. They are
2670        /// not run by a plain `cargo test`. The `from_file_guard_page_shim`
2671        /// test in the parent module re-executes each one in a subprocess
2672        /// and asserts the trap occurred.
2673        mod guard_page_crash_tests {
2674            use super::make_temp_file;
2675            use crate::mem::shared_mem::{ReadonlySharedMemory, SharedMemory};
2676
2677            /// Loads from the byte immediately before the mapping.
2678            #[test]
2679            #[ignore]
2680            pub(super) fn leading_guard_page_traps() {
2681                let tmp = make_temp_file(page_size::get());
2682                let rsm = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get())
2683                    .expect("from_file should succeed");
2684                let guard_ptr = unsafe { rsm.base_ptr().sub(page_size::get()) };
2685                println!("reached_guard");
2686                let _ = unsafe { std::ptr::read_volatile(guard_ptr) };
2687                println!("survived_guard");
2688            }
2689
2690            /// Loads from the byte immediately after the mapping.
2691            #[test]
2692            #[ignore]
2693            pub(super) fn trailing_guard_page_traps() {
2694                let tmp = make_temp_file(page_size::get());
2695                let rsm = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get())
2696                    .expect("from_file should succeed");
2697                let guard_ptr = unsafe { rsm.base_ptr().add(rsm.mem_size()) };
2698                println!("reached_guard");
2699                let _ = unsafe { std::ptr::read_volatile(guard_ptr) };
2700                println!("survived_guard");
2701            }
2702        }
2703
2704        /// Spawn each ignored guard-page test as a subprocess and assert
2705        /// it terminated by an OS access violation. Re-executes the
2706        /// current test binary so no rebuild or `cargo` invocation is
2707        /// needed.
2708        #[test]
2709        #[cfg_attr(miri, ignore)] // miri can't spawn subprocesses
2710        fn from_file_guard_page_shim() {
2711            use guard_page_crash_tests::{leading_guard_page_traps, trailing_guard_page_traps};
2712
2713            use super::guard_page_subprocess::{run_guard_page_subprocess, test_path};
2714            let ignored_test_paths = [
2715                test_path(leading_guard_page_traps),
2716                test_path(trailing_guard_page_traps),
2717            ];
2718
2719            let exe = std::env::current_exe().expect("current_exe");
2720            for path in &ignored_test_paths {
2721                run_guard_page_subprocess(&exe, path);
2722            }
2723        }
2724    }
2725}