1use std::ffi::c_void;
5use std::io::Error;
6use std::mem::{align_of, size_of};
7#[cfg(unix)]
8use std::ptr::null_mut;
9use std::sync::atomic::Ordering;
10use std::sync::{Arc, RwLock};
11
12use bytemuck::Pod;
13use thiserror::Error;
14use tracing::{Span, instrument};
15#[cfg(target_os = "windows")]
16use windows::Win32::Foundation::{CloseHandle, HANDLE, INVALID_HANDLE_VALUE};
17#[cfg(target_os = "windows")]
18use windows::Win32::System::Memory::PAGE_READWRITE;
19#[cfg(target_os = "windows")]
20use windows::Win32::System::Memory::{
21 CreateFileMappingA, FILE_MAP_ALL_ACCESS, MEM_PRESERVE_PLACEHOLDER, MEM_RELEASE,
22 MEM_REPLACE_PLACEHOLDER, MEM_RESERVE, MEM_RESERVE_PLACEHOLDER, MEMORY_MAPPED_VIEW_ADDRESS,
23 MapViewOfFile, MapViewOfFile3, PAGE_NOACCESS, PAGE_PROTECTION_FLAGS, PAGE_READONLY,
24 UnmapViewOfFile, VIRTUAL_ALLOCATION_TYPE, VIRTUAL_FREE_TYPE, VirtualAlloc2, VirtualFree,
25 VirtualProtect,
26};
27#[cfg(target_os = "windows")]
28use windows::core::PCSTR;
29
30use super::memory_region::{
31 HostGuestMemoryRegion, MemoryRegion, MemoryRegionFlags, MemoryRegionKind, MemoryRegionType,
32};
33use crate::log_then_return;
34
35type Result<T> = core::result::Result<T, SharedMemoryError>;
36
37type ThisErrorHackBacktrace = std::backtrace::Backtrace;
41
42#[derive(Error, Debug)]
44pub enum SharedMemoryError {
45 #[error("Cannot access a value with size {0} at offset {1} in memory of size {2}")]
48 Bounds(usize, usize, usize),
49
50 #[error("Atomic access at offset {0} is not aligned to {1} bytes")]
52 AtomicAccessUnaligned(usize, usize),
53
54 #[error("Invalid atomic load ordering: {0:?}")]
56 InvalidAtomicLoadOrdering(Ordering),
57
58 #[error("Invalid atomic store ordering: {0:?}")]
60 InvalidAtomicStoreOrdering(Ordering),
61
62 #[error("Could not access metadata for file: {0}")]
65 FileMetadata(std::io::Error),
66
67 #[error("File size exceeded usize: {0}")]
70 FileTooLarge(std::num::TryFromIntError),
71
72 #[error("Could not acquire memory lock: {0} at {1}")]
75 LockError(String, ThisErrorHackBacktrace),
76
77 #[error("Memory Allocation Failed with OS Error {0:?}.")]
80 MemoryAllocationFailed(Option<i32>),
81
82 #[error(
85 "Memory request does not satisfy constraints: 0x{1:x} < 0x{0:x} <= 0x{2:x} && 0x{0:x} % 0x{3:x} = 0"
86 )]
87 MemoryRequest(usize, usize, usize, usize),
88
89 #[error("mmap failed with os error {0:?}")]
92 MmapFailed(Option<i32>),
93
94 #[error("mprotect failed with os error {0:?}")]
98 MprotectFailed(Option<i32>),
99
100 #[cfg(target_os = "windows")]
102 #[error("Windows API Error Result {0:?}")]
103 WindowsAPIError(#[from] windows_result::Error),
104
105 #[error("Cannot take exclusive access to a ReadonlySharedMemory")]
108 ReadonlySharedMemoryExclusiveRequest,
109
110 #[error("Error reading slice {0}")]
113 TryFromSlice(#[from] std::array::TryFromSliceError),
114
115 #[error("Error reading int: {0}")]
118 TryFromInt(#[from] std::num::TryFromIntError),
119}
120impl<T> From<std::sync::TryLockError<T>> for SharedMemoryError {
121 fn from(e: std::sync::TryLockError<T>) -> SharedMemoryError {
122 SharedMemoryError::LockError(format!("{:?}", e), std::backtrace::Backtrace::capture())
123 }
124}
125
126macro_rules! bounds_check {
128 ($offset:expr, $size:expr, $mem_size:expr) => {
129 if $offset.checked_add($size).is_none_or(|end| end > $mem_size) {
130 return Err(SharedMemoryError::Bounds($offset, $size, $mem_size));
131 }
132 };
133}
134
135mod atomic_access {
136 pub trait Sealed {}
137}
138
139#[allow(private_bounds)]
144pub trait AtomicAccess: atomic_access::Sealed {
145 type Value: Copy;
147
148 #[doc(hidden)]
150 fn load(&self, ordering: Ordering) -> Self::Value;
151
152 #[doc(hidden)]
154 fn store(&self, value: Self::Value, ordering: Ordering);
155}
156
157macro_rules! impl_atomic_access {
158 ($atomic:ty, $value:ty) => {
159 impl atomic_access::Sealed for $atomic {}
160
161 impl AtomicAccess for $atomic {
162 type Value = $value;
163
164 fn load(&self, ordering: Ordering) -> Self::Value {
165 <$atomic>::load(self, ordering)
166 }
167
168 fn store(&self, value: Self::Value, ordering: Ordering) {
169 <$atomic>::store(self, value, ordering);
170 }
171 }
172 };
173}
174
175macro_rules! generate_reader {
177 ($fname:ident, $ty:ty) => {
178 #[allow(dead_code)]
180 #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
181 pub(crate) fn $fname(&self, offset: usize) -> Result<$ty> {
182 let data = self.as_slice();
183 bounds_check!(offset, std::mem::size_of::<$ty>(), data.len());
184 Ok(<$ty>::from_le_bytes(
185 data[offset..offset + std::mem::size_of::<$ty>()].try_into()?,
186 ))
187 }
188 };
189}
190
191macro_rules! generate_writer {
193 ($fname:ident, $ty:ty) => {
194 #[allow(dead_code)]
196 pub(crate) fn $fname(&mut self, offset: usize, value: $ty) -> Result<()> {
197 let data = self.as_mut_slice();
198 bounds_check!(offset, std::mem::size_of::<$ty>(), data.len());
199 data[offset..offset + std::mem::size_of::<$ty>()].copy_from_slice(&value.to_le_bytes());
200 Ok(())
201 }
202 };
203}
204
205impl_atomic_access!(std::sync::atomic::AtomicI8, i8);
206impl_atomic_access!(std::sync::atomic::AtomicI16, i16);
207impl_atomic_access!(std::sync::atomic::AtomicI32, i32);
208impl_atomic_access!(std::sync::atomic::AtomicI64, i64);
209impl_atomic_access!(std::sync::atomic::AtomicIsize, isize);
210impl_atomic_access!(std::sync::atomic::AtomicU8, u8);
211impl_atomic_access!(std::sync::atomic::AtomicU16, u16);
212impl_atomic_access!(std::sync::atomic::AtomicU32, u32);
213impl_atomic_access!(std::sync::atomic::AtomicU64, u64);
214impl_atomic_access!(std::sync::atomic::AtomicUsize, usize);
215
216#[derive(Debug)]
221pub struct HostMapping {
222 #[cfg(not(target_os = "windows"))]
223 mmap: Mmap,
224 #[cfg(target_os = "windows")]
225 mapping: WindowsMapping,
226}
227
228#[cfg(target_os = "windows")]
230#[derive(Debug)]
231enum WindowsMapping {
232 Anonymous {
241 view: MappedView,
242 file_mapping: FileMapping,
243 },
244 FileBacked {
253 leading: Placeholder,
254 view: MappedView,
255 trailing: Placeholder,
256 file_mapping: FileMapping,
257 },
258}
259
260impl HostMapping {
261 pub(crate) fn ptr(&self) -> *mut u8 {
263 #[cfg(not(target_os = "windows"))]
264 {
265 self.mmap.base as *mut u8
266 }
267 #[cfg(target_os = "windows")]
268 match &self.mapping {
269 WindowsMapping::Anonymous { view, .. } => view.addr as *mut u8,
270 WindowsMapping::FileBacked { leading, .. } => leading.addr as *mut u8,
271 }
272 }
273
274 pub(crate) fn size(&self) -> usize {
276 #[cfg(not(target_os = "windows"))]
277 {
278 self.mmap.len
279 }
280 #[cfg(target_os = "windows")]
281 match &self.mapping {
282 WindowsMapping::Anonymous { view, .. } => view.len,
283 WindowsMapping::FileBacked {
284 leading,
285 view,
286 trailing,
287 ..
288 } => leading.size + view.len + trailing.size,
289 }
290 }
291
292 #[cfg(target_os = "windows")]
294 pub(crate) fn file_mapping_handle(&self) -> HANDLE {
295 match &self.mapping {
296 WindowsMapping::Anonymous { file_mapping, .. }
297 | WindowsMapping::FileBacked { file_mapping, .. } => file_mapping.0,
298 }
299 }
300}
301
302#[cfg(unix)]
304#[derive(Debug)]
305struct Mmap {
306 base: *mut c_void,
307 len: usize,
308}
309
310#[cfg(unix)]
311impl Drop for Mmap {
312 fn drop(&mut self) {
313 unsafe {
317 if libc::munmap(self.base, self.len) != 0 {
318 tracing::error!(
319 "Mmap::drop: munmap failed: {:?}",
320 std::io::Error::last_os_error()
321 );
322 }
323 }
324 }
325}
326
327#[cfg(target_os = "windows")]
329#[derive(Debug)]
330struct MappedView {
331 addr: *mut c_void,
332 len: usize,
333}
334
335#[cfg(target_os = "windows")]
336impl Drop for MappedView {
337 fn drop(&mut self) {
338 let view = MEMORY_MAPPED_VIEW_ADDRESS { Value: self.addr };
339 if let Err(e) = unsafe { UnmapViewOfFile(view) } {
348 tracing::error!(
349 "MappedView::drop(addr={:?}, len={}) UnmapViewOfFile failed: {:?}",
350 self.addr,
351 self.len,
352 e
353 );
354 }
355 }
356}
357
358#[cfg(target_os = "windows")]
360#[derive(Debug)]
361struct FileMapping(HANDLE);
362
363#[cfg(target_os = "windows")]
364impl Drop for FileMapping {
365 fn drop(&mut self) {
366 unsafe {
369 if let Err(e) = CloseHandle(self.0) {
370 tracing::error!(
371 "FileMapping::drop(handle={:?}) CloseHandle failed: {:?}",
372 self.0,
373 e
374 );
375 }
376 }
377 }
378}
379
380#[cfg(target_os = "windows")]
385#[derive(Debug)]
386pub(crate) struct Placeholder {
387 addr: *mut c_void,
388 size: usize,
389}
390
391#[cfg(target_os = "windows")]
392impl Placeholder {
393 fn reserve(size: usize) -> Result<Self> {
394 let addr = unsafe {
400 VirtualAlloc2(
401 None,
402 None,
403 size,
404 VIRTUAL_ALLOCATION_TYPE(MEM_RESERVE.0 | MEM_RESERVE_PLACEHOLDER.0),
405 PAGE_NOACCESS.0,
406 None,
407 )
408 };
409 if addr.is_null() {
410 log_then_return!(SharedMemoryError::MemoryAllocationFailed(
411 Error::last_os_error().raw_os_error()
412 ));
413 }
414 Ok(Placeholder { addr, size })
415 }
416
417 fn split_front(self, front_size: usize) -> Result<(Placeholder, Placeholder)> {
418 debug_assert!(front_size > 0 && front_size < self.size);
419 debug_assert!(front_size.is_multiple_of(page_size::get()));
420 if let Err(e) = unsafe {
426 VirtualFree(
427 self.addr,
428 front_size,
429 VIRTUAL_FREE_TYPE(MEM_RELEASE.0 | MEM_PRESERVE_PLACEHOLDER.0),
430 )
431 } {
432 log_then_return!(SharedMemoryError::WindowsAPIError(e.clone()));
434 }
435 let addr = self.addr;
436 let total = self.size;
437 std::mem::forget(self);
440 let front = Placeholder {
441 addr,
442 size: front_size,
443 };
444 let back = Placeholder {
445 addr: unsafe { (addr as *mut u8).add(front_size) as *mut c_void },
448 size: total - front_size,
449 };
450 Ok((front, back))
451 }
452
453 fn split_into_three(
454 self,
455 front_size: usize,
456 middle_size: usize,
457 ) -> Result<(Placeholder, Placeholder, Placeholder)> {
458 let (front, rest) = self.split_front(front_size)?;
459 let (middle, back) = rest.split_front(middle_size)?;
460 Ok((front, middle, back))
461 }
462
463 fn map_file_view(self, file_mapping: HANDLE) -> Result<MappedView> {
464 let mapped = unsafe {
472 MapViewOfFile3(
473 file_mapping,
474 None,
475 Some(self.addr),
476 0,
477 self.size,
478 MEM_REPLACE_PLACEHOLDER,
479 PAGE_READONLY.0,
480 None,
481 )
482 };
483 if mapped.Value.is_null() {
484 log_then_return!(SharedMemoryError::MemoryAllocationFailed(
486 Error::last_os_error().raw_os_error()
487 ));
488 }
489 let addr = self.addr;
490 let len = self.size;
491 std::mem::forget(self);
492 Ok(MappedView { addr, len })
493 }
494}
495
496#[cfg(target_os = "windows")]
497impl Drop for Placeholder {
498 fn drop(&mut self) {
499 if let Err(e) = unsafe { VirtualFree(self.addr, 0, VIRTUAL_FREE_TYPE(MEM_RELEASE.0)) } {
503 tracing::error!(
504 "Placeholder::drop(addr={:?}, size={}) VirtualFree failed: {:?}",
505 self.addr,
506 self.size,
507 e
508 );
509 }
510 }
511}
512
513pub trait SharedMemory {
518 fn region(&self) -> &HostMapping;
520
521 fn base_addr(&self) -> usize {
526 self.region().ptr() as usize + page_size::get()
527 }
528
529 fn base_ptr(&self) -> *mut u8 {
534 self.region().ptr().wrapping_add(page_size::get())
535 }
536
537 fn mem_size(&self) -> usize {
541 self.region().size() - 2 * page_size::get()
542 }
543
544 fn raw_ptr(&self) -> *mut u8 {
547 self.region().ptr()
548 }
549
550 fn raw_mem_size(&self) -> usize {
553 self.region().size()
554 }
555
556 fn host_region_base(&self) -> <HostGuestMemoryRegion as MemoryRegionKind>::HostBaseType {
563 #[cfg(not(windows))]
564 {
565 self.base_addr()
566 }
567 #[cfg(windows)]
568 {
569 super::memory_region::HostRegionBase {
570 from_handle: self.region().file_mapping_handle().into(),
571 handle_base: self.region().ptr() as usize,
572 handle_size: self.region().size(),
573 offset: page_size::get(),
574 }
575 }
576 }
577
578 fn host_region_end(&self) -> <HostGuestMemoryRegion as MemoryRegionKind>::HostBaseType {
580 <HostGuestMemoryRegion as MemoryRegionKind>::add(self.host_region_base(), self.mem_size())
581 }
582
583 fn with_exclusivity<T, F: FnOnce(&mut ExclusiveSharedMemory) -> T>(
589 &mut self,
590 f: F,
591 ) -> Result<T>;
592
593 fn with_contents<T, F: FnOnce(&[u8]) -> T>(&mut self, f: F) -> Result<T> {
599 self.with_exclusivity(|m| f(m.as_slice()))
600 }
601}
602
603fn mapping_at(
604 s: &impl SharedMemory,
605 gpa: u64,
606 size: usize,
607 region_type: MemoryRegionType,
608 flags: MemoryRegionFlags,
609) -> MemoryRegion {
610 let guest_base = gpa as usize;
611
612 MemoryRegion {
613 guest_region: guest_base..(guest_base + size),
614 host_region: s.host_region_base()
615 ..<HostGuestMemoryRegion as MemoryRegionKind>::add(s.host_region_base(), size),
616 region_type,
617 flags,
618 }
619}
620
621#[derive(Debug)]
627pub struct ExclusiveSharedMemory {
628 region: Arc<HostMapping>,
629}
630unsafe impl Send for ExclusiveSharedMemory {}
631
632impl ExclusiveSharedMemory {
633 fn total_size(min_size_bytes: usize) -> Result<usize> {
636 if min_size_bytes > 0 &&
637 let Some(total_size) = min_size_bytes.checked_add(2 * page_size::get()) &&
639 total_size % page_size::get() == 0 &&
640 total_size <= isize::MAX as usize
644 {
645 Ok(total_size)
646 } else {
647 Err(SharedMemoryError::MemoryRequest(
648 min_size_bytes,
649 2,
650 isize::MAX as usize - 2 * page_size::get(),
651 page_size::get(),
652 ))
653 }
654 }
655
656 #[cfg(unix)]
661 #[instrument(skip_all, parent = Span::current(), level= "Trace")]
662 pub fn new(min_size_bytes: usize) -> Result<Self> {
663 use libc::{
664 MAP_ANONYMOUS, MAP_FAILED, MAP_PRIVATE, PROT_READ, PROT_WRITE, c_int, mmap, off_t,
665 size_t,
666 };
667
668 let total_size = Self::total_size(min_size_bytes)?;
669
670 #[cfg(not(miri))]
671 use libc::{MAP_NORESERVE, PROT_NONE, mprotect};
672
673 #[cfg(not(miri))]
675 let flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE;
676 #[cfg(miri)]
677 let flags = MAP_ANONYMOUS | MAP_PRIVATE;
678
679 let addr = unsafe {
680 mmap(
681 null_mut(),
682 total_size as size_t,
683 PROT_READ | PROT_WRITE,
684 flags,
685 -1 as c_int,
686 0 as off_t,
687 )
688 };
689 if addr == MAP_FAILED {
690 log_then_return!(SharedMemoryError::MmapFailed(
691 Error::last_os_error().raw_os_error()
692 ));
693 }
694 let mmap = Mmap {
695 base: addr,
696 len: total_size,
697 };
698
699 #[cfg(not(miri))]
701 {
702 let res = unsafe { mprotect(mmap.base, page_size::get(), PROT_NONE) };
703 if res != 0 {
704 return Err(SharedMemoryError::MprotectFailed(
705 Error::last_os_error().raw_os_error(),
706 ));
707 }
708 let res = unsafe {
709 mprotect(
710 (mmap.base as *const u8).add(total_size - page_size::get()) as *mut c_void,
711 page_size::get(),
712 PROT_NONE,
713 )
714 };
715 if res != 0 {
716 return Err(SharedMemoryError::MprotectFailed(
717 Error::last_os_error().raw_os_error(),
718 ));
719 }
720 }
721
722 Ok(Self {
723 #[allow(clippy::arc_with_non_send_sync)]
732 region: Arc::new(HostMapping { mmap }),
733 })
734 }
735
736 #[cfg(target_os = "windows")]
741 #[instrument(skip_all, parent = Span::current(), level= "Trace")]
742 pub fn new(min_size_bytes: usize) -> Result<Self> {
743 let total_size = Self::total_size(min_size_bytes)?;
744
745 let mut dwmaximumsizehigh = 0;
746 let mut dwmaximumsizelow = 0;
747
748 if std::mem::size_of::<usize>() == 8 {
749 dwmaximumsizehigh = (total_size >> 32) as u32;
750 dwmaximumsizelow = (total_size & 0xFFFFFFFF) as u32;
751 }
752
753 let flags = PAGE_READWRITE;
757
758 let handle = unsafe {
759 CreateFileMappingA(
760 INVALID_HANDLE_VALUE,
761 None,
762 flags,
763 dwmaximumsizehigh,
764 dwmaximumsizelow,
765 PCSTR::null(),
766 )?
767 };
768
769 if handle.is_invalid() {
770 log_then_return!(SharedMemoryError::MemoryAllocationFailed(
771 Error::last_os_error().raw_os_error()
772 ));
773 }
774 let file_mapping = FileMapping(handle);
775
776 let file_map = FILE_MAP_ALL_ACCESS;
777 let addr = unsafe { MapViewOfFile(file_mapping.0, file_map, 0, 0, 0) };
778
779 if addr.Value.is_null() {
780 log_then_return!(SharedMemoryError::MemoryAllocationFailed(
781 Error::last_os_error().raw_os_error()
782 ));
783 }
784 let view = MappedView {
785 addr: addr.Value,
786 len: total_size,
787 };
788
789 let mut unused_out_old_prot_flags = PAGE_PROTECTION_FLAGS(0);
792
793 let first_guard_page_start = view.addr;
796 if let Err(e) = unsafe {
797 VirtualProtect(
798 first_guard_page_start,
799 page_size::get(),
800 PAGE_NOACCESS,
801 &mut unused_out_old_prot_flags,
802 )
803 } {
804 log_then_return!(SharedMemoryError::WindowsAPIError(e.clone()));
805 }
806
807 let last_guard_page_start = unsafe { view.addr.add(total_size - page_size::get()) };
808 if let Err(e) = unsafe {
809 VirtualProtect(
810 last_guard_page_start,
811 page_size::get(),
812 PAGE_NOACCESS,
813 &mut unused_out_old_prot_flags,
814 )
815 } {
816 log_then_return!(SharedMemoryError::WindowsAPIError(e.clone()));
817 }
818
819 Ok(Self {
820 #[allow(clippy::arc_with_non_send_sync)]
829 region: Arc::new(HostMapping {
830 mapping: WindowsMapping::Anonymous { view, file_mapping },
831 }),
832 })
833 }
834
835 pub(super) fn as_mut_slice(&mut self) -> &mut [u8] {
871 unsafe { std::slice::from_raw_parts_mut(self.base_ptr(), self.mem_size()) }
872 }
873
874 #[instrument(skip_all, parent = Span::current(), level= "Trace")]
887 pub fn as_slice<'a>(&'a self) -> &'a [u8] {
888 unsafe { std::slice::from_raw_parts(self.base_ptr(), self.mem_size()) }
889 }
890
891 #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
893 #[cfg(test)]
894 pub(crate) fn copy_all_to_vec(&self) -> Result<Vec<u8>> {
895 let data = self.as_slice();
896 Ok(data.to_vec())
897 }
898
899 #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
901 pub fn copy_from_slice(&mut self, src: &[u8], offset: usize) -> Result<()> {
902 let data = self.as_mut_slice();
903 bounds_check!(offset, src.len(), data.len());
904 data[offset..offset + src.len()].copy_from_slice(src);
905 Ok(())
906 }
907
908 generate_reader!(read_u8, u8);
909 generate_reader!(read_i8, i8);
910 generate_reader!(read_u16, u16);
911 generate_reader!(read_i16, i16);
912 generate_reader!(read_u32, u32);
913 generate_reader!(read_i32, i32);
914 generate_reader!(read_u64, u64);
915 generate_reader!(read_i64, i64);
916 generate_reader!(read_usize, usize);
917 generate_reader!(read_isize, isize);
918
919 generate_writer!(write_u8, u8);
920 generate_writer!(write_i8, i8);
921 generate_writer!(write_u16, u16);
922 generate_writer!(write_i16, i16);
923 generate_writer!(write_u32, u32);
924 generate_writer!(write_i32, i32);
925 generate_writer!(write_u64, u64);
926 generate_writer!(write_i64, i64);
927 generate_writer!(write_usize, usize);
928 generate_writer!(write_isize, isize);
929
930 pub fn build(self) -> (HostSharedMemory, GuestSharedMemory) {
937 let lock = Arc::new(RwLock::new(()));
938 let hshm = HostSharedMemory {
939 region: self.region.clone(),
940 lock: lock.clone(),
941 };
942 (
943 hshm,
944 GuestSharedMemory {
945 region: self.region.clone(),
946 lock,
947 },
948 )
949 }
950
951 #[cfg(target_os = "windows")]
953 pub fn get_mmap_file_handle(&self) -> HANDLE {
954 self.region.file_mapping_handle()
955 }
956}
957
958impl SharedMemory for ExclusiveSharedMemory {
959 fn region(&self) -> &HostMapping {
960 &self.region
961 }
962 fn with_exclusivity<T, F: FnOnce(&mut ExclusiveSharedMemory) -> T>(
963 &mut self,
964 f: F,
965 ) -> Result<T> {
966 Ok(f(self))
967 }
968}
969
970#[derive(Debug)]
979pub struct GuestSharedMemory {
980 region: Arc<HostMapping>,
981 pub lock: Arc<RwLock<()>>,
992}
993unsafe impl Send for GuestSharedMemory {}
994
995impl GuestSharedMemory {
996 pub(crate) fn mapping_at(
999 &self,
1000 guest_base: u64,
1001 region_type: MemoryRegionType,
1002 ) -> MemoryRegion {
1003 let flags = match region_type {
1004 MemoryRegionType::Scratch => {
1005 MemoryRegionFlags::READ | MemoryRegionFlags::WRITE | MemoryRegionFlags::EXECUTE
1006 }
1007 #[cfg(unshared_snapshot_mem)]
1008 MemoryRegionType::Snapshot => {
1009 MemoryRegionFlags::READ | MemoryRegionFlags::WRITE | MemoryRegionFlags::EXECUTE
1010 }
1011 #[allow(clippy::panic)]
1012 _ => panic!(
1021 "GuestSharedMemory::mapping_at should only be used for Scratch or Snapshot regions"
1022 ),
1023 };
1024 mapping_at(self, guest_base, self.mem_size(), region_type, flags)
1025 }
1026}
1027
1028impl SharedMemory for GuestSharedMemory {
1029 fn region(&self) -> &HostMapping {
1030 &self.region
1031 }
1032 fn with_exclusivity<T, F: FnOnce(&mut ExclusiveSharedMemory) -> T>(
1033 &mut self,
1034 f: F,
1035 ) -> Result<T> {
1036 let guard = self.lock.try_write()?;
1037 let mut excl = ExclusiveSharedMemory {
1038 region: self.region.clone(),
1039 };
1040 let ret = f(&mut excl);
1041 drop(excl);
1042 drop(guard);
1043 Ok(ret)
1044 }
1045}
1046
1047#[derive(Clone, Debug)]
1202pub struct HostSharedMemory {
1203 region: Arc<HostMapping>,
1204 lock: Arc<RwLock<()>>,
1205}
1206unsafe impl Send for HostSharedMemory {}
1207
1208impl HostSharedMemory {
1209 pub fn read<T: Pod>(&self, offset: usize) -> Result<T> {
1212 bounds_check!(offset, std::mem::size_of::<T>(), self.mem_size());
1213 let mut ret = T::zeroed();
1214 self.copy_to_slice(bytemuck::bytes_of_mut(&mut ret), offset)?;
1215 Ok(ret)
1216 }
1217
1218 pub fn write<T: Pod>(&self, offset: usize, data: T) -> Result<()> {
1221 bounds_check!(offset, std::mem::size_of::<T>(), self.mem_size());
1222 self.copy_from_slice(bytemuck::bytes_of(&data), offset)
1223 }
1224
1225 pub fn load_atomic<A: AtomicAccess>(
1227 &self,
1228 offset: usize,
1229 ordering: Ordering,
1230 ) -> Result<A::Value> {
1231 if matches!(ordering, Ordering::Release | Ordering::AcqRel) {
1232 return Err(SharedMemoryError::InvalidAtomicLoadOrdering(ordering));
1233 }
1234
1235 bounds_check!(offset, size_of::<A>(), self.mem_size());
1236 let ptr = self.base_ptr().wrapping_add(offset);
1237 if !(ptr as usize).is_multiple_of(align_of::<A>()) {
1238 return Err(SharedMemoryError::AtomicAccessUnaligned(
1239 offset,
1240 align_of::<A>(),
1241 ));
1242 }
1243
1244 let _guard = self.lock.try_read()?;
1245
1246 let atomic = unsafe { &*ptr.cast::<A>() };
1249 Ok(atomic.load(ordering))
1250 }
1251
1252 pub fn store_atomic<A: AtomicAccess>(
1254 &self,
1255 offset: usize,
1256 value: A::Value,
1257 ordering: Ordering,
1258 ) -> Result<()> {
1259 if matches!(ordering, Ordering::Acquire | Ordering::AcqRel) {
1260 return Err(SharedMemoryError::InvalidAtomicStoreOrdering(ordering));
1261 }
1262
1263 bounds_check!(offset, size_of::<A>(), self.mem_size());
1264 let ptr = self.base_ptr().wrapping_add(offset);
1265 if !(ptr as usize).is_multiple_of(align_of::<A>()) {
1266 return Err(SharedMemoryError::AtomicAccessUnaligned(
1267 offset,
1268 align_of::<A>(),
1269 ));
1270 }
1271
1272 let _guard = self.lock.try_read()?;
1273
1274 let atomic = unsafe { &*ptr.cast::<A>() };
1277 atomic.store(value, ordering);
1278 Ok(())
1279 }
1280
1281 pub fn copy_to_slice(&self, slice: &mut [u8], offset: usize) -> Result<()> {
1284 bounds_check!(offset, slice.len(), self.mem_size());
1285 let base = self.base_ptr().wrapping_add(offset);
1286 let guard = self.lock.try_read()?;
1287
1288 const CHUNK: usize = size_of::<u128>();
1289 let len = slice.len();
1290 let mut i = 0;
1291
1292 let align_offset = base.align_offset(align_of::<u128>());
1297 let head_len = align_offset.min(len);
1298 while i < head_len {
1299 unsafe {
1300 slice[i] = base.add(i).read_volatile();
1301 }
1302 i += 1;
1303 }
1304
1305 let dst = slice.as_mut_ptr();
1309 while i + CHUNK <= len {
1310 unsafe {
1311 let value = (base.add(i) as *const u128).read_volatile();
1312 std::ptr::write_unaligned(dst.add(i) as *mut u128, value);
1313 }
1314 i += CHUNK;
1315 }
1316
1317 while i < len {
1319 unsafe {
1320 slice[i] = base.add(i).read_volatile();
1321 }
1322 i += 1;
1323 }
1324
1325 drop(guard);
1326 Ok(())
1327 }
1328
1329 pub fn copy_from_slice(&self, slice: &[u8], offset: usize) -> Result<()> {
1332 bounds_check!(offset, slice.len(), self.mem_size());
1333 let base = self.base_ptr().wrapping_add(offset);
1334 let guard = self.lock.try_read()?;
1335
1336 const CHUNK: usize = size_of::<u128>();
1337 let len = slice.len();
1338 let mut i = 0;
1339
1340 let align_offset = base.align_offset(align_of::<u128>());
1345 let head_len = align_offset.min(len);
1346 while i < head_len {
1347 unsafe {
1348 base.add(i).write_volatile(slice[i]);
1349 }
1350 i += 1;
1351 }
1352
1353 let src = slice.as_ptr();
1357 while i + CHUNK <= len {
1358 unsafe {
1359 let value = std::ptr::read_unaligned(src.add(i) as *const u128);
1360 (base.add(i) as *mut u128).write_volatile(value);
1361 }
1362 i += CHUNK;
1363 }
1364
1365 while i < len {
1367 unsafe {
1368 base.add(i).write_volatile(slice[i]);
1369 }
1370 i += 1;
1371 }
1372
1373 drop(guard);
1374 Ok(())
1375 }
1376
1377 #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
1379 pub fn fill(&mut self, value: u8, offset: usize, len: usize) -> Result<()> {
1380 bounds_check!(offset, len, self.mem_size());
1381 let base = self.base_ptr().wrapping_add(offset);
1382 let guard = self.lock.try_read()?;
1383
1384 const CHUNK: usize = size_of::<u128>();
1385 let value_u128 = u128::from_ne_bytes([value; CHUNK]);
1386 let mut i = 0;
1387
1388 let align_offset = base.align_offset(align_of::<u128>());
1393 let head_len = align_offset.min(len);
1394 while i < head_len {
1395 unsafe {
1396 base.add(i).write_volatile(value);
1397 }
1398 i += 1;
1399 }
1400
1401 while i + CHUNK <= len {
1404 unsafe {
1405 (base.add(i) as *mut u128).write_volatile(value_u128);
1406 }
1407 i += CHUNK;
1408 }
1409
1410 while i < len {
1412 unsafe {
1413 base.add(i).write_volatile(value);
1414 }
1415 i += 1;
1416 }
1417
1418 drop(guard);
1419 Ok(())
1420 }
1421}
1422
1423impl HostSharedMemory {
1424 pub(crate) fn zero_or_replace(&mut self) -> Result<Option<GuestSharedMemory>> {
1441 #[cfg(target_os = "windows")]
1442 {
1443 let new_mem = ExclusiveSharedMemory::new(self.mem_size())?;
1444 let (hscratch, gscratch) = new_mem.build();
1445 *self = hscratch;
1446 Ok(Some(gscratch))
1447 }
1448 #[cfg(not(target_os = "windows"))]
1449 {
1450 self.with_exclusivity(|e| {
1451 #[allow(unused_mut)]
1452 let mut do_copy = true;
1453 #[cfg(all(feature = "kvm", not(any(feature = "mshv3"))))]
1456 unsafe {
1457 let ret = libc::madvise(
1458 e.region.ptr() as *mut libc::c_void,
1459 e.region.size(),
1460 libc::MADV_DONTNEED,
1461 );
1462 if ret == 0 {
1463 do_copy = false;
1464 }
1465 }
1466 if do_copy {
1467 e.as_mut_slice().fill(0);
1468 }
1469 })?;
1470 Ok(None)
1471 }
1472 }
1473}
1474
1475impl SharedMemory for HostSharedMemory {
1476 fn region(&self) -> &HostMapping {
1477 &self.region
1478 }
1479 fn with_exclusivity<T, F: FnOnce(&mut ExclusiveSharedMemory) -> T>(
1480 &mut self,
1481 f: F,
1482 ) -> Result<T> {
1483 let guard = self.lock.try_write()?;
1484 let mut excl = ExclusiveSharedMemory {
1485 region: self.region.clone(),
1486 };
1487 let ret = f(&mut excl);
1488 drop(excl);
1489 drop(guard);
1490 Ok(ret)
1491 }
1492}
1493
1494#[derive(Clone, Debug)]
1499pub struct ReadonlySharedMemory {
1500 region: Arc<HostMapping>,
1501 #[cfg_attr(unshared_snapshot_mem, allow(dead_code))]
1506 guest_mapped_size: usize,
1507}
1508unsafe impl Send for ReadonlySharedMemory {}
1519unsafe impl Sync for ReadonlySharedMemory {}
1520
1521impl ReadonlySharedMemory {
1522 pub(crate) fn from_bytes(contents: &[u8], guest_mapped_size: usize) -> Result<Self> {
1523 if guest_mapped_size == 0
1524 || guest_mapped_size > contents.len()
1525 || !guest_mapped_size.is_multiple_of(page_size::get())
1526 {
1527 return Err(SharedMemoryError::MemoryRequest(
1528 guest_mapped_size,
1529 0,
1530 contents.len(),
1531 page_size::get(),
1532 ));
1533 }
1534 let mut anon =
1535 ExclusiveSharedMemory::new(contents.len().next_multiple_of(page_size::get()))?;
1536 anon.copy_from_slice(contents, 0)?;
1537 Ok(ReadonlySharedMemory {
1538 region: anon.region,
1539 guest_mapped_size,
1540 })
1541 }
1542
1543 #[cfg(not(unshared_snapshot_mem))]
1545 pub(crate) fn guest_mapped_size(&self) -> usize {
1546 self.guest_mapped_size
1547 }
1548
1549 pub(crate) fn from_file(file: &std::fs::File, guest_mapped_size: usize) -> Result<Self> {
1555 let len: usize = file
1556 .metadata()
1557 .map_err(SharedMemoryError::FileMetadata)?
1558 .len()
1559 .try_into()
1560 .map_err(SharedMemoryError::FileTooLarge)?;
1561
1562 if len == 0 || !len.is_multiple_of(page_size::get()) {
1563 return Err(SharedMemoryError::MemoryRequest(
1564 len,
1565 0,
1566 usize::MAX,
1567 page_size::get(),
1568 ));
1569 }
1570
1571 if guest_mapped_size == 0
1572 || guest_mapped_size > len
1573 || !guest_mapped_size.is_multiple_of(page_size::get())
1574 {
1575 return Err(SharedMemoryError::MemoryRequest(
1576 guest_mapped_size,
1577 0,
1578 len,
1579 page_size::get(),
1580 ));
1581 }
1582
1583 let region = Self::map_file(file, len)?;
1584 Ok(ReadonlySharedMemory {
1585 region,
1586 guest_mapped_size,
1587 })
1588 }
1589
1590 #[cfg(unix)]
1594 fn map_file(file: &std::fs::File, len: usize) -> Result<Arc<HostMapping>> {
1595 use std::os::unix::io::AsRawFd;
1596
1597 #[cfg(mshv3)]
1598 use libc::PROT_WRITE;
1599 use libc::{
1600 MAP_ANONYMOUS, MAP_FAILED, MAP_FIXED, MAP_NORESERVE, MAP_PRIVATE, PROT_NONE, PROT_READ,
1601 mmap, off_t, size_t,
1602 };
1603
1604 let total_size =
1605 len.checked_add(2 * page_size::get())
1606 .ok_or(SharedMemoryError::MemoryRequest(
1607 len,
1608 0,
1609 usize::MAX - 2 * page_size::get(),
1610 1,
1611 ))?;
1612
1613 let fd = file.as_raw_fd();
1614
1615 let base = unsafe {
1621 mmap(
1622 null_mut(),
1623 total_size as size_t,
1624 PROT_NONE,
1625 MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE,
1626 -1,
1627 0 as off_t,
1628 )
1629 };
1630 if base == MAP_FAILED {
1631 return Err(SharedMemoryError::MmapFailed(
1632 std::io::Error::last_os_error().raw_os_error(),
1633 ));
1634 }
1635 let reservation = Mmap {
1636 base,
1637 len: total_size,
1638 };
1639
1640 #[cfg(mshv3)]
1649 let file_prot = PROT_READ | PROT_WRITE;
1650 #[cfg(not(mshv3))]
1651 let file_prot = PROT_READ;
1652 let usable_ptr = unsafe { (base as *mut u8).add(page_size::get()) };
1655 let mapped = unsafe {
1661 mmap(
1662 usable_ptr as *mut c_void,
1663 len as size_t,
1664 file_prot,
1665 MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE,
1666 fd,
1667 0 as off_t,
1668 )
1669 };
1670 if mapped == MAP_FAILED {
1671 return Err(SharedMemoryError::MmapFailed(
1672 std::io::Error::last_os_error().raw_os_error(),
1673 ));
1674 }
1675
1676 #[allow(clippy::arc_with_non_send_sync)]
1680 Ok(Arc::new(HostMapping { mmap: reservation }))
1681 }
1682
1683 #[cfg(target_os = "windows")]
1687 fn map_file(file: &std::fs::File, len: usize) -> Result<Arc<HostMapping>> {
1688 use std::os::windows::io::AsRawHandle;
1689
1690 let total_size =
1691 len.checked_add(2 * page_size::get())
1692 .ok_or(SharedMemoryError::MemoryRequest(
1693 len,
1694 0,
1695 usize::MAX - 2 * page_size::get(),
1696 1,
1697 ))?;
1698
1699 let file_handle = HANDLE(file.as_raw_handle());
1700
1701 let whole = Placeholder::reserve(total_size)?;
1704
1705 let (leading, middle, trailing) = whole.split_into_three(page_size::get(), len)?;
1709
1710 let raw_handle =
1715 unsafe { CreateFileMappingA(file_handle, None, PAGE_READONLY, 0, 0, PCSTR::null()) }?;
1716 if raw_handle.is_invalid() {
1717 log_then_return!(SharedMemoryError::MemoryAllocationFailed(
1718 Error::last_os_error().raw_os_error()
1719 ));
1720 }
1721 let file_mapping = FileMapping(raw_handle);
1722
1723 let view = middle.map_file_view(raw_handle)?;
1726
1727 #[allow(clippy::arc_with_non_send_sync)]
1728 Ok(Arc::new(HostMapping {
1729 mapping: WindowsMapping::FileBacked {
1730 leading,
1731 view,
1732 trailing,
1733 file_mapping,
1734 },
1735 }))
1736 }
1737
1738 pub(crate) fn as_slice(&self) -> &[u8] {
1739 unsafe { std::slice::from_raw_parts(self.base_ptr(), self.mem_size()) }
1740 }
1741
1742 #[cfg(unshared_snapshot_mem)]
1743 pub(crate) fn copy_to_writable(&self) -> Result<ExclusiveSharedMemory> {
1744 let mut writable = ExclusiveSharedMemory::new(self.mem_size())?;
1745 writable.copy_from_slice(self.as_slice(), 0)?;
1746 Ok(writable)
1747 }
1748
1749 #[cfg(not(unshared_snapshot_mem))]
1750 pub(crate) fn build(self) -> (Self, Self) {
1751 (self.clone(), self)
1752 }
1753
1754 #[cfg(not(unshared_snapshot_mem))]
1755 pub(crate) fn mapping_at(
1756 &self,
1757 guest_base: u64,
1758 region_type: MemoryRegionType,
1759 ) -> MemoryRegion {
1760 #[allow(clippy::panic)]
1761 if region_type != MemoryRegionType::Snapshot {
1765 panic!("ReadonlySharedMemory::mapping_at should only be used for Snapshot regions");
1766 }
1767 mapping_at(
1768 self,
1769 guest_base,
1770 self.guest_mapped_size(),
1771 region_type,
1772 MemoryRegionFlags::READ | MemoryRegionFlags::EXECUTE,
1773 )
1774 }
1775}
1776
1777impl SharedMemory for ReadonlySharedMemory {
1778 fn region(&self) -> &HostMapping {
1779 &self.region
1780 }
1781 #[cfg(windows)]
1791 fn host_region_base(&self) -> <HostGuestMemoryRegion as MemoryRegionKind>::HostBaseType {
1792 match &self.region().mapping {
1793 WindowsMapping::Anonymous { .. } => super::memory_region::HostRegionBase {
1794 from_handle: self.region().file_mapping_handle().into(),
1795 handle_base: self.region().ptr() as usize,
1796 handle_size: self.region().size(),
1797 offset: page_size::get(),
1798 },
1799 WindowsMapping::FileBacked { .. } => super::memory_region::HostRegionBase {
1800 from_handle: self.region().file_mapping_handle().into(),
1801 handle_base: self.base_ptr() as usize,
1802 handle_size: self.mem_size(),
1803 offset: 0,
1804 },
1805 }
1806 }
1807 fn with_exclusivity<T, F: FnOnce(&mut ExclusiveSharedMemory) -> T>(
1810 &mut self,
1811 _: F,
1812 ) -> Result<T> {
1813 Err(SharedMemoryError::ReadonlySharedMemoryExclusiveRequest)
1814 }
1815 fn with_contents<T, F: FnOnce(&[u8]) -> T>(&mut self, f: F) -> Result<T> {
1817 Ok(f(self.as_slice()))
1818 }
1819}
1820
1821impl<S: SharedMemory> PartialEq<S> for ReadonlySharedMemory {
1822 fn eq(&self, other: &S) -> bool {
1823 self.raw_ptr() == other.raw_ptr()
1824 }
1825}
1826
1827#[cfg(test)]
1828mod tests {
1829 use std::sync::atomic::{AtomicI32, AtomicU16, Ordering};
1830
1831 #[cfg(not(miri))]
1832 use proptest::prelude::*;
1833
1834 #[cfg(not(miri))]
1835 use super::HostSharedMemory;
1836 use super::{ExclusiveSharedMemory, Result, SharedMemory};
1837 #[cfg(not(miri))]
1838 use crate::mem::shared_mem_tests::read_write_test_suite;
1839
1840 #[test]
1841 fn fill() {
1842 let mem_size: usize = page_size::get();
1843 let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
1844 let (mut hshm, _) = eshm.build();
1845
1846 hshm.fill(1, 0, 1024).unwrap();
1847 hshm.fill(2, 1024, 1024).unwrap();
1848 hshm.fill(3, 2048, 1024).unwrap();
1849 hshm.fill(4, 3072, 1024).unwrap();
1850
1851 let vec = hshm
1852 .with_exclusivity(|e| e.copy_all_to_vec().unwrap())
1853 .unwrap();
1854
1855 assert!(vec[0..1024].iter().all(|&x| x == 1));
1856 assert!(vec[1024..2048].iter().all(|&x| x == 2));
1857 assert!(vec[2048..3072].iter().all(|&x| x == 3));
1858 assert!(vec[3072..4096].iter().all(|&x| x == 4));
1859
1860 hshm.fill(5, 0, mem_size).unwrap();
1861
1862 let vec2 = hshm
1863 .with_exclusivity(|e| e.copy_all_to_vec().unwrap())
1864 .unwrap();
1865 assert!(vec2.iter().all(|&x| x == 5));
1866
1867 assert!(hshm.fill(0, 0, mem_size + 1).is_err());
1868 assert!(hshm.fill(0, mem_size, 1).is_err());
1869 }
1870
1871 #[test]
1874 fn bounds_check_overflow() {
1875 let mem_size: usize = page_size::get();
1876 let mut eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
1877
1878 assert!(eshm.read_i32(usize::MAX).is_err());
1880 assert!(eshm.write_i32(usize::MAX, 0).is_err());
1881 assert!(eshm.copy_from_slice(&[0u8; 1], usize::MAX).is_err());
1882
1883 let (mut hshm, _) = eshm.build();
1885
1886 assert!(hshm.read::<u8>(usize::MAX).is_err());
1887 assert!(hshm.read::<u64>(usize::MAX - 3).is_err());
1888 assert!(hshm.write::<u8>(usize::MAX, 0).is_err());
1889 assert!(hshm.write::<u64>(usize::MAX - 3, 0).is_err());
1890
1891 let mut buf = [0u8; 1];
1892 assert!(hshm.copy_to_slice(&mut buf, usize::MAX).is_err());
1893 assert!(hshm.copy_from_slice(&[0u8; 1], usize::MAX).is_err());
1894
1895 assert!(hshm.fill(0, usize::MAX, 1).is_err());
1896 assert!(hshm.fill(0, 1, usize::MAX).is_err());
1897 }
1898
1899 #[test]
1900 fn atomic_access() {
1901 let mem_size = page_size::get();
1902 let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
1903 let (hshm, _) = eshm.build();
1904
1905 hshm.store_atomic::<AtomicU16>(0, 0x1234, Ordering::Release)
1906 .unwrap();
1907 assert_eq!(
1908 hshm.load_atomic::<AtomicU16>(0, Ordering::Acquire).unwrap(),
1909 0x1234
1910 );
1911
1912 hshm.store_atomic::<AtomicI32>(4, -42, Ordering::SeqCst)
1913 .unwrap();
1914 assert_eq!(
1915 hshm.load_atomic::<AtomicI32>(4, Ordering::SeqCst).unwrap(),
1916 -42
1917 );
1918
1919 assert!(hshm.load_atomic::<AtomicU16>(1, Ordering::Relaxed).is_err());
1920 assert!(
1921 hshm.load_atomic::<AtomicU16>(mem_size - 1, Ordering::Relaxed)
1922 .is_err()
1923 );
1924 assert!(hshm.load_atomic::<AtomicU16>(0, Ordering::Release).is_err());
1925 assert!(
1926 hshm.store_atomic::<AtomicU16>(0, 0, Ordering::Acquire)
1927 .is_err()
1928 );
1929 }
1930
1931 #[test]
1932 fn atomic_access_observes_exclusivity() {
1933 let eshm = ExclusiveSharedMemory::new(page_size::get()).unwrap();
1934 let (mut hshm, _) = eshm.build();
1935 let other = hshm.clone();
1936
1937 hshm.with_exclusivity(|_| {
1938 assert!(
1939 other
1940 .load_atomic::<AtomicU16>(0, Ordering::Relaxed)
1941 .is_err()
1942 );
1943 })
1944 .unwrap();
1945 }
1946
1947 #[test]
1948 fn copy_into_from() -> Result<()> {
1949 let mem_size: usize = page_size::get();
1950 let vec_len = 10;
1951 let eshm = ExclusiveSharedMemory::new(mem_size)?;
1952 let (hshm, _) = eshm.build();
1953 let vec = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
1954 hshm.copy_from_slice(&vec, 0)?;
1956
1957 let mut vec2 = vec![0; vec_len];
1958 hshm.copy_to_slice(vec2.as_mut_slice(), 0)?;
1960 assert_eq!(vec, vec2);
1961
1962 let offset = mem_size - vec.len();
1963 hshm.copy_from_slice(&vec, offset)?;
1965
1966 let mut vec3 = vec![0; vec_len];
1967 hshm.copy_to_slice(&mut vec3, offset)?;
1969 assert_eq!(vec, vec3);
1970
1971 let offset = mem_size / 2;
1972 hshm.copy_from_slice(&vec, offset)?;
1974
1975 let mut vec4 = vec![0; vec_len];
1976 hshm.copy_to_slice(&mut vec4, offset)?;
1978 assert_eq!(vec, vec4);
1979
1980 let mut vec5 = vec![0; vec_len];
1982 assert!(hshm.copy_to_slice(&mut vec5, mem_size).is_err());
1983
1984 assert!(hshm.copy_from_slice(&vec5, mem_size).is_err());
1986
1987 let mut vec6 = vec![0; vec_len];
1989 assert!(hshm.copy_to_slice(&mut vec6, mem_size * 2).is_err());
1990
1991 assert!(hshm.copy_from_slice(&vec6, mem_size * 2).is_err());
1993
1994 let mut vec7 = vec![0; mem_size * 2];
1996 assert!(hshm.copy_to_slice(&mut vec7, 0).is_err());
1997
1998 assert!(hshm.copy_from_slice(&vec7, 0).is_err());
2000
2001 Ok(())
2002 }
2003
2004 #[cfg(not(miri))]
2006 proptest! {
2007 #[test]
2008 fn read_write_i32(val in -0x1000_i32..0x1000_i32) {
2009 read_write_test_suite(
2010 val,
2011 ExclusiveSharedMemory::new,
2012 Box::new(ExclusiveSharedMemory::read_i32),
2013 Box::new(ExclusiveSharedMemory::write_i32),
2014 )
2015 .unwrap();
2016 read_write_test_suite(
2017 val,
2018 |s| {
2019 let e = ExclusiveSharedMemory::new(s)?;
2020 let (h, _) = e.build();
2021 Ok(h)
2022 },
2023 Box::new(HostSharedMemory::read::<i32>),
2024 Box::new(|h, o, v| h.write::<i32>(o, v)),
2025 )
2026 .unwrap();
2027 }
2028 }
2029
2030 #[test]
2031 fn alloc_fail() {
2032 let gm = ExclusiveSharedMemory::new(0);
2033 assert!(gm.is_err());
2034 let gm = ExclusiveSharedMemory::new(usize::MAX);
2035 assert!(gm.is_err());
2036 }
2037
2038 #[test]
2039 fn clone() {
2040 let eshm = ExclusiveSharedMemory::new(page_size::get()).unwrap();
2041 let (hshm1, _) = eshm.build();
2042 let hshm2 = hshm1.clone();
2043
2044 assert_eq!(hshm1.mem_size(), hshm2.mem_size());
2047 assert_eq!(hshm1.base_addr(), hshm2.base_addr());
2048
2049 hshm1.copy_from_slice(b"a", 0).unwrap();
2052 hshm2.copy_from_slice(b"b", 1).unwrap();
2053
2054 for (raw_offset, expected) in &[(0, b'a'), (1, b'b')] {
2057 assert_eq!(hshm1.read::<u8>(*raw_offset).unwrap(), *expected);
2058 assert_eq!(hshm2.read::<u8>(*raw_offset).unwrap(), *expected);
2059 }
2060
2061 drop(hshm1);
2064
2065 for (raw_offset, expected) in &[(0, b'a'), (1, b'b')] {
2067 assert_eq!(hshm2.read::<u8>(*raw_offset).unwrap(), *expected);
2068 }
2069 hshm2.copy_from_slice(b"c", 2).unwrap();
2070 assert_eq!(hshm2.read::<u8>(2).unwrap(), b'c');
2071 drop(hshm2);
2072 }
2073
2074 #[test]
2075 fn copy_all_to_vec() {
2076 let mut data = vec![b'a', b'b', b'c'];
2077 data.resize(page_size::get(), 0);
2078 let mut eshm = ExclusiveSharedMemory::new(data.len()).unwrap();
2079 eshm.copy_from_slice(data.as_slice(), 0).unwrap();
2080 let ret_vec = eshm.copy_all_to_vec().unwrap();
2081 assert_eq!(data, ret_vec);
2082 }
2083
2084 #[test]
2087 #[cfg(all(target_os = "linux", not(miri)))]
2088 fn test_drop() {
2089 use proc_maps::get_process_maps;
2090
2091 let unique_size: usize = page_size::get() * 17;
2101
2102 let pid = std::process::id();
2103
2104 let eshm = ExclusiveSharedMemory::new(unique_size).unwrap();
2105 let (hshm1, gshm) = eshm.build();
2106 let hshm2 = hshm1.clone();
2107
2108 let base_ptr = hshm1.base_ptr() as usize;
2110 let mem_size = hshm1.mem_size();
2111
2112 let has_exact_mapping = |ptr: usize, size: usize| -> bool {
2114 get_process_maps(pid.try_into().unwrap())
2115 .unwrap()
2116 .iter()
2117 .any(|m| m.start() == ptr && m.size() == size)
2118 };
2119
2120 assert!(
2122 has_exact_mapping(base_ptr, mem_size),
2123 "shared memory mapping not found at {:#x} with size {}",
2124 base_ptr,
2125 mem_size
2126 );
2127
2128 drop(hshm1);
2130 drop(hshm2);
2131 drop(gshm);
2132
2133 assert!(
2135 !has_exact_mapping(base_ptr, mem_size),
2136 "shared memory mapping still exists at {:#x} with size {} after drop",
2137 base_ptr,
2138 mem_size
2139 );
2140 }
2141
2142 mod alignment_tests {
2146 use super::*;
2147
2148 const CHUNK_SIZE: usize = size_of::<u128>();
2149
2150 #[test]
2152 fn copy_with_various_alignments() {
2153 let mem_size: usize = page_size::get();
2155 let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2156 let (hshm, _) = eshm.build();
2157
2158 for start_offset in 0..CHUNK_SIZE {
2160 let test_len = 64; let test_data: Vec<u8> = (0..test_len).map(|i| (i + start_offset) as u8).collect();
2162
2163 hshm.copy_from_slice(&test_data, start_offset).unwrap();
2165
2166 let mut read_buf = vec![0u8; test_len];
2168 hshm.copy_to_slice(&mut read_buf, start_offset).unwrap();
2169
2170 assert_eq!(
2171 test_data, read_buf,
2172 "Mismatch at alignment offset {}",
2173 start_offset
2174 );
2175 }
2176 }
2177
2178 #[test]
2180 fn copy_small_lengths() {
2181 let mem_size: usize = page_size::get();
2182 let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2183 let (hshm, _) = eshm.build();
2184
2185 for len in 0..CHUNK_SIZE {
2186 let test_data: Vec<u8> = (0..len).map(|i| i as u8).collect();
2187
2188 hshm.copy_from_slice(&test_data, 0).unwrap();
2189
2190 let mut read_buf = vec![0u8; len];
2191 hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2192
2193 assert_eq!(test_data, read_buf, "Mismatch for length {}", len);
2194 }
2195 }
2196
2197 #[test]
2199 fn copy_non_aligned_lengths() {
2200 let mem_size: usize = page_size::get();
2201 let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2202 let (hshm, _) = eshm.build();
2203
2204 let test_lengths = [17, 31, 33, 47, 63, 65, 100, 127, 129, 255, 257];
2206
2207 for &len in &test_lengths {
2208 let test_data: Vec<u8> = (0..len).map(|i| (i % 256) as u8).collect();
2209
2210 hshm.copy_from_slice(&test_data, 0).unwrap();
2211
2212 let mut read_buf = vec![0u8; len];
2213 hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2214
2215 assert_eq!(test_data, read_buf, "Mismatch for length {}", len);
2216 }
2217 }
2218
2219 #[test]
2221 fn copy_exact_chunk_size() {
2222 let mem_size: usize = page_size::get();
2223 let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2224 let (hshm, _) = eshm.build();
2225
2226 let test_data: Vec<u8> = (0..CHUNK_SIZE).map(|i| i as u8).collect();
2227
2228 hshm.copy_from_slice(&test_data, 0).unwrap();
2229
2230 let mut read_buf = vec![0u8; CHUNK_SIZE];
2231 hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2232
2233 assert_eq!(test_data, read_buf);
2234 }
2235
2236 #[test]
2238 fn fill_with_various_alignments() {
2239 let mem_size: usize = page_size::get();
2240 let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2241 let (mut hshm, _) = eshm.build();
2242
2243 for start_offset in 0..CHUNK_SIZE {
2244 let fill_len = 64;
2245 let fill_value = (start_offset % 256) as u8;
2246
2247 hshm.fill(0, 0, mem_size).unwrap();
2249
2250 hshm.fill(fill_value, start_offset, fill_len).unwrap();
2252
2253 let mut read_buf = vec![0u8; fill_len];
2255 hshm.copy_to_slice(&mut read_buf, start_offset).unwrap();
2256
2257 assert!(
2258 read_buf.iter().all(|&b| b == fill_value),
2259 "Fill mismatch at alignment offset {}",
2260 start_offset
2261 );
2262 }
2263 }
2264
2265 #[test]
2267 fn fill_small_lengths() {
2268 let mem_size: usize = page_size::get();
2269 let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2270 let (mut hshm, _) = eshm.build();
2271
2272 for len in 0..CHUNK_SIZE {
2273 let fill_value = 0xAB;
2274
2275 hshm.fill(0, 0, mem_size).unwrap(); hshm.fill(fill_value, 0, len).unwrap();
2277
2278 let mut read_buf = vec![0u8; len];
2279 hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2280
2281 assert!(
2282 read_buf.iter().all(|&b| b == fill_value),
2283 "Fill mismatch for length {}",
2284 len
2285 );
2286 }
2287 }
2288
2289 #[test]
2291 fn fill_non_aligned_lengths() {
2292 let mem_size: usize = page_size::get();
2293 let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2294 let (mut hshm, _) = eshm.build();
2295
2296 let test_lengths = [17, 31, 33, 47, 63, 65, 100, 127, 129, 255, 257];
2297
2298 for &len in &test_lengths {
2299 let fill_value = 0xCD;
2300
2301 hshm.fill(0, 0, mem_size).unwrap(); hshm.fill(fill_value, 0, len).unwrap();
2303
2304 let mut read_buf = vec![0u8; len];
2305 hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2306
2307 assert!(
2308 read_buf.iter().all(|&b| b == fill_value),
2309 "Fill mismatch for length {}",
2310 len
2311 );
2312 }
2313 }
2314
2315 #[test]
2317 fn copy_edge_cases() {
2318 let mem_size: usize = page_size::get();
2319 let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2320 let (hshm, _) = eshm.build();
2321
2322 let empty: Vec<u8> = vec![];
2324 hshm.copy_from_slice(&empty, 0).unwrap();
2325 let mut read_buf: Vec<u8> = vec![];
2326 hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2327 assert!(read_buf.is_empty());
2328
2329 let single = vec![0x42u8];
2331 hshm.copy_from_slice(&single, 0).unwrap();
2332 let mut read_buf = vec![0u8; 1];
2333 hshm.copy_to_slice(&mut read_buf, 0).unwrap();
2334 assert_eq!(single, read_buf);
2335 }
2336
2337 #[test]
2339 fn copy_unaligned_start_and_length() {
2340 let mem_size: usize = page_size::get();
2341 let eshm = ExclusiveSharedMemory::new(mem_size).unwrap();
2342 let (hshm, _) = eshm.build();
2343
2344 let start_offset = 7;
2346 let len = 37;
2347 let test_data: Vec<u8> = (0..len).map(|i| (i * 3) as u8).collect();
2348
2349 hshm.copy_from_slice(&test_data, start_offset).unwrap();
2350
2351 let mut read_buf = vec![0u8; len];
2352 hshm.copy_to_slice(&mut read_buf, start_offset).unwrap();
2353
2354 assert_eq!(test_data, read_buf);
2355 }
2356 }
2357
2358 #[cfg(target_os = "linux")]
2359 mod guard_page_crash_test {
2360 use crate::mem::shared_mem::{ExclusiveSharedMemory, SharedMemory};
2361
2362 const TEST_EXIT_CODE: u8 = 211; fn setup_signal_handler() {
2368 unsafe {
2369 signal_hook_registry::register_signal_unchecked(libc::SIGSEGV, || {
2370 std::process::exit(TEST_EXIT_CODE.into());
2371 })
2372 .unwrap();
2373 }
2374 }
2375
2376 #[test]
2377 #[ignore] fn read() {
2379 setup_signal_handler();
2380
2381 let eshm = ExclusiveSharedMemory::new(page_size::get()).unwrap();
2382 let (hshm, _) = eshm.build();
2383 let guard_page_ptr = hshm.raw_ptr();
2384 unsafe { std::ptr::read_volatile(guard_page_ptr) };
2385 }
2386
2387 #[test]
2388 #[ignore] fn write() {
2390 setup_signal_handler();
2391
2392 let eshm = ExclusiveSharedMemory::new(page_size::get()).unwrap();
2393 let (hshm, _) = eshm.build();
2394 let guard_page_ptr = hshm.raw_ptr();
2395 unsafe { std::ptr::write_volatile(guard_page_ptr, 0u8) };
2396 }
2397
2398 #[test]
2399 #[ignore] fn exec() {
2401 setup_signal_handler();
2402
2403 let eshm = ExclusiveSharedMemory::new(page_size::get()).unwrap();
2404 let (hshm, _) = eshm.build();
2405 let guard_page_ptr = hshm.raw_ptr();
2406 let func: fn() = unsafe { std::mem::transmute(guard_page_ptr) };
2407 func();
2408 }
2409
2410 #[test]
2412 #[cfg_attr(miri, ignore)] fn guard_page_testing_shim() {
2414 let tests = vec!["read", "write", "exec"];
2415 for test in tests {
2416 let triple = std::env::var("TARGET_TRIPLE").ok();
2417 let target_args = if let Some(triple) = triple.filter(|t| !t.is_empty()) {
2418 vec!["--target".to_string(), triple.to_string()]
2419 } else {
2420 vec![]
2421 };
2422 let output = std::process::Command::new("cargo")
2423 .args(["test", "-p", "hyperlight-host", "--lib"])
2424 .args(target_args)
2425 .args(["--", "--ignored", test])
2426 .stdin(std::process::Stdio::null())
2427 .output()
2428 .expect("Unable to launch tests");
2429 let exit_code = output.status.code();
2430 if exit_code != Some(TEST_EXIT_CODE.into()) {
2431 eprintln!("=== Guard Page test '{}' failed ===", test);
2432 eprintln!("Exit code: {:?} (expected {})", exit_code, TEST_EXIT_CODE);
2433 eprintln!("=== STDOUT ===");
2434 eprintln!("{}", String::from_utf8_lossy(&output.stdout));
2435 eprintln!("=== STDERR ===");
2436 eprintln!("{}", String::from_utf8_lossy(&output.stderr));
2437 panic!(
2438 "Guard Page test failed: {} (exit code {:?}, expected {})",
2439 test, exit_code, TEST_EXIT_CODE
2440 );
2441 }
2442 }
2443 }
2444 }
2445
2446 #[cfg(not(miri))]
2447 mod from_file_tests {
2448 use std::io::Write;
2449
2450 use tempfile::NamedTempFile;
2451
2452 use crate::mem::shared_mem::{ReadonlySharedMemory, SharedMemory};
2453
2454 pub(super) fn make_temp_file(len: usize) -> NamedTempFile {
2455 let mut f = NamedTempFile::new().expect("create temp file");
2456 if len > 0 {
2457 let mut buf = vec![0u8; len];
2458 for (i, b) in buf.iter_mut().enumerate() {
2459 *b = (i & 0xff) as u8;
2460 }
2461 f.write_all(&buf).expect("write temp file");
2462 f.flush().expect("flush temp file");
2463 }
2464 f
2465 }
2466
2467 #[test]
2468 fn from_file_success_single_page() {
2469 let tmp = make_temp_file(page_size::get());
2470 let mut rsm = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get())
2471 .expect("from_file should succeed");
2472 assert_eq!(rsm.mem_size(), page_size::get());
2473 rsm.with_contents(|slice| {
2474 for (i, b) in slice.iter().enumerate() {
2475 assert_eq!(*b, (i & 0xff) as u8);
2476 }
2477 })
2478 .expect("with_contents should succeed");
2479 }
2480
2481 #[test]
2482 fn from_file_success_smaller_guest_mapped_size() {
2483 let tmp = make_temp_file(2 * page_size::get());
2484 let rsm = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get())
2485 .expect("from_file should succeed");
2486 assert_eq!(rsm.mem_size(), 2 * page_size::get());
2487 }
2488
2489 #[test]
2490 fn from_file_rejects_empty_file() {
2491 let tmp = make_temp_file(0);
2492 let err = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get())
2493 .expect_err("empty file should be rejected");
2494 assert!(format!("{}", err).contains("0x0 < 0x0"));
2495 }
2496
2497 #[test]
2498 fn from_file_rejects_unaligned_file_length() {
2499 let tmp = make_temp_file(page_size::get() + 1);
2500 let err = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get())
2501 .expect_err("unaligned file length should be rejected");
2502 assert!(format!("{}", err).contains(&format!(
2503 "0x{:x} % 0x{:x} = 0",
2504 page_size::get() + 1,
2505 page_size::get()
2506 )));
2507 }
2508
2509 #[test]
2510 fn from_file_rejects_zero_guest_mapped_size() {
2511 let tmp = make_temp_file(page_size::get());
2512 let err = ReadonlySharedMemory::from_file(tmp.as_file(), 0)
2513 .expect_err("zero guest_mapped_size should be rejected");
2514 assert!(format!("{}", err).contains("0x0 < 0x0"));
2515 }
2516
2517 #[test]
2518 fn from_file_rejects_unaligned_guest_mapped_size() {
2519 let tmp = make_temp_file(2 * page_size::get());
2520 let err = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get() + 1)
2521 .expect_err("unaligned guest_mapped_size should be rejected");
2522 assert!(format!("{}", err).contains(&format!(
2523 "0x{:x} % 0x{:x} = ",
2524 page_size::get() + 1,
2525 page_size::get()
2526 )));
2527 }
2528
2529 #[test]
2530 fn from_file_rejects_guest_mapped_size_exceeding_file() {
2531 let tmp = make_temp_file(page_size::get());
2532 let err = ReadonlySharedMemory::from_file(tmp.as_file(), 2 * page_size::get())
2533 .expect_err("guest_mapped_size > file length should be rejected");
2534 assert!(format!("{}", err).contains(&format!(
2535 "0x{:x} <= 0x{:x}",
2536 2 * page_size::get(),
2537 page_size::get()
2538 )));
2539 }
2540
2541 mod guard_page_crash_tests {
2547 use super::make_temp_file;
2548 use crate::mem::shared_mem::{ReadonlySharedMemory, SharedMemory};
2549
2550 #[test]
2552 #[ignore]
2553 pub(super) fn leading_guard_page_traps() {
2554 let tmp = make_temp_file(page_size::get());
2555 let rsm = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get())
2556 .expect("from_file should succeed");
2557 let guard_ptr = unsafe { rsm.base_ptr().sub(page_size::get()) };
2558 println!("reached_guard");
2559 let _ = unsafe { std::ptr::read_volatile(guard_ptr) };
2560 println!("survived_guard");
2561 }
2562
2563 #[test]
2565 #[ignore]
2566 pub(super) fn trailing_guard_page_traps() {
2567 let tmp = make_temp_file(page_size::get());
2568 let rsm = ReadonlySharedMemory::from_file(tmp.as_file(), page_size::get())
2569 .expect("from_file should succeed");
2570 let guard_ptr = unsafe { rsm.base_ptr().add(rsm.mem_size()) };
2571 println!("reached_guard");
2572 let _ = unsafe { std::ptr::read_volatile(guard_ptr) };
2573 println!("survived_guard");
2574 }
2575 }
2576
2577 #[test]
2582 #[cfg_attr(miri, ignore)] fn from_file_guard_page_shim() {
2584 use guard_page_crash_tests::{leading_guard_page_traps, trailing_guard_page_traps};
2585 let ignored_test_paths = [
2586 test_path(leading_guard_page_traps),
2587 test_path(trailing_guard_page_traps),
2588 ];
2589
2590 let exe = std::env::current_exe().expect("current_exe");
2591 for path in &ignored_test_paths {
2592 run_guard_page_subprocess(&exe, path);
2593 }
2594 }
2595
2596 fn test_path<F: Fn()>(_: F) -> &'static str {
2600 let full = std::any::type_name::<F>();
2601 let (_, rest) = full
2602 .split_once("::")
2603 .expect("type_name of a function item is always qualified by the crate name");
2604 rest
2605 }
2606
2607 fn run_guard_page_subprocess(exe: &std::path::Path, ignored_test_path: &str) {
2608 let output = std::process::Command::new(exe)
2609 .args([
2610 "--ignored",
2611 "--nocapture",
2612 "--exact",
2613 "--test-threads=1",
2614 ignored_test_path,
2615 ])
2616 .stdin(std::process::Stdio::null())
2617 .output()
2618 .expect("Unable to launch subprocess test");
2619
2620 let stdout = String::from_utf8_lossy(&output.stdout);
2621 let stderr = String::from_utf8_lossy(&output.stderr);
2622
2623 let ran_test = stdout.contains("running 1 test");
2626 let reached = stdout.contains("reached_guard");
2627 let survived = stdout.contains("survived_guard");
2628 let by_access_violation = killed_by_access_violation(&output.status);
2629
2630 let ok = reached && !survived && by_access_violation && ran_test;
2631 if !ok {
2632 eprintln!("=== Guard page shim failed for {} ===", ignored_test_path);
2633 eprintln!(
2634 "status={:?} ran_test={} reached={} survived={} by_access_violation={}",
2635 output.status, ran_test, reached, survived, by_access_violation
2636 );
2637 eprintln!("=== STDOUT ===\n{}", stdout);
2638 eprintln!("=== STDERR ===\n{}", stderr);
2639 let hint = if !ran_test {
2640 format!(
2641 "\nHINT: ran_test=false (subprocess reported 'running 0 tests'). \
2642 Most likely cause is a stale test path in the shim. Verify that \
2643 `{}` still exists and matches the path passed via --exact above.",
2644 ignored_test_path
2645 )
2646 } else {
2647 String::new()
2648 };
2649 panic!(
2650 "Expected subprocess to run {}, print 'reached_guard', \
2651 then die from a memory access fault. ran_test={}, reached={}, \
2652 survived={}, by_access_violation={}, status={:?}{}",
2653 ignored_test_path,
2654 ran_test,
2655 reached,
2656 survived,
2657 by_access_violation,
2658 output.status,
2659 hint
2660 );
2661 }
2662
2663 println!(
2664 "guard page trap confirmed for {}: subprocess terminated with {:?}",
2665 ignored_test_path, output.status
2666 );
2667 }
2668
2669 fn killed_by_access_violation(status: &std::process::ExitStatus) -> bool {
2673 #[cfg(unix)]
2674 {
2675 use std::os::unix::process::ExitStatusExt;
2676 let expected_signal = if cfg!(target_os = "macos") {
2677 libc::SIGBUS
2678 } else {
2679 libc::SIGSEGV
2680 };
2681 status.signal() == Some(expected_signal)
2682 }
2683 #[cfg(windows)]
2684 {
2685 use windows::Win32::Foundation::STATUS_ACCESS_VIOLATION;
2686 status.code() == Some(STATUS_ACCESS_VIOLATION.0) || status.code() == Some(0xDEAD)
2688 }
2689 }
2690 }
2691}