msb-vm-memory 0.18.0-msb.2

// Copyright (C) 2019 CrowdStrike, Inc. All rights reserved.
// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause

//! Helper structure for working with mmaped memory regions in Windows.

use std;
use std::io;
use std::os::windows::io::{AsRawHandle, RawHandle};
use std::ptr::{null, null_mut};
use std::sync::Arc;

use libc::{c_void, size_t};

use winapi::um::errhandlingapi::GetLastError;
use winapi::um::libloaderapi::{GetModuleHandleA, GetProcAddress};
use winapi::um::processthreadsapi::GetCurrentProcess;

use crate::bitmap::{Bitmap, NewBitmap, BS};
use crate::guest_memory::FileOffset;
use crate::volatile_memory::{self, compute_offset, VolatileMemory, VolatileSlice};

#[allow(non_snake_case)]
#[link(name = "kernel32")]
extern "system" {
    pub fn VirtualAlloc(
        lpAddress: *mut c_void,
        dwSize: size_t,
        flAllocationType: u32,
        flProtect: u32,
    ) -> *mut c_void;

    pub fn VirtualFree(lpAddress: *mut c_void, dwSize: size_t, dwFreeType: u32) -> u32;

    pub fn CreateFileMappingA(
        hFile: RawHandle,                       // HANDLE
        lpFileMappingAttributes: *const c_void, // LPSECURITY_ATTRIBUTES
        flProtect: u32,                         // DWORD
        dwMaximumSizeHigh: u32,                 // DWORD
        dwMaximumSizeLow: u32,                  // DWORD
        lpName: *const u8,                      // LPCSTR
    ) -> RawHandle; // HANDLE

    pub fn MapViewOfFile(
        hFileMappingObject: RawHandle,
        dwDesiredAccess: u32,
        dwFileOffsetHigh: u32,
        dwFileOffsetLow: u32,
        dwNumberOfBytesToMap: size_t,
    ) -> *mut c_void;

    pub fn CloseHandle(hObject: RawHandle) -> u32; // BOOL
    pub fn UnmapViewOfFile(address: *const c_void) -> u32;
}

const MM_HIGHEST_VAD_ADDRESS: u64 = 0x000007FFFFFDFFFF;

const MEM_COMMIT: u32 = 0x00001000;
const MEM_RESERVE: u32 = 0x00002000;
const MEM_RELEASE: u32 = 0x00008000;
const FILE_MAP_ALL_ACCESS: u32 = 0xf001f;
const PAGE_READWRITE: u32 = 0x04;
const PAGE_WRITECOPY: u32 = 0x08;
const FILE_MAP_COPY: u32 = 0x01;
const MEM_EXTENDED_PARAMETER_NUMA_NODE: u64 = 2;

type VirtualAlloc2Fn = unsafe extern "system" fn(
    process: RawHandle,
    base_address: *mut c_void,
    size: size_t,
    allocation_type: u32,
    page_protection: u32,
    extended_parameters: *mut MemExtendedParameter,
    parameter_count: u32,
) -> *mut c_void;

/// ABI-compatible representation of the Windows `MEM_EXTENDED_PARAMETER` structure.
///
/// Both the type/reserved bitfield and its value union occupy one 64-bit word. The NUMA-node
/// parameter stores the parameter type in the low bits of the first word and the preferred node
/// number in the second word.
#[repr(C)]
struct MemExtendedParameter {
    type_and_reserved: u64,
    value: u64,
}

pub const MAP_FAILED: *mut c_void = null_mut::<c_void>();
pub const INVALID_HANDLE_VALUE: RawHandle = (-1isize) as RawHandle;
#[allow(dead_code)]
pub const ERROR_INVALID_PARAMETER: i32 = 87;

/// Helper structure for working with mmaped memory regions in Unix.
///
/// The structure is used for accessing the guest's physical memory by mmapping it into
/// the current process.
///
/// # Limitations
/// When running a 64-bit virtual machine on a 32-bit hypervisor, only part of the guest's
/// physical memory may be mapped into the current process due to the limited virtual address
/// space size of the process.
#[derive(Debug)]
pub struct MmapRegion<B> {
    addr: *mut u8,
    size: usize,
    bitmap: B,
    file_offset: Option<FileOffset>,
    // Slices of a file view share one address mapping and one final unmap. In
    // particular, slicing must not establish independent CoW copies of aliases.
    view: Option<Arc<MappedView>>,
}

#[derive(Debug)]
struct MappedView {
    address: usize,
}

impl Drop for MappedView {
    fn drop(&mut self) {
        // The final region owner has released all users of this mapped view.
        unsafe {
            UnmapViewOfFile(self.address as *const c_void);
        }
    }
}

// Send and Sync aren't automatically inherited for the raw address pointer.
// Accessing that pointer is only done through the stateless interface which
// allows the object to be shared by multiple threads without a decrease in
// safety.
unsafe impl<B: Send> Send for MmapRegion<B> {}
unsafe impl<B: Sync> Sync for MmapRegion<B> {}

impl<B: NewBitmap> MmapRegion<B> {
    /// Creates a shared anonymous mapping of `size` bytes.
    ///
    /// # Arguments
    /// * `size` - The size of the memory region in bytes.
    pub fn new(size: usize) -> io::Result<Self> {
        if (size == 0) || (size > MM_HIGHEST_VAD_ADDRESS as usize) {
            return Err(io::Error::from_raw_os_error(libc::EINVAL));
        }
        // This is safe because we are creating an anonymous mapping in a place not already used by
        // any other area in this process.
        let addr = unsafe { VirtualAlloc(null_mut::<c_void>(), size, MEM_COMMIT, PAGE_READWRITE) };
        if addr == MAP_FAILED {
            return Err(io::Error::last_os_error());
        }
        Ok(Self {
            addr: addr as *mut u8,
            size,
            bitmap: B::with_len(size),
            file_offset: None,
            view: None,
        })
    }

    /// Creates an anonymous mapping whose physical pages prefer `node` when first accessed.
    ///
    /// Windows treats the NUMA node as a preference rather than a strict binding. The virtual range
    /// is reserved and committed up front, while physical pages remain demand-faulted according to
    /// the operating system's ordinary committed-memory behavior.
    ///
    /// Returns [`io::ErrorKind::Unsupported`] when the host does not export `VirtualAlloc2`.
    pub fn new_on_numa_node(size: usize, node: u32) -> io::Result<Self> {
        if (size == 0) || (size > MM_HIGHEST_VAD_ADDRESS as usize) {
            return Err(io::Error::from_raw_os_error(libc::EINVAL));
        }

        let virtual_alloc2 = resolve_virtual_alloc2()?;
        let mut numa_parameter = MemExtendedParameter {
            type_and_reserved: MEM_EXTENDED_PARAMETER_NUMA_NODE,
            value: u64::from(node),
        };

        // `VirtualAlloc2` owns the returned allocation exactly like `VirtualAlloc`, so the existing
        // `Drop` implementation can release both constructor paths with `VirtualFree`.
        let addr = unsafe {
            virtual_alloc2(
                GetCurrentProcess() as RawHandle,
                null_mut(),
                size,
                MEM_RESERVE | MEM_COMMIT,
                PAGE_READWRITE,
                &mut numa_parameter,
                1,
            )
        };
        if addr == MAP_FAILED {
            return Err(io::Error::last_os_error());
        }

        Ok(Self {
            addr: addr.cast::<u8>(),
            size,
            bitmap: B::with_len(size),
            file_offset: None,
            view: None,
        })
    }

    /// Creates a shared file mapping of `size` bytes.
    ///
    /// # Arguments
    /// * `file_offset` - The mapping will be created at offset `file_offset.start` in the file
    ///   referred to by `file_offset.file`.
    /// * `size` - The size of the memory region in bytes.
    pub fn from_file(file_offset: FileOffset, size: usize) -> io::Result<Self> {
        Self::map_file(file_offset, size, false)
    }

    /// Creates a private copy-on-write view of immutable file bytes.
    ///
    /// Writes never modify the file. The caller must prevent other handles from
    /// modifying or truncating the backing while any region still references it.
    pub fn from_file_private(file_offset: FileOffset, size: usize) -> io::Result<Self> {
        Self::map_file(file_offset, size, true)
    }

    /// Creates a region sharing a checked subrange of this file view.
    ///
    /// This does not remap memory or create another CoW instance. It permits
    /// guest slots with page-aligned offsets inside one allocation-granularity-
    /// aligned Windows mapping. The slice retains ownership after `self` drops.
    pub fn file_slice(&self, offset: usize, size: usize) -> io::Result<Self> {
        let view = self
            .view
            .as_ref()
            .ok_or_else(|| io::Error::from_raw_os_error(libc::EINVAL))?;
        let file = self.file_offset.as_ref().unwrap();
        if size == 0 || offset.checked_add(size).is_none_or(|end| end > self.size) {
            return Err(io::Error::from_raw_os_error(libc::EINVAL));
        }
        Ok(Self {
            // Bounds above keep the entire slice inside the live view.
            addr: unsafe { self.addr.add(offset) },
            size,
            bitmap: B::with_len(size),
            file_offset: Some(FileOffset::from_arc(
                file.arc().clone(),
                file.start() + offset as u64,
            )),
            view: Some(view.clone()),
        })
    }

    fn map_file(file_offset: FileOffset, size: usize, private: bool) -> io::Result<Self> {
        let handle = file_offset.file().as_raw_handle();
        if handle == INVALID_HANDLE_VALUE {
            return Err(io::Error::from_raw_os_error(libc::EBADF));
        }
        let file_len = file_offset.file().metadata()?.len();
        if size == 0
            || file_offset
                .start()
                .checked_add(size as u64)
                .is_none_or(|end| end > file_len)
        {
            return Err(io::Error::from_raw_os_error(libc::EINVAL));
        }

        let mapping = unsafe {
            CreateFileMappingA(
                handle,
                null(),
                if private {
                    PAGE_WRITECOPY
                } else {
                    PAGE_READWRITE
                },
                0,
                0,
                null(),
            )
        };
        if mapping == 0 as RawHandle {
            return Err(io::Error::last_os_error());
        }

        let offset = file_offset.start();

        // This is safe because we are creating a mapping in a place not already used by any other
        // area in this process.
        let addr = unsafe {
            MapViewOfFile(
                mapping,
                if private {
                    FILE_MAP_COPY
                } else {
                    FILE_MAP_ALL_ACCESS
                },
                (offset >> 32) as u32,
                offset as u32,
                size,
            )
        };

        // Closing the section may overwrite the thread's last-error value.
        // Preserve the mapping failure before releasing that temporary handle.
        let map_error = addr.is_null().then(io::Error::last_os_error);
        unsafe {
            CloseHandle(mapping);
        }

        if let Some(error) = map_error {
            return Err(error);
        }
        Ok(Self {
            addr: addr as *mut u8,
            size,
            bitmap: B::with_len(size),
            file_offset: Some(file_offset),
            view: Some(Arc::new(MappedView {
                address: addr as usize,
            })),
        })
    }
}

fn resolve_virtual_alloc2() -> io::Result<VirtualAlloc2Fn> {
    // Resolve dynamically so binaries using the ordinary constructor retain their existing host
    // compatibility and a managed NUMA request can fail with a regular capability error.
    // The SDK documents Kernel32, while current Windows hosts implement the forwarded export in
    // KernelBase. Probe both so this also works on hosts where GetProcAddress does not expose the
    // forwarder from Kernel32.
    for module_name in [b"kernelbase.dll\0".as_slice(), b"kernel32.dll\0".as_slice()] {
        let module = unsafe { GetModuleHandleA(module_name.as_ptr().cast()) };
        if module.is_null() {
            continue;
        }

        let procedure = unsafe { GetProcAddress(module, b"VirtualAlloc2\0".as_ptr().cast()) };
        if !procedure.is_null() {
            // `GetProcAddress` erases the signature. The symbol is accepted only under the
            // documented `VirtualAlloc2` name, whose ABI is fixed by memoryapi.h.
            return Ok(unsafe { std::mem::transmute::<_, VirtualAlloc2Fn>(procedure) });
        }
    }

    Err(io::Error::new(
        io::ErrorKind::Unsupported,
        "VirtualAlloc2 is unavailable on this Windows host",
    ))
}

impl<B: Bitmap> MmapRegion<B> {
    /// Returns a pointer to the beginning of the memory region. Mutable accesses performed
    /// using the resulting pointer are not automatically accounted for by the dirty bitmap
    /// tracking functionality.
    ///
    /// Should only be used for passing this region to ioctls for setting guest memory.
    pub fn as_ptr(&self) -> *mut u8 {
        self.addr
    }

    /// Returns the size of this region.
    pub fn size(&self) -> usize {
        self.size
    }

    /// Returns information regarding the offset into the file backing this region (if any).
    pub fn file_offset(&self) -> Option<&FileOffset> {
        self.file_offset.as_ref()
    }

    /// Returns a reference to the inner bitmap object.
    pub fn bitmap(&self) -> &B {
        &self.bitmap
    }
}

impl<B: Bitmap> VolatileMemory for MmapRegion<B> {
    type B = B;

    fn len(&self) -> usize {
        self.size
    }

    fn get_slice(
        &self,
        offset: usize,
        count: usize,
    ) -> volatile_memory::Result<VolatileSlice<'_, BS<'_, Self::B>>> {
        let end = compute_offset(offset, count)?;
        if end > self.size {
            return Err(volatile_memory::Error::OutOfBounds { addr: end });
        }

        // Safe because we checked that offset + count was within our range and we only ever hand
        // out volatile accessors.
        Ok(unsafe {
            VolatileSlice::with_bitmap(
                self.addr.add(offset),
                count,
                self.bitmap.slice_at(offset),
                None,
            )
        })
    }
}

impl<B> Drop for MmapRegion<B> {
    fn drop(&mut self) {
        if self.view.is_some() {
            // Arc releases the base address with UnmapViewOfFile, once even
            // when this region is only an interior slice of the original view.
            return;
        }
        // This is safe because we mmap the area at addr ourselves, and nobody
        // else is holding a reference to it.
        // Note that the size must be set to 0 when using MEM_RELEASE,
        // otherwise the function fails.
        unsafe {
            let ret_val = VirtualFree(self.addr as *mut libc::c_void, 0, MEM_RELEASE);
            if ret_val == 0 {
                let err = GetLastError();
                // We can't use any fancy logger here, yet we want to
                // pin point memory leaks.
                println!(
                    "WARNING: Could not deallocate mmap region. \
                     Address: {:?}. Size: {}. Error: {}",
                    self.addr, self.size, err
                )
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use std::os::windows::io::FromRawHandle;

    #[cfg(feature = "backend-bitmap")]
    use crate::bitmap::AtomicBitmap;
    use crate::guest_memory::FileOffset;
    use crate::mmap::windows::INVALID_HANDLE_VALUE;

    type MmapRegion = super::MmapRegion<()>;

    #[test]
    fn private_file_views_share_slices_but_isolate_children() {
        use std::fs::{File, OpenOptions};
        use std::io::Write;
        let path = std::env::temp_dir().join(format!(
            "private-view-{}-{}.ram",
            std::process::id(),
            std::time::SystemTime::now()
                .duration_since(std::time::UNIX_EPOCH)
                .unwrap()
                .as_nanos()
        ));
        let mut writer = OpenOptions::new()
            .write(true)
            .create_new(true)
            .open(&path)
            .unwrap();
        writer.write_all(&vec![0x11; 65536]).unwrap();
        drop(writer);
        let a =
            MmapRegion::from_file_private(FileOffset::new(File::open(&path).unwrap(), 0), 65536)
                .unwrap();
        let b =
            MmapRegion::from_file_private(FileOffset::new(File::open(&path).unwrap(), 0), 65536)
                .unwrap();
        let slice = a.file_slice(4096, 4096).unwrap();
        unsafe {
            slice.as_ptr().write_volatile(0x42);
            assert_eq!(a.as_ptr().add(4096).read_volatile(), 0x42);
            assert_eq!(b.as_ptr().add(4096).read_volatile(), 0x11);
        }
        assert!(a.file_slice(65536, 1).is_err());
        assert!(a.file_slice(usize::MAX, 2).is_err());
        drop(a);
        assert_eq!(unsafe { slice.as_ptr().read_volatile() }, 0x42);
        assert_eq!(std::fs::read(&path).unwrap(), vec![0x11; 65536]);
        drop(slice);
        drop(b);
        std::fs::remove_file(path).unwrap();
    }

    #[test]
    fn map_invalid_handle() {
        let file = unsafe { std::fs::File::from_raw_handle(INVALID_HANDLE_VALUE) };
        let file_offset = FileOffset::new(file, 0);
        let e = MmapRegion::from_file(file_offset, 1024).unwrap_err();
        assert_eq!(e.raw_os_error(), Some(libc::EBADF));
    }

    #[test]
    fn map_on_numa_node_rejects_invalid_sizes() {
        assert_eq!(
            MmapRegion::new_on_numa_node(0, 0)
                .unwrap_err()
                .raw_os_error(),
            Some(libc::EINVAL)
        );
    }

    #[test]
    fn map_on_numa_node_allocates_accessible_memory() {
        let mapping = MmapRegion::new_on_numa_node(0x1_0000, 0).unwrap();
        assert_eq!(mapping.size(), 0x1_0000);

        unsafe {
            mapping.as_ptr().write_volatile(0x5a);
            assert_eq!(mapping.as_ptr().read_volatile(), 0x5a);
        }
    }

    #[test]
    #[cfg(feature = "backend-bitmap")]
    fn test_dirty_tracking() {
        // Using the `crate` prefix because we aliased `MmapRegion` to `MmapRegion<()>` for
        // the rest of the unit tests above.
        let m = crate::MmapRegion::<AtomicBitmap>::new(0x1_0000).unwrap();
        crate::bitmap::tests::test_volatile_memory(&m);
    }
}