vmi_driver_xen/

lib.rs

//! VMI driver for Xen hypervisor.

mod arch;
mod busy_wait;
mod convert;
mod core;

use std::{
    cell::{Cell, RefCell},
    collections::HashMap,
    os::fd::AsRawFd as _,
    time::{Duration, Instant},
};

use vmi_core::{
    Architecture, Gfn, MemoryAccess, MemoryAccessOptions, VcpuId, View, VmiDriver, VmiError,
    VmiEvent, VmiEventResponse, VmiInfo, VmiMappedPage,
    driver::{
        VmiEventControl, VmiQueryProtection, VmiQueryRegisters, VmiRead, VmiSetProtection,
        VmiSetRegisters, VmiViewControl, VmiVmControl, VmiWrite,
    },
};
pub use xen::XenDomainId; // Only `libxen` type in `VmiXenDriver`'s public API.
use xen::{
    XenAltP2M, XenAltP2MView, XenControl, XenDeviceModel, XenDomain, XenDomainInfo,
    XenEventChannelPort, XenForeignMemory, XenForeignMemoryProtection, XenMonitor, XenStore,
    ctrl::VmEventRing,
};

use self::convert::{FromExt, IntoExt, TryFromExt};
pub use self::{arch::ArchAdapter, busy_wait::VmiXenBusyWait};

struct XenView {
    view: XenAltP2MView,
    default_access: xen::MemoryAccess,
}

fn is_missing_altp2m_entry(err: &xen::XenError) -> bool {
    matches!(
        err,
        xen::XenError::Io(inner) if inner.raw_os_error() == Some(libc::ESRCH)
    )
}

/// VMI driver for Xen hypervisor.
pub struct VmiXenDriver<Arch>
where
    Arch: ArchAdapter,
{
    pub(crate) domain: XenDomain<Arch::XenArch>,
    pub(crate) devicemodel: XenDeviceModel,
    pub(crate) monitor: XenMonitor,
    pub(crate) altp2m: XenAltP2M,
    pub(crate) evtchn: XenEventChannelPort,
    pub(crate) foreign_memory: XenForeignMemory,
    pub(crate) info: XenDomainInfo,

    pub(crate) ring: RefCell<VmEventRing>,
    pub(crate) views: RefCell<HashMap<u16, XenView>>,
    pub(crate) event_processing_overhead: Cell<Duration>,

    pub(crate) busy_wait: VmiXenBusyWait,
    pub(crate) busy_window: Cell<Duration>,
    pub(crate) masked: Cell<bool>,
    pub(crate) answered: Cell<Option<Instant>>,
}

impl<Arch> Drop for VmiXenDriver<Arch>
where
    Arch: ArchAdapter,
{
    fn drop(&mut self) {
        let max_memkb = self.info.max_pages * Arch::PAGE_SIZE / 1024;

        let _ = self.domain.set_max_mem(max_memkb);
        let _ = self.monitor.emul_unimplemented(false);
        let _ = self.monitor.inguest_pagefault(false);
    }
}

impl<Arch> VmiXenDriver<Arch>
where
    Arch: ArchAdapter,
{
    /// Creates a new VMI driver for Xen hypervisor.
    pub fn new(domain_id: XenDomainId) -> Result<Self, VmiError> {
        let xc = XenControl::new().map_err(VmiError::driver)?;
        let domain = xc.domain(domain_id).map_err(VmiError::driver)?;
        domain.set_max_mem(u64::MAX).map_err(VmiError::driver)?;

        let devicemodel = domain.device_model().map_err(VmiError::driver)?;
        let (monitor, ring) = domain.monitor().map_err(VmiError::driver)?;
        let altp2m = domain.altp2m().map_err(VmiError::driver)?;
        let evtchn = monitor.channel().map_err(VmiError::driver)?;
        let foreign_memory = XenForeignMemory::new().map_err(VmiError::driver)?;
        let info = domain.info().map_err(VmiError::driver)?;

        monitor.inguest_pagefault(true).map_err(VmiError::driver)?;
        monitor.emul_unimplemented(true).map_err(VmiError::driver)?;

        Ok(Self {
            domain,
            devicemodel,
            monitor,
            altp2m,
            evtchn,
            foreign_memory,
            info,
            ring: RefCell::new(ring),
            views: RefCell::new(HashMap::new()),
            event_processing_overhead: Cell::new(Duration::ZERO),
            busy_wait: VmiXenBusyWait::default(),
            busy_window: Cell::new(Duration::ZERO),
            masked: Cell::new(false),
            answered: Cell::new(None),
        })
    }

    /// Creates a new VMI driver from environment variables.
    ///
    /// The first of the following variables that is **set** selects the target
    /// domain.
    ///
    /// - `VMI_XEN_DOMAIN`: interpreted as a numeric domain ID, falling back to
    ///   a domain name if the value is not numeric.
    /// - `VMI_XEN_DOMAIN_ID`: interpreted as a numeric domain ID.
    /// - `VMI_XEN_DOMAIN_NAME`: interpreted as a domain name.
    ///
    /// Returns `Ok(None)` if none of the variables are set, or if the selected
    /// variable does not resolve to a domain.
    pub fn try_from_env() -> Result<Option<Self>, VmiError> {
        fn resolve_domain(domain_name: &str) -> Result<Option<XenDomainId>, VmiError> {
            XenStore::new()
                .map_err(VmiError::driver)?
                .domain_id_from_name(domain_name)
                .map_err(VmiError::driver)
        }

        if let Ok(var) = std::env::var("VMI_XEN_DOMAIN") {
            if let Ok(domain_id) = var.parse() {
                return Ok(Some(Self::new(XenDomainId(domain_id))?));
            }

            if let Some(domain_id) = resolve_domain(&var)? {
                return Ok(Some(Self::new(domain_id)?));
            }
        }
        else if let Ok(var) = std::env::var("VMI_XEN_DOMAIN_ID") {
            match var.parse() {
                Ok(domain_id) => return Ok(Some(Self::new(XenDomainId(domain_id))?)),
                Err(err) => tracing::error!(%err, var, "invalid value for VMI_XEN_DOMAIN_ID"),
            }
        }
        else if let Ok(var) = std::env::var("VMI_XEN_DOMAIN_NAME")
            && let Some(domain_id) = resolve_domain(&var)?
        {
            return Ok(Some(Self::new(domain_id)?));
        }

        Ok(None)
    }

    /// Sets how the driver waits for the next request after an answer.
    ///
    /// The default is [`VmiXenBusyWait::Adaptive`] with a `start` of 10
    /// microseconds and a `max` of 50 microseconds.
    pub fn with_busy_wait(mut self, busy_wait: VmiXenBusyWait) -> Self {
        self.busy_wait = busy_wait;
        self
    }

    /// Returns the port of the event channel that notifies the driver of
    /// events.
    ///
    /// The port number belongs to the domain that runs the driver, usually
    /// dom0.
    pub fn event_channel_port(&self) -> u32 {
        self.evtchn.local_port()
    }

    /// Checks the ring until a request arrives or the busy-wait time ends.
    ///
    /// Returns whether a request arrived. Unmasks the event channel when none
    /// did.
    fn busy_wait_for_request(&self) -> Result<bool, VmiError> {
        if !self.masked.get() {
            return Ok(false);
        }

        let end = Instant::now() + self.busy_window.get();
        loop {
            if self.ring.borrow().has_unconsumed_requests() {
                return Ok(true);
            }

            if Instant::now() >= end {
                break;
            }

            std::hint::spin_loop();
        }

        self.evtchn.unmask().map_err(VmiError::driver)?;
        self.masked.set(false);
        Ok(false)
    }

    /// Sleeps until Xen notifies the driver of a request or the deadline
    /// passes.
    ///
    /// Afterwards the event channel stays masked while the busy-wait time is
    /// not zero.
    fn sleep_for_request(&self, deadline: Instant) -> Result<(), VmiError> {
        let mut fds = [libc::pollfd {
            fd: self.evtchn.as_raw_fd(),
            events: libc::POLLIN | libc::POLLERR,
            revents: 0,
        }];

        loop {
            let remaining = deadline.saturating_duration_since(Instant::now());
            let timeout = remaining
                .as_micros()
                .div_ceil(1000)
                .try_into()
                .map_err(|_| VmiError::InvalidTimeout)?;

            #[rustfmt::skip]
            let poll_result = unsafe {
                libc::poll(
                    fds.as_mut_ptr() as _,
                    fds.len() as _,
                    timeout
                )
            };

            match poll_result {
                0 => return Err(VmiError::Timeout),
                -1 => return Err(VmiError::Io(std::io::Error::last_os_error())),
                _ => (),
            }

            let port = self.evtchn.pending().map_err(VmiError::driver)?;
            assert_eq!(port, self.evtchn.local_port());

            if self.ring.borrow().has_unconsumed_requests() {
                break;
            }

            // Xen notified the port while it was masked, for a request that
            // the busy wait already handled.
            self.evtchn.unmask().map_err(VmiError::driver)?;
        }

        if let Some(answered) = self.answered.get() {
            let window = self
                .busy_wait
                .next_window(self.busy_window.get(), answered.elapsed());
            self.busy_window.set(window);
        }

        if self.busy_window.get().is_zero() {
            self.evtchn.unmask().map_err(VmiError::driver)?;
        }
        else {
            self.masked.set(true);
        }

        Ok(())
    }
}

impl<Arch> VmiDriver for VmiXenDriver<Arch>
where
    Arch: ArchAdapter,
{
    type Architecture = Arch;

    fn info(&self) -> Result<VmiInfo, VmiError> {
        Ok(VmiInfo {
            page_size: Arch::PAGE_SIZE,
            page_shift: Arch::PAGE_SHIFT,
            max_gfn: Gfn::new(self.domain.maximum_gpfn().map_err(VmiError::driver)?),
            vcpus: self.info.max_vcpu_id + 1,
        })
    }
}

impl<Arch> VmiRead for VmiXenDriver<Arch>
where
    Arch: ArchAdapter,
{
    fn read_page(&self, gfn: Gfn) -> Result<VmiMappedPage, VmiError> {
        let page = self
            .foreign_memory
            .map(
                self.domain.id(),
                XenForeignMemoryProtection::READ,
                &[u64::from(gfn)],
                None,
            )
            .map_err(VmiError::driver)?;

        Ok(VmiMappedPage::new(page))
    }
}

impl<Arch> VmiWrite for VmiXenDriver<Arch>
where
    Arch: ArchAdapter,
{
    fn write_page(&self, gfn: Gfn, offset: u64, content: &[u8]) -> Result<VmiMappedPage, VmiError> {
        let mut page = self
            .foreign_memory
            .map(
                self.domain.id(),
                XenForeignMemoryProtection::WRITE,
                &[u64::from(gfn)],
                None,
            )
            .map_err(VmiError::driver)?;

        let offset = offset as usize;
        if offset + content.len() > Arch::PAGE_SIZE as usize {
            return Err(VmiError::OutOfBounds);
        }

        page[offset..offset + content.len()].copy_from_slice(content);

        Ok(VmiMappedPage::new(page))
    }
}

impl<Arch> VmiQueryProtection for VmiXenDriver<Arch>
where
    Arch: ArchAdapter,
{
    fn memory_access(&self, gfn: Gfn, view: View) -> Result<MemoryAccess, VmiError> {
        Ok(self.memory_access_with_options(gfn, view)?.0)
    }

    fn memory_access_with_options(
        &self,
        gfn: Gfn,
        view: View,
    ) -> Result<(MemoryAccess, MemoryAccessOptions), VmiError> {
        // As of Xen 4.22, xc_altp2m_get_mem_access() reports ESRCH for a valid
        // GFN whose entry has not been materialized in a lazy altp2m view.
        // Ideally, Xen would return the view's effective default access in
        // this case. Until it does, validate the GFN through the domain p2m,
        // discard its unrelated access value and fall back to the default
        // access recorded for the view. Materialized Xen modes are decoded
        // into both permissions and options.

        if view.0 == 0 {
            let xen_access = self
                .domain
                .get_mem_access(gfn.0)
                .map_err(VmiError::driver)?;

            let access = MemoryAccess::from_ext(xen_access);
            let options = MemoryAccessOptions::from_ext(xen_access);

            tracing::trace!(%gfn, %view, ?xen_access, "get memory access");

            return Ok((access, options));
        }

        let views = self.views.borrow();
        let xen_view = match views.get(&view.0) {
            Some(view) => view,
            None => return Err(VmiError::ViewNotFound),
        };

        let xen_access = match xen_view.view.get_mem_access(gfn.0) {
            Ok(xen_access) => xen_access,
            Err(err) if is_missing_altp2m_entry(&err) => {
                // Confirm the GFN exists in the base p2m before treating ESRCH
                // as a lazy, unmaterialized altp2m entry.
                self.domain
                    .get_mem_access(gfn.0)
                    .map_err(VmiError::driver)?;

                let xen_access = xen_view.default_access;

                let access = MemoryAccess::from_ext(xen_access);
                let options = MemoryAccessOptions::from_ext(xen_access);

                tracing::trace!(%gfn, %view, ?xen_access, "get memory access (default)");

                return Ok((access, options));
            }
            Err(err) => return Err(VmiError::driver(err)),
        };

        let access = MemoryAccess::from_ext(xen_access);
        let options = MemoryAccessOptions::from_ext(xen_access);

        tracing::trace!(%gfn, %view, ?xen_access, "get memory access");

        Ok((access, options))
    }
}

impl<Arch> VmiSetProtection for VmiXenDriver<Arch>
where
    Arch: ArchAdapter,
{
    fn set_memory_access(
        &self,
        gfn: Gfn,
        view: View,
        access: MemoryAccess,
    ) -> Result<(), VmiError> {
        tracing::trace!(%gfn, %view, %access, "set memory access");

        if view.0 == 0 {
            return self
                .domain
                .set_mem_access(gfn.into(), access.into_ext())
                .map_err(VmiError::driver);
        }

        match self.views.borrow().get(&view.0) {
            Some(view) => view
                .view
                .set_mem_access(gfn.into(), access.into_ext())
                .map_err(VmiError::driver),
            None => Err(VmiError::ViewNotFound),
        }
    }

    fn set_memory_access_with_options(
        &self,
        gfn: Gfn,
        view: View,
        access: MemoryAccess,
        options: MemoryAccessOptions,
    ) -> Result<(), VmiError> {
        tracing::trace!(%gfn, %view, %access, "set memory access");

        let mut xen_access = access.into_ext();

        if options.contains(MemoryAccessOptions::IGNORE_PAGE_WALK_UPDATES) {
            if access != MemoryAccess::R {
                return Err(VmiError::NotSupported);
            }

            xen_access = xen::MemoryAccess::R_PW;
        }

        if view.0 == 0 {
            return self
                .domain
                .set_mem_access(gfn.into(), xen_access)
                .map_err(VmiError::driver);
        }

        match self.views.borrow().get(&view.0) {
            Some(view) => view
                .view
                .set_mem_access(gfn.into(), xen_access)
                .map_err(VmiError::driver),
            None => Err(VmiError::ViewNotFound),
        }
    }
}

impl<Arch> VmiQueryRegisters for VmiXenDriver<Arch>
where
    Arch: ArchAdapter,
{
    fn registers(&self, vcpu: VcpuId) -> Result<Arch::Registers, VmiError> {
        Arch::registers(self, vcpu)
    }
}

impl<Arch> VmiSetRegisters for VmiXenDriver<Arch>
where
    Arch: ArchAdapter,
{
    fn set_registers(&self, vcpu: VcpuId, registers: Arch::Registers) -> Result<(), VmiError> {
        Arch::set_registers(self, vcpu, registers)
    }
}

impl<Arch> VmiViewControl for VmiXenDriver<Arch>
where
    Arch: ArchAdapter,
{
    fn default_view(&self) -> View {
        View(0)
    }

    fn create_view(&self, default_access: MemoryAccess) -> Result<View, VmiError> {
        let default_access = default_access.into_ext();

        let view = self
            .altp2m
            .create_view(default_access)
            .map_err(VmiError::driver)?;

        let id = view.id();
        self.views.borrow_mut().insert(
            id,
            XenView {
                view,
                default_access,
            },
        );

        Ok(View(id))
    }

    fn destroy_view(&self, view: View) -> Result<(), VmiError> {
        if view.0 == 0 {
            return Ok(());
        }

        match self.views.borrow_mut().remove(&view.0) {
            // View is destroyed automatically when it goes out of scope
            Some(_view) => Ok(()),
            None => Err(VmiError::ViewNotFound),
        }
    }

    fn switch_to_view(&self, view: View) -> Result<(), VmiError> {
        if view.0 == 0 {
            return self.altp2m.reset_view().map_err(VmiError::driver);
        }

        match self.views.borrow().get(&view.0) {
            Some(view) => view.view.switch().map_err(VmiError::driver),
            None => Err(VmiError::ViewNotFound),
        }
    }

    fn change_view_gfn(&self, view: View, old_gfn: Gfn, new_gfn: Gfn) -> Result<(), VmiError> {
        if view.0 == 0 {
            return Ok(());
        }

        match self.views.borrow().get(&view.0) {
            // WARNING: This will change access permissions of the GFN!
            Some(view) => view
                .view
                .change_gfn(old_gfn.into(), new_gfn.into())
                .map_err(VmiError::driver),
            None => Err(VmiError::ViewNotFound),
        }
    }

    fn reset_view_gfn(&self, view: View, gfn: Gfn) -> Result<(), VmiError> {
        if view.0 == 0 {
            return Ok(());
        }

        match self.views.borrow().get(&view.0) {
            // WARNING: This will change access permissions of the GFN!
            Some(view) => view
                .view
                .change_gfn(gfn.into(), u64::MAX)
                .map_err(VmiError::driver),
            None => Err(VmiError::ViewNotFound),
        }
    }
}

impl<Arch> VmiEventControl for VmiXenDriver<Arch>
where
    Arch: ArchAdapter,
{
    fn monitor_enable(&self, option: Arch::EventMonitor) -> Result<(), VmiError> {
        Arch::monitor_enable(self, option)
    }

    fn monitor_disable(&self, option: Arch::EventMonitor) -> Result<(), VmiError> {
        Arch::monitor_disable(self, option)
    }

    fn events_pending(&self) -> usize {
        self.ring.borrow().unconsumed_requests()
    }

    fn event_processing_overhead(&self) -> Duration {
        self.event_processing_overhead.get()
    }

    fn wait_for_event(
        &self,
        timeout: Duration,
        mut handler: impl FnMut(&VmiEvent<Arch>) -> VmiEventResponse<Arch>,
    ) -> Result<(), VmiError> {
        i32::try_from(timeout.as_millis()).map_err(|_| VmiError::InvalidTimeout)?;
        let deadline = Instant::now() + timeout;

        if !self.busy_wait_for_request()? {
            self.sleep_for_request(deadline)?;
        }

        struct OverheadGuard<'a, Arch>
        where
            Arch: ArchAdapter,
        {
            driver: &'a VmiXenDriver<Arch>,
            start: Instant,
        }

        impl<'a, Arch> OverheadGuard<'a, Arch>
        where
            Arch: ArchAdapter,
        {
            fn new(driver: &'a VmiXenDriver<Arch>) -> Self {
                Self {
                    driver,
                    start: Instant::now(),
                }
            }
        }

        impl<Arch> Drop for OverheadGuard<'_, Arch>
        where
            Arch: ArchAdapter,
        {
            fn drop(&mut self) {
                let elapsed = Instant::now().duration_since(self.start);
                let overhead = &self.driver.event_processing_overhead;
                overhead.set(overhead.get() + elapsed);
            }
        }

        {
            let _overhead_guard = OverheadGuard::new(self);

            while self.ring.borrow().has_unconsumed_requests() {
                let mut event = self.ring.borrow_mut().get_request();
                Arch::process_event(self, &mut event, &mut handler)?;
                self.ring.borrow_mut().put_response(event);
            }
        }

        self.evtchn.notify().map_err(VmiError::driver)?;
        self.answered.set(Some(Instant::now()));

        Ok(())
    }
}

impl<Arch> VmiVmControl for VmiXenDriver<Arch>
where
    Arch: ArchAdapter,
{
    fn pause(&self) -> Result<(), VmiError> {
        self.domain.pause().map_err(VmiError::driver)
    }

    fn resume(&self) -> Result<(), VmiError> {
        self.domain.unpause().map_err(VmiError::driver)
    }

    fn allocate_gfn(&self) -> Result<Gfn, VmiError> {
        let gfn = Gfn::new(self.domain.maximum_gpfn().map_err(VmiError::driver)?) + 1;
        self.allocate_gfn_at(gfn)?;
        Ok(gfn)
    }

    fn allocate_gfn_at(&self, gfn: Gfn) -> Result<(), VmiError> {
        self.domain
            .populate_physmap_exact(0, 0, &[gfn.into()])
            .map_err(VmiError::driver)
    }

    fn free_gfn(&self, gfn: Gfn) -> Result<(), VmiError> {
        self.domain
            .decrease_reservation_exact(0, &[gfn.into()])
            .map_err(VmiError::driver)
    }

    fn inject_interrupt(&self, vcpu: VcpuId, interrupt: Arch::Interrupt) -> Result<(), VmiError> {
        Arch::inject_interrupt(self, vcpu, interrupt)
    }

    fn reset_state(&self) -> Result<(), VmiError> {
        Arch::reset_state(self)
    }
}

#[cfg(test)]
mod tests {
    use super::is_missing_altp2m_entry;

    #[test]
    fn identifies_missing_altp2m_entries() {
        let missing = xen::XenError::Io(std::io::Error::from_raw_os_error(libc::ESRCH));
        let invalid = xen::XenError::Io(std::io::Error::from_raw_os_error(libc::EINVAL));

        assert!(is_missing_altp2m_entry(&missing));
        assert!(!is_missing_altp2m_entry(&invalid));
        assert!(!is_missing_altp2m_entry(&xen::XenError::Other(
            "other error"
        )));
    }
}