#[cfg(target_arch = "x86_64")]
mod x86_64;
#[cfg(target_arch = "aarch64")]
mod aarch64;
#[cfg(gdb)]
use std::collections::HashMap;
use std::str::FromStr;
use std::sync::{Arc, Mutex};
#[cfg(target_arch = "aarch64")]
pub(crate) use aarch64::*;
use hyperlight_common::log_level::GuestLogFilter;
use tracing_core::LevelFilter;
use crate::HyperlightError;
#[cfg(gdb)]
use crate::hypervisor::gdb::DebuggableVm;
#[cfg(gdb)]
use crate::hypervisor::gdb::arch::VcpuStopReasonError;
#[cfg(gdb)]
use crate::hypervisor::gdb::{
DebugCommChannel, DebugError, DebugMsg, DebugResponse, GdbTargetError, VcpuStopReason,
};
#[cfg(gdb)]
use crate::hypervisor::hyperlight_vm::x86_64::debug::ProcessDebugRequestError;
#[cfg(not(gdb))]
use crate::hypervisor::virtual_machine::VirtualMachine;
use crate::hypervisor::virtual_machine::{
MapMemoryError, RegisterError, RunVcpuError, UnmapMemoryError, VmError, VmExit,
};
use crate::hypervisor::{InterruptHandle, InterruptHandleImpl};
use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags, MemoryRegionType};
use crate::mem::mgr::{SandboxMemoryManager, SnapshotSharedMemory};
use crate::mem::shared_mem::{GuestSharedMemory, HostSharedMemory, SharedMemory};
use crate::metrics::{METRIC_ERRONEOUS_VCPU_KICKS, METRIC_GUEST_CANCELLATION};
use crate::sandbox::host_funcs::FunctionRegistry;
use crate::sandbox::outb::{HandleOutbError, handle_outb};
use crate::sandbox::snapshot::NextAction;
#[cfg(feature = "mem_profile")]
use crate::sandbox::trace::MemTraceInfo;
#[cfg(crashdump)]
use crate::sandbox::uninitialized::SandboxRuntimeConfig;
fn get_max_log_level_filter(rust_log: String) -> LevelFilter {
let level_str = rust_log
.split(',')
.find_map(|part| {
let mut kv = part.splitn(2, '=');
match (kv.next(), kv.next()) {
(Some(k), Some(v)) if k.trim().contains("hyperlight_guest") => Some(v.trim()),
_ => None,
}
})
.or_else(|| {
rust_log.split(',').find_map(|part| {
let mut kv = part.splitn(2, '=');
match (kv.next(), kv.next()) {
(Some(k), Some(v)) if k.trim().contains("hyperlight_host") => Some(v.trim()),
_ => None,
}
})
})
.or_else(|| {
rust_log.split(',').find_map(|part| {
if part.contains("=") {
None
} else {
Some(part.trim())
}
})
})
.unwrap_or("");
tracing::info!("Determined guest log level: {}", level_str);
LevelFilter::from_str(level_str).unwrap_or(LevelFilter::ERROR)
}
pub(super) fn get_guest_log_filter(guest_max_log_level: Option<LevelFilter>) -> u64 {
let guest_log_level_filter = match guest_max_log_level {
Some(level) => level,
None => get_max_log_level_filter(std::env::var("RUST_LOG").unwrap_or_default()),
};
GuestLogFilter::from(guest_log_level_filter).into()
}
#[derive(Debug, thiserror::Error)]
pub enum DispatchGuestCallError {
#[error("Failed to run vm: {0}")]
Run(#[from] RunVmError),
#[error("Failed to setup registers: {0}")]
SetupRegs(RegisterError),
#[error("VM was uninitialized")]
Uninitialized,
}
impl DispatchGuestCallError {
pub(crate) fn is_poison_error(&self) -> bool {
match self {
DispatchGuestCallError::Run(_) => true,
DispatchGuestCallError::SetupRegs(_) | DispatchGuestCallError::Uninitialized => false,
}
}
pub(crate) fn promote(self) -> (HyperlightError, bool) {
let should_poison = self.is_poison_error();
let promoted_error = match self {
DispatchGuestCallError::Run(RunVmError::ExecutionCancelledByHost) => {
HyperlightError::ExecutionCanceledByHost()
}
DispatchGuestCallError::Run(RunVmError::HandleIo(HandleIoError::Outb(
HandleOutbError::GuestAborted { code, message },
))) => HyperlightError::GuestAborted(code, message),
DispatchGuestCallError::Run(RunVmError::MemoryAccessViolation {
addr,
access_type,
region_flags,
}) => HyperlightError::MemoryAccessViolation(addr, access_type, region_flags),
other => HyperlightVmError::DispatchGuestCall(other).into(),
};
(promoted_error, should_poison)
}
}
#[derive(Debug, thiserror::Error)]
pub enum InitializeError {
#[error("Failed to convert pointer: {0}")]
ConvertPointer(String),
#[error("Failed to run vm: {0}")]
Run(#[from] RunVmError),
#[error("Failed to setup registers: {0}")]
SetupRegs(#[from] RegisterError),
#[error("Guest initialised stack pointer to architecturally invalid value: {0}")]
InvalidStackPointer(u64),
}
#[derive(Debug, thiserror::Error)]
pub enum RunVmError {
#[cfg(crashdump)]
#[error("Crashdump generation error: {0}")]
CrashdumpGeneration(Box<HyperlightError>),
#[cfg(gdb)]
#[error("Debug handler error: {0}")]
DebugHandler(#[from] HandleDebugError),
#[error("Execution was cancelled by the host")]
ExecutionCancelledByHost,
#[error("Failed to access page: {0}")]
PageTableAccess(AccessPageTableError),
#[cfg(feature = "trace_guest")]
#[error("Failed to get registers: {0}")]
GetRegs(RegisterError),
#[error("IO handling error: {0}")]
HandleIo(#[from] HandleIoError),
#[error(
"Memory access violation at address {addr:#x}: {access_type} access, but memory is marked as {region_flags}"
)]
MemoryAccessViolation {
addr: u64,
access_type: MemoryRegionFlags,
region_flags: MemoryRegionFlags,
},
#[error("MMIO READ access to unmapped address {0:#x}")]
MmioReadUnmapped(u64),
#[error("MMIO WRITE access to unmapped address {0:#x}")]
MmioWriteUnmapped(u64),
#[error("vCPU run failed: {0}")]
RunVcpu(#[from] RunVcpuError),
#[error("Unexpected VM exit: {0}")]
UnexpectedVmExit(String),
#[cfg(gdb)]
#[error("vCPU stop reason error: {0}")]
VcpuStopReason(#[from] VcpuStopReasonError),
}
#[derive(Debug, thiserror::Error)]
pub enum HandleIoError {
#[cfg(feature = "mem_profile")]
#[error("Failed to get registers: {0}")]
GetRegs(RegisterError),
#[error("No data was given in IO interrupt")]
NoData,
#[error("{0}")]
Outb(#[from] HandleOutbError),
}
#[derive(Debug, thiserror::Error)]
pub enum MapRegionError {
#[error("VM map memory error: {0}")]
MapMemory(#[from] MapMemoryError),
#[error("Region is not page-aligned (page size: {0:#x})")]
NotPageAligned(usize),
}
#[derive(Debug, thiserror::Error)]
pub enum UnmapRegionError {
#[error("Region not found in mapped regions")]
RegionNotFound,
#[error("VM unmap memory error: {0}")]
UnmapMemory(#[from] UnmapMemoryError),
}
#[derive(Debug, thiserror::Error)]
pub enum UpdateRegionError {
#[error("VM map memory error: {0}")]
MapMemory(#[from] MapMemoryError),
#[error("VM unmap memory error: {0}")]
UnmapMemory(#[from] UnmapMemoryError),
}
#[derive(Debug, thiserror::Error)]
pub enum AccessPageTableError {
#[error("Failed to get/set registers: {0}")]
AccessRegs(#[from] RegisterError),
}
#[cfg(crashdump)]
#[derive(Debug, thiserror::Error)]
pub enum CrashDumpError {
#[error("Failed to generate crashdump because of a register error: {0}")]
GetRegs(#[from] RegisterError),
#[error("Failed to get root PT during crashdump generation: {0}")]
GetRootPt(#[from] AccessPageTableError),
#[error("Failed to get guest memory mapping during crashdump generation: {0}")]
AccessPageTable(Box<HyperlightError>),
}
#[derive(Debug, thiserror::Error)]
pub enum CreateHyperlightVmError {
#[cfg(gdb)]
#[error("Failed to add hardware breakpoint: {0}")]
AddHwBreakpoint(DebugError),
#[error("No hypervisor was found")]
NoHypervisorFound,
#[cfg(gdb)]
#[error("Failed to send debug message: {0}")]
SendDbgMsg(#[from] SendDbgMsgError),
#[error("VM operation error: {0}")]
Vm(#[from] VmError),
#[error("Set scratch error: {0}")]
UpdateRegion(#[from] UpdateRegionError),
}
#[cfg(gdb)]
#[derive(Debug, thiserror::Error)]
pub enum HandleDebugError {
#[error("Debug is not enabled")]
DebugNotEnabled,
#[error("Error processing debug request: {0}")]
ProcessRequest(#[from] ProcessDebugRequestError),
#[error("Failed to receive message from GDB thread: {0}")]
ReceiveMessage(#[from] RecvDbgMsgError),
#[error("Failed to send message to GDB thread: {0}")]
SendMessage(#[from] SendDbgMsgError),
}
#[cfg(gdb)]
#[derive(Debug, thiserror::Error)]
pub enum SendDbgMsgError {
#[error("Debug is not enabled")]
DebugNotEnabled,
#[error("Failed to send message: {0}")]
SendFailed(#[from] GdbTargetError),
}
#[cfg(gdb)]
#[derive(Debug, thiserror::Error)]
pub enum RecvDbgMsgError {
#[error("Debug is not enabled")]
DebugNotEnabled,
#[error("Failed to receive message: {0}")]
RecvFailed(#[from] GdbTargetError),
}
#[derive(Debug, thiserror::Error)]
pub enum HyperlightVmError {
#[error("Create VM error: {0}")]
Create(#[from] CreateHyperlightVmError),
#[error("Dispatch guest call error: {0}")]
DispatchGuestCall(#[from] DispatchGuestCallError),
#[error("Initialize error: {0}")]
Initialize(#[from] InitializeError),
#[error("Map region error: {0}")]
MapRegion(#[from] MapRegionError),
#[error("Restore VM (vcpu) error: {0}")]
Restore(#[from] RegisterError),
#[error("Unmap region error: {0}")]
UnmapRegion(#[from] UnmapRegionError),
#[error("Update region error: {0}")]
UpdateRegion(#[from] UpdateRegionError),
#[error("Access page table error: {0}")]
AccessPageTable(#[from] AccessPageTableError),
}
pub(crate) struct HyperlightVm {
#[cfg(gdb)]
pub(super) vm: Box<dyn DebuggableVm>,
#[cfg(not(gdb))]
pub(super) vm: Box<dyn VirtualMachine>,
pub(super) page_size: usize,
pub(super) entrypoint: NextAction, pub(super) rsp_gva: u64,
pub(super) interrupt_handle: Arc<dyn InterruptHandleImpl>,
pub(super) next_slot: u32, pub(super) freed_slots: Vec<u32>,
pub(super) snapshot_slot: u32,
pub(super) snapshot_memory: Option<SnapshotSharedMemory<GuestSharedMemory>>,
pub(super) scratch_slot: u32, pub(super) scratch_memory: Option<GuestSharedMemory>,
pub(super) mmap_regions: Vec<(u32, MemoryRegion)>,
pub(super) pending_tlb_flush: bool,
#[cfg(gdb)]
pub(super) gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
#[cfg(gdb)]
pub(super) sw_breakpoints: HashMap<u64, u8>, #[cfg(feature = "mem_profile")]
pub(super) trace_info: MemTraceInfo,
#[cfg(crashdump)]
pub(super) rt_cfg: SandboxRuntimeConfig,
}
impl HyperlightVm {
pub(crate) unsafe fn map_region(
&mut self,
region: &MemoryRegion,
) -> std::result::Result<(), MapRegionError> {
if [
region.guest_region.start,
region.guest_region.end,
#[allow(clippy::useless_conversion)]
region.host_region.start.into(),
#[allow(clippy::useless_conversion)]
region.host_region.end.into(),
]
.iter()
.any(|x| x % self.page_size != 0)
{
return Err(MapRegionError::NotPageAligned(self.page_size));
}
let slot = if let Some(freed_slot) = self.freed_slots.pop() {
freed_slot
} else {
let slot = self.next_slot;
self.next_slot += 1;
slot
};
unsafe { self.vm.map_memory((slot, region))? };
self.mmap_regions.push((slot, region.clone()));
Ok(())
}
pub(crate) fn unmap_region(
&mut self,
region: &MemoryRegion,
) -> std::result::Result<(), UnmapRegionError> {
let pos = self
.mmap_regions
.iter()
.position(|(_, r)| r == region)
.ok_or(UnmapRegionError::RegionNotFound)?;
let (slot, _) = self.mmap_regions.remove(pos);
self.freed_slots.push(slot);
self.vm.unmap_memory((slot, region))?;
Ok(())
}
pub(crate) fn get_mapped_regions(&self) -> impl Iterator<Item = &MemoryRegion> {
self.mmap_regions.iter().map(|(_, region)| region)
}
pub(crate) fn update_snapshot_mapping(
&mut self,
snapshot: SnapshotSharedMemory<GuestSharedMemory>,
) -> Result<(), UpdateRegionError> {
let guest_base = crate::mem::layout::SandboxMemoryLayout::BASE_ADDRESS as u64;
let rgn = snapshot.mapping_at(guest_base, MemoryRegionType::Snapshot);
if let Some(old_snapshot) = self.snapshot_memory.replace(snapshot) {
let old_rgn = old_snapshot.mapping_at(guest_base, MemoryRegionType::Snapshot);
self.vm.unmap_memory((self.snapshot_slot, &old_rgn))?;
}
unsafe { self.vm.map_memory((self.snapshot_slot, &rgn))? };
Ok(())
}
pub(crate) fn update_scratch_mapping(
&mut self,
scratch: GuestSharedMemory,
) -> Result<(), UpdateRegionError> {
let guest_base = hyperlight_common::layout::scratch_base_gpa(scratch.mem_size());
let rgn = scratch.mapping_at(guest_base, MemoryRegionType::Scratch);
if let Some(old_scratch) = self.scratch_memory.replace(scratch) {
let old_base = hyperlight_common::layout::scratch_base_gpa(old_scratch.mem_size());
let old_rgn = old_scratch.mapping_at(old_base, MemoryRegionType::Scratch);
self.vm.unmap_memory((self.scratch_slot, &old_rgn))?;
}
unsafe { self.vm.map_memory((self.scratch_slot, &rgn))? };
Ok(())
}
pub(crate) fn get_stack_top(&mut self) -> u64 {
self.rsp_gva
}
pub(crate) fn set_stack_top(&mut self, gva: u64) {
self.rsp_gva = gva;
}
pub(crate) fn get_entrypoint(&self) -> NextAction {
self.entrypoint
}
pub(crate) fn set_entrypoint(&mut self, entrypoint: NextAction) {
self.entrypoint = entrypoint
}
pub(crate) fn interrupt_handle(&self) -> Arc<dyn InterruptHandle> {
self.interrupt_handle.clone()
}
pub(crate) fn clear_cancel(&self) {
self.interrupt_handle.clear_cancel();
}
pub(super) fn run(
&mut self,
mem_mgr: &mut SandboxMemoryManager<HostSharedMemory>,
host_funcs: &Arc<Mutex<FunctionRegistry>>,
#[cfg(gdb)] dbg_mem_access_fn: Arc<Mutex<SandboxMemoryManager<HostSharedMemory>>>,
) -> std::result::Result<(), RunVmError> {
#[cfg(feature = "trace_guest")]
let mut tc = crate::sandbox::trace::TraceContext::new();
let result = loop {
#[cfg(any(kvm, mshv3))]
self.interrupt_handle.set_tid();
self.interrupt_handle.set_running();
let exit_reason = if self.interrupt_handle.is_cancelled()
|| self.interrupt_handle.is_debug_interrupted()
{
Ok(VmExit::Cancelled())
} else {
let result = self.vm.run_vcpu(
#[cfg(feature = "trace_guest")]
&mut tc,
);
#[cfg(feature = "trace_guest")]
{
tc.end_host_trace();
let regs = self.vm.regs().map_err(RunVmError::GetRegs)?;
if tc.has_trace_data(®s) {
let root_pt = self.get_root_pt().map_err(RunVmError::PageTableAccess)?;
tc.handle_trace(®s, mem_mgr, root_pt)
.unwrap_or_else(|e| {
tracing::error!("Cannot handle trace data: {}", e);
});
}
}
result
};
self.interrupt_handle.clear_running();
let cancel_requested = self.interrupt_handle.is_cancelled();
let debug_interrupted = self.interrupt_handle.is_debug_interrupted();
match exit_reason {
#[cfg(gdb)]
Ok(VmExit::Debug { dr6, exception }) => {
let initialise = match self.entrypoint {
NextAction::Initialise(initialise) => initialise,
_ => 0,
};
let stop_reason = crate::hypervisor::gdb::arch::vcpu_stop_reason(
self.vm.as_mut(),
dr6,
initialise,
exception,
)?;
if let Err(e) = self.handle_debug(dbg_mem_access_fn.clone(), stop_reason) {
break Err(e.into());
}
}
Ok(VmExit::Halt()) => {
break Ok(());
}
Ok(VmExit::IoOut(port, data)) => {
self.handle_io(mem_mgr, host_funcs, port, data)?;
}
Ok(VmExit::MmioRead(addr)) => {
let all_regions = self.get_mapped_regions();
match get_memory_access_violation(
addr as usize,
MemoryRegionFlags::READ,
all_regions,
) {
Some(MemoryAccess::AccessViolation(region_flags)) => {
break Err(RunVmError::MemoryAccessViolation {
addr,
access_type: MemoryRegionFlags::READ,
region_flags,
});
}
None => {
break Err(RunVmError::MmioReadUnmapped(addr));
}
}
}
Ok(VmExit::MmioWrite(addr)) => {
let all_regions = self.get_mapped_regions();
match get_memory_access_violation(
addr as usize,
MemoryRegionFlags::WRITE,
all_regions,
) {
Some(MemoryAccess::AccessViolation(region_flags)) => {
break Err(RunVmError::MemoryAccessViolation {
addr,
access_type: MemoryRegionFlags::WRITE,
region_flags,
});
}
None => {
break Err(RunVmError::MmioWriteUnmapped(addr));
}
}
}
Ok(VmExit::Cancelled()) => {
if !cancel_requested && !debug_interrupted {
metrics::counter!(METRIC_ERRONEOUS_VCPU_KICKS).increment(1);
continue;
}
#[cfg(gdb)]
{
self.interrupt_handle.clear_debug_interrupt();
if let Err(e) =
self.handle_debug(dbg_mem_access_fn.clone(), VcpuStopReason::Interrupt)
{
break Err(e.into());
}
}
metrics::counter!(METRIC_GUEST_CANCELLATION).increment(1);
break Err(RunVmError::ExecutionCancelledByHost);
}
Ok(VmExit::Unknown(reason)) => {
break Err(RunVmError::UnexpectedVmExit(reason));
}
Ok(VmExit::Retry()) => continue,
Err(e) => {
break Err(RunVmError::RunVcpu(e));
}
}
};
match result {
Ok(_) => Ok(()),
Err(RunVmError::ExecutionCancelledByHost) => {
Err(RunVmError::ExecutionCancelledByHost)
}
Err(e) => {
#[cfg(crashdump)]
if self.rt_cfg.guest_core_dump {
crate::hypervisor::crashdump::generate_crashdump(self, mem_mgr, None)
.map_err(|e| RunVmError::CrashdumpGeneration(Box::new(e)))?;
}
#[cfg(gdb)]
if self.gdb_conn.is_some() {
self.handle_debug(dbg_mem_access_fn.clone(), VcpuStopReason::Crash)?
}
Err(e)
}
}
}
fn handle_io(
&mut self,
mem_mgr: &mut SandboxMemoryManager<HostSharedMemory>,
host_funcs: &Arc<Mutex<FunctionRegistry>>,
port: u16,
data: Vec<u8>,
) -> std::result::Result<(), HandleIoError> {
if data.is_empty() {
return Err(HandleIoError::NoData);
}
#[allow(clippy::get_first)]
let val = u32::from_le_bytes([
data.get(0).copied().unwrap_or(0),
data.get(1).copied().unwrap_or(0),
data.get(2).copied().unwrap_or(0),
data.get(3).copied().unwrap_or(0),
]);
#[cfg(feature = "mem_profile")]
{
let regs = self.vm.regs().map_err(HandleIoError::GetRegs)?;
handle_outb(mem_mgr, host_funcs, port, val, ®s, &mut self.trace_info)?;
}
#[cfg(not(feature = "mem_profile"))]
{
handle_outb(mem_mgr, host_funcs, port, val)?;
}
Ok(())
}
}
impl Drop for HyperlightVm {
fn drop(&mut self) {
self.interrupt_handle.set_dropped();
}
}
enum MemoryAccess {
AccessViolation(MemoryRegionFlags),
}
fn get_memory_access_violation<'a>(
gpa: usize,
tried: MemoryRegionFlags,
mut mem_regions: impl Iterator<Item = &'a MemoryRegion>,
) -> Option<MemoryAccess> {
let region = mem_regions.find(|region| region.guest_region.contains(&gpa))?;
if !region.flags.contains(tried) {
return Some(MemoryAccess::AccessViolation(region.flags));
}
None
}