use flatbuffers::FlatBufferBuilder;
use hyperlight_common::flatbuffer_wrappers::function_call::FunctionCall;
use hyperlight_common::flatbuffer_wrappers::function_types::FunctionCallResult;
use hyperlight_common::flatbuffer_wrappers::host_function_details::HostFunctionDetails;
use hyperlight_common::flatbuffer_wrappers::util::estimate_flatbuffer_capacity;
use hyperlight_common::log_level::GuestLogFilter;
use hyperlight_common::transport::{Buf, EncodedMessage, ExternalValues, MailboxValue, MsgKind};
use hyperlight_common::virtq::ReplyChain;
use hyperlight_common::vmem::{self, PAGE_TABLE_SIZE};
#[cfg(crashdump)]
use hyperlight_common::vmem::{BasicMapping, MappingKind};
use tracing::{Span, instrument};
use super::layout::SandboxMemoryLayout;
use super::shared_mem::{
ExclusiveSharedMemory, GuestSharedMemory, HostSharedMemory, ReadonlySharedMemory, SharedMemory,
};
use super::virtq::{self, G2hConsumer, H2gConsumer, VirtqSnapshot};
use crate::hypervisor::regs::CommonSpecialRegisters;
use crate::mem::memory_region::MemoryRegion;
#[cfg(crashdump)]
use crate::mem::memory_region::{CrashDumpRegion, MemoryRegionFlags, MemoryRegionType};
use crate::sandbox::snapshot::{NextAction, Snapshot};
use crate::{HyperlightError, Result, new_error};
#[cfg(crashdump)]
fn mapping_kind_to_flags(kind: &MappingKind) -> (MemoryRegionFlags, MemoryRegionType) {
match kind {
MappingKind::Basic(BasicMapping {
readable,
writable,
executable,
}) => {
let mut flags = MemoryRegionFlags::empty();
if *readable {
flags |= MemoryRegionFlags::READ;
}
if *writable {
flags |= MemoryRegionFlags::WRITE;
}
if *executable {
flags |= MemoryRegionFlags::EXECUTE;
}
(flags, MemoryRegionType::Snapshot)
}
MappingKind::Cow(cow) => {
let mut flags = MemoryRegionFlags::empty();
if cow.readable {
flags |= MemoryRegionFlags::READ;
}
if cow.executable {
flags |= MemoryRegionFlags::EXECUTE;
}
(flags, MemoryRegionType::Scratch)
}
MappingKind::Unmapped => (MemoryRegionFlags::empty(), MemoryRegionType::Snapshot),
MappingKind::ZeroInit(bm) => {
let mut flags = MemoryRegionFlags::empty();
if bm.readable {
flags |= MemoryRegionFlags::READ;
}
if bm.writable {
flags |= MemoryRegionFlags::WRITE;
}
if bm.executable {
flags |= MemoryRegionFlags::EXECUTE;
}
(flags, MemoryRegionType::Snapshot)
}
}
}
#[cfg(crashdump)]
fn try_coalesce_region(
regions: &mut [CrashDumpRegion],
virt_base: usize,
virt_end: usize,
host_base: usize,
flags: MemoryRegionFlags,
) -> bool {
if let Some(last) = regions.last_mut()
&& last.guest_region.end == virt_base
&& last.host_region.end == host_base
&& last.flags == flags
{
last.guest_region.end = virt_end;
last.host_region.end = host_base + (virt_end - virt_base);
return true;
}
false
}
mod unused_hack {
#[cfg(not(unshared_snapshot_mem))]
use crate::mem::shared_mem::ReadonlySharedMemory;
use crate::mem::shared_mem::SharedMemory;
pub trait SnapshotSharedMemoryT {
type T<S: SharedMemory>;
}
pub struct SnapshotSharedMemory_;
impl SnapshotSharedMemoryT for SnapshotSharedMemory_ {
#[cfg(not(unshared_snapshot_mem))]
type T<S: SharedMemory> = ReadonlySharedMemory;
#[cfg(unshared_snapshot_mem)]
type T<S: SharedMemory> = S;
}
pub type SnapshotSharedMemory<S> = <SnapshotSharedMemory_ as SnapshotSharedMemoryT>::T<S>;
}
impl ReadonlySharedMemory {
pub(crate) fn to_mgr_snapshot_mem(
&self,
) -> Result<SnapshotSharedMemory<ExclusiveSharedMemory>> {
#[cfg(not(unshared_snapshot_mem))]
let ret = self.clone();
#[cfg(unshared_snapshot_mem)]
let ret = self.copy_to_writable()?;
Ok(ret)
}
}
pub(crate) use unused_hack::SnapshotSharedMemory;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum ExecutionMode {
Unrestricted,
Restricted,
}
pub(crate) struct SandboxMemoryManager<S: SharedMemory> {
pub(crate) shared_mem: SnapshotSharedMemory<S>,
pub(crate) scratch_mem: S,
pub(crate) layout: SandboxMemoryLayout,
pub(crate) next_action: NextAction,
pub(crate) original_entrypoint: u64,
pub(crate) abort_buffer: Vec<u8>,
pub(crate) execution_mode: ExecutionMode,
pub(crate) snapshot_count: u64,
pub(crate) g2h_consumer: Option<G2hConsumer>,
pub(crate) h2g_consumer: Option<H2gConsumer>,
next_guest_cid: u32,
}
impl<S: Clone + SharedMemory> Clone for SandboxMemoryManager<S> {
fn clone(&self) -> Self {
Self {
shared_mem: self.shared_mem.clone(),
scratch_mem: self.scratch_mem.clone(),
layout: self.layout,
next_action: self.next_action,
original_entrypoint: self.original_entrypoint,
abort_buffer: self.abort_buffer.clone(),
execution_mode: self.execution_mode,
snapshot_count: self.snapshot_count,
g2h_consumer: None,
h2g_consumer: None,
next_guest_cid: self.next_guest_cid,
}
}
}
pub(crate) struct GuestPageTableBuffer {
buffer: std::cell::RefCell<Vec<u8>>,
phys_base: usize,
root: std::cell::Cell<u64>,
}
impl vmem::TableReadOps for GuestPageTableBuffer {
type TableAddr = u64;
fn entry_addr(addr: u64, offset: u64) -> u64 {
addr + offset
}
unsafe fn read_entry(&self, addr: u64) -> vmem::PageTableEntry {
let buffer = self.buffer.borrow();
let byte_offset = addr as usize - self.phys_base;
let pte_size = core::mem::size_of::<vmem::PageTableEntry>();
let Some(bytes) = buffer.get(byte_offset..byte_offset + pte_size) else {
return 0;
};
let mut buf = [0u8; 8];
buf[..pte_size].copy_from_slice(bytes);
vmem::PageTableEntry::from_le_bytes(buf[..pte_size].try_into().unwrap_or_default())
}
fn to_phys(addr: u64) -> vmem::PhysAddr {
addr as vmem::PhysAddr
}
fn from_phys(addr: vmem::PhysAddr) -> u64 {
#[allow(clippy::unnecessary_cast)]
{
addr as u64
}
}
fn root_table(&self) -> u64 {
self.root.get()
}
}
impl vmem::TableOps for GuestPageTableBuffer {
type TableMovability = vmem::MayNotMoveTable;
unsafe fn alloc_table(&self) -> u64 {
let mut b = self.buffer.borrow_mut();
let offset = b.len();
b.resize(offset + PAGE_TABLE_SIZE, 0);
(self.phys_base + offset) as u64
}
unsafe fn write_entry(&self, addr: u64, entry: vmem::PageTableEntry) -> Option<vmem::Void> {
let mut b = self.buffer.borrow_mut();
let byte_offset = addr as usize - self.phys_base;
let pte_size = core::mem::size_of::<vmem::PageTableEntry>();
if let Some(slice) = b.get_mut(byte_offset..byte_offset + pte_size) {
slice.copy_from_slice(&entry.to_le_bytes()[..pte_size]);
}
None
}
unsafe fn update_root(&self, impossible: vmem::Void) {
match impossible {}
}
}
impl core::convert::AsRef<GuestPageTableBuffer> for GuestPageTableBuffer {
fn as_ref(&self) -> &Self {
self
}
}
impl GuestPageTableBuffer {
pub(crate) fn new(phys_base: usize) -> Self {
GuestPageTableBuffer {
buffer: std::cell::RefCell::new(vec![0u8; PAGE_TABLE_SIZE]),
phys_base,
root: std::cell::Cell::new(phys_base as u64),
}
}
pub(crate) fn set_root(&self, addr: u64) {
self.root.set(addr);
}
pub(crate) fn initial_root(&self) -> u64 {
self.phys_base as u64
}
#[cfg(test)]
#[allow(dead_code)]
pub(crate) fn size(&self) -> usize {
self.buffer.borrow().len()
}
pub(crate) fn into_bytes(self) -> Box<[u8]> {
self.buffer.into_inner().into_boxed_slice()
}
}
impl<S> SandboxMemoryManager<S>
where
S: SharedMemory,
{
#[instrument(skip_all, parent = Span::current(), level= "Trace")]
pub(crate) fn new(
layout: SandboxMemoryLayout,
shared_mem: SnapshotSharedMemory<S>,
scratch_mem: S,
next_action: NextAction,
) -> Self {
Self {
layout,
shared_mem,
scratch_mem,
next_action,
original_entrypoint: 0,
abort_buffer: Vec::new(),
execution_mode: ExecutionMode::Unrestricted,
snapshot_count: 0,
g2h_consumer: None,
h2g_consumer: None,
next_guest_cid: 1,
}
}
pub(crate) fn get_abort_buffer_mut(&mut self) -> &mut Vec<u8> {
&mut self.abort_buffer
}
}
impl SandboxMemoryManager<ExclusiveSharedMemory> {
pub(crate) fn from_snapshot(s: &Snapshot) -> Result<Self> {
let layout = *s.layout();
let shared_mem = s.memory().to_mgr_snapshot_mem()?;
let scratch_mem = ExclusiveSharedMemory::new(s.layout().get_scratch_size())?;
let next_action = s.next_action();
let mut mgr = Self::new(layout, shared_mem, scratch_mem, next_action);
mgr.original_entrypoint = s.original_entrypoint();
mgr.snapshot_count = s.snapshot_generation();
Ok(mgr)
}
pub fn build(
self,
) -> Result<(
SandboxMemoryManager<HostSharedMemory>,
SandboxMemoryManager<GuestSharedMemory>,
)> {
let (hshm, gshm) = self.shared_mem.build();
let (hscratch, gscratch) = self.scratch_mem.build();
let mut host_mgr = SandboxMemoryManager {
shared_mem: hshm,
scratch_mem: hscratch,
layout: self.layout,
next_action: self.next_action,
original_entrypoint: self.original_entrypoint,
abort_buffer: self.abort_buffer,
execution_mode: self.execution_mode,
snapshot_count: self.snapshot_count,
g2h_consumer: None,
h2g_consumer: None,
next_guest_cid: self.next_guest_cid,
};
let guest_mgr = SandboxMemoryManager {
shared_mem: gshm,
scratch_mem: gscratch,
layout: self.layout,
next_action: self.next_action,
original_entrypoint: self.original_entrypoint,
abort_buffer: Vec::new(), execution_mode: self.execution_mode,
snapshot_count: self.snapshot_count,
g2h_consumer: None,
h2g_consumer: None,
next_guest_cid: self.next_guest_cid,
};
host_mgr.update_scratch_bookkeeping()?;
if matches!(host_mgr.next_action, NextAction::Initialise(_)) {
let cp = MailboxValue::CheckpointPending.as_bytes();
let mbx = host_mgr.layout.get_transport_arena().mbx_offset();
host_mgr.scratch_mem.copy_from_slice(cp, mbx)?;
host_mgr.create_virtq_consumers()?;
}
Ok((host_mgr, guest_mgr))
}
}
impl SandboxMemoryManager<HostSharedMemory> {
#[allow(clippy::too_many_arguments)]
pub(crate) fn snapshot(
&mut self,
mapped_regions: Vec<MemoryRegion>,
root_pt_gpas: &[u64],
rsp_gva: u64,
sregs: CommonSpecialRegisters,
#[cfg(target_arch = "x86_64")] msrs: Vec<crate::hypervisor::regs::MsrEntry>,
next_action: NextAction,
host_functions: HostFunctionDetails,
) -> Result<Snapshot> {
let virtq = match (&self.g2h_consumer, &self.h2g_consumer) {
(Some(_), Some(_)) => Some(VirtqSnapshot::capture(&self.layout, &self.scratch_mem)?),
(None, None) => None,
_ => return Err(new_error!("virtqueue consumer ownership is incomplete")),
};
let snapshot = Snapshot::new(
&mut self.shared_mem,
&mut self.scratch_mem,
self.layout,
crate::mem::exe::LoadInfo::dummy(),
mapped_regions,
root_pt_gpas,
rsp_gva,
sregs,
#[cfg(target_arch = "x86_64")]
msrs,
next_action,
self.original_entrypoint,
self.snapshot_count + 1,
host_functions,
virtq,
)?;
self.snapshot_count = snapshot.snapshot_generation();
Ok(snapshot)
}
fn create_virtq_consumers(&mut self) -> Result<()> {
if self.g2h_consumer.is_some() || self.h2g_consumer.is_some() {
return Err(new_error!("virtqueue consumers already exist"));
}
let (g2h, h2g) = virtq::create_consumers(&self.layout, &self.scratch_mem)?;
self.g2h_consumer = Some(g2h);
self.h2g_consumer = Some(h2g);
Ok(())
}
pub(crate) fn restore_virtq(&mut self, snapshot: &VirtqSnapshot) -> Result<()> {
if self.g2h_consumer.is_some() || self.h2g_consumer.is_some() {
return Err(new_error!("virtqueue consumers are already attached"));
}
let (g2h, h2g) = snapshot.restore(&self.layout, &self.scratch_mem)?;
self.g2h_consumer = Some(g2h);
self.h2g_consumer = Some(h2g);
Ok(())
}
#[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
pub(crate) fn write_guest_function_call(&mut self, call: &FunctionCall) -> Result<u32> {
let cid = self.next_guest_cid;
let params = call.parameters.as_deref().unwrap_or_default();
let cap = estimate_flatbuffer_capacity(&call.function_name, params);
let mut builder = FlatBufferBuilder::with_capacity(cap);
let mut externals = ExternalValues::new();
let control = call.encode(&mut builder, &mut externals)?;
let Some(msg) = EncodedMessage::new(MsgKind::Request, cid, control, externals) else {
return Err(new_error!("H2G request exceeds the wire payload limit"));
};
self.write_h2g_message(&msg)?;
self.next_guest_cid = cid.wrapping_add(1);
if self.next_guest_cid == 0 {
self.next_guest_cid = 1;
}
Ok(cid)
}
fn write_h2g_message(&mut self, message: &EncodedMessage<'_>) -> Result<()> {
let consumer = self.h2g_consumer.as_mut().ok_or_else(|| {
HyperlightError::TransportError("H2G consumer is not attached".into())
})?;
let buffer_size = self.layout.get_h2g_buffer_size();
let buffer_count = message.total_len().div_ceil(buffer_size);
let control_reserve = usize::from(message.external_len() != 0);
let maybe_buffers = consumer
.poll_batch(buffer_count, control_reserve, 0)
.map_err(|error| {
HyperlightError::TransportError(format!("H2G poll failed: {error}"))
})?;
let Some(buffers) = maybe_buffers else {
return Err(new_error!(
"H2G capacity cannot provide {buffer_count} buffers with {control_reserve} spare"
));
};
let mut message = message.as_buf();
for (recv, reply) in buffers {
let ReplyChain::Writable(mut buffer) = reply else {
return Err(HyperlightError::TransportError(
"H2G receive buffer is not writable".into(),
));
};
if buffer.desc_count() != 1 || buffer.capacity() != buffer_size {
return Err(HyperlightError::TransportError(
"H2G receive buffer has an invalid shape".into(),
));
}
while message.has_remaining() && buffer.remaining() != 0 {
let written = buffer.write(message.chunk()).map_err(|err| {
HyperlightError::TransportError(format!("H2G write failed: {err}"))
})?;
message.advance(written);
}
consumer.complete(recv, buffer).map_err(|err| {
HyperlightError::TransportError(format!("H2G completion failed: {err}"))
})?;
}
debug_assert!(!message.has_remaining());
Ok(())
}
#[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
pub(crate) fn read_h2g_result_from_g2h(&mut self, cid: u32) -> Result<FunctionCallResult> {
let max_recv_len = self.layout.get_g2h_queue_dims().pool_len();
let Some(consumer) = self.g2h_consumer.as_mut() else {
return Err(HyperlightError::TransportError(
"G2H consumer is not attached".into(),
));
};
loop {
let maybe_next = consumer.poll(max_recv_len).map_err(|err| {
HyperlightError::TransportError(format!("G2H poll failed: {err}"))
})?;
let Some((mut recv, reply)) = maybe_next else {
return Err(HyperlightError::TransportError(
"G2H has no guest function result after halt".into(),
));
};
let header = virtq::read_message_header(&mut recv).map_err(|err| {
HyperlightError::TransportError(format!("Failed to read G2H result header: {err}"))
})?;
if !matches!(&reply, ReplyChain::Ack(_)) {
return Err(HyperlightError::TransportError(
"G2H result entry has writable buffers".into(),
));
}
match header.kind {
MsgKind::Log => {
if header.cid != 0 {
return Err(HyperlightError::TransportError(
"G2H log has a correlation ID".into(),
));
}
let log = virtq::read_guest_log_data(&mut recv).map_err(|err| {
HyperlightError::TransportError(format!("Failed to read G2H log: {err}"))
})?;
consumer.complete(recv, reply).map_err(|err| {
HyperlightError::TransportError(format!(
"Failed to complete G2H log: {err}"
))
})?;
crate::sandbox::outb::emit_guest_log(&log);
}
MsgKind::Response => {
if header.cid != cid {
return Err(HyperlightError::TransportError(
"G2H guest function result correlation ID mismatch".into(),
));
}
let result = virtq::read_guest_function_call_result(&mut recv);
consumer.complete(recv, reply).map_err(|err| {
HyperlightError::TransportError(format!(
"Failed to complete G2H guest function result: {err}"
))
})?;
return result.map_err(|err| {
HyperlightError::TransportError(format!(
"Failed to decode G2H guest function result: {err}"
))
});
}
kind => {
return Err(HyperlightError::TransportError(format!(
"Expected G2H guest function result, got {kind:?}"
)));
}
}
}
}
pub(crate) fn begin_snapshot_checkpoint(&mut self) -> Result<()> {
let offset = self.layout.get_transport_arena().mbx_offset();
let cp = MailboxValue::CheckpointPending.as_bytes();
self.scratch_mem.copy_from_slice(cp, offset)?;
let message = EncodedMessage::new_snapshot_cp();
self.write_h2g_message(&message)
}
pub(crate) fn finish_snapshot_checkpoint(&mut self) -> Result<()> {
let Some(g2h) = self.g2h_consumer.as_mut() else {
return Err(new_error!("G2H consumer is not attached"));
};
let Some(h2g) = self.h2g_consumer.as_mut() else {
return Err(new_error!("H2G consumer is not attached"));
};
g2h.reset()?;
h2g.reset()?;
let status = self
.scratch_mem
.read::<u64>(self.layout.get_transport_arena().mbx_offset())
.map_err(|error| HyperlightError::TransportError(error.to_string()))?;
match MailboxValue::try_from(status) {
Ok(MailboxValue::CheckpointComplete) => Ok(()),
Ok(MailboxValue::CheckpointPending) => Err(HyperlightError::TransportError(
"Guest did not publish snapshot checkpoint status".into(),
)),
Err(_) => Err(HyperlightError::TransportError(format!(
"Guest published invalid snapshot checkpoint status {status:#x}"
))),
}
}
pub(crate) fn restore_snapshot(
&mut self,
snapshot: &Snapshot,
) -> Result<(
Option<SnapshotSharedMemory<GuestSharedMemory>>,
Option<GuestSharedMemory>,
)> {
let virtq = snapshot.virtq();
if virtq.is_none() && matches!(snapshot.next_action(), NextAction::Call(_)) {
return Err(new_error!(
"running snapshot has no canonical transport state"
));
}
self.g2h_consumer = None;
self.h2g_consumer = None;
self.execution_mode = ExecutionMode::Unrestricted;
self.execution_mode = ExecutionMode::Unrestricted;
let gsnapshot = if *snapshot.memory() == self.shared_mem {
None
} else {
let new_snapshot_mem = snapshot.memory().to_mgr_snapshot_mem()?;
let (hsnapshot, gsnapshot) = new_snapshot_mem.build();
self.shared_mem = hsnapshot;
Some(gsnapshot)
};
let new_scratch_size = snapshot.layout().get_scratch_size();
let gscratch = if new_scratch_size == self.scratch_mem.mem_size() {
self.scratch_mem.zero_or_replace()?
} else {
let new_scratch_mem = ExclusiveSharedMemory::new(new_scratch_size)?;
let (hscratch, gscratch) = new_scratch_mem.build();
self.scratch_mem = hscratch;
Some(gscratch)
};
self.layout = *snapshot.layout();
self.snapshot_count = snapshot.snapshot_generation();
self.original_entrypoint = snapshot.original_entrypoint();
self.update_scratch_bookkeeping()?;
if let Some(virtq) = virtq {
self.restore_virtq(virtq)?;
} else if matches!(snapshot.next_action(), NextAction::Initialise(_)) {
self.scratch_mem.copy_from_slice(
MailboxValue::CheckpointPending.as_bytes(),
self.layout.get_transport_arena().mbx_offset(),
)?;
self.create_virtq_consumers()?;
}
Ok((gsnapshot, gscratch))
}
#[inline]
fn update_scratch_bookkeeping_item(&mut self, offset: u64, value: u64) -> Result<()> {
let scratch_size = self.scratch_mem.mem_size();
let base_offset = scratch_size - offset as usize;
self.scratch_mem
.write::<u64>(base_offset, value)
.map_err(From::from)
}
pub(crate) fn request_libc_rng_reseed(&mut self, seed: u32) -> Result<()> {
self.update_scratch_bookkeeping_item(
hyperlight_common::layout::SCRATCH_TOP_LIBC_RNG_SEED_OFFSET,
(1_u64 << 32) | u64::from(seed),
)
}
pub(crate) fn request_guest_log_level_update(
&mut self,
log_level: tracing_core::LevelFilter,
) -> Result<()> {
self.update_scratch_bookkeeping_item(
hyperlight_common::layout::SCRATCH_TOP_GUEST_LOG_LEVEL_OFFSET,
(hyperlight_common::log_level::GUEST_LOG_FILTER_UPDATE_PENDING)
| u64::from(GuestLogFilter::from(log_level)),
)
}
fn update_scratch_bookkeeping(&mut self) -> Result<()> {
use hyperlight_common::layout::*;
let scratch_size = self.scratch_mem.mem_size();
self.update_scratch_bookkeeping_item(SCRATCH_TOP_SIZE_OFFSET, scratch_size as u64)?;
self.update_scratch_bookkeeping_item(
SCRATCH_TOP_ALLOCATOR_OFFSET,
self.layout.get_first_free_scratch_gpa(),
)?;
self.update_scratch_bookkeeping_item(
SCRATCH_TOP_SNAPSHOT_PT_GPA_BASE_OFFSET,
self.layout.get_pt_base_gpa(),
)?;
self.update_scratch_bookkeeping_item(
SCRATCH_TOP_SNAPSHOT_GENERATION_OFFSET,
self.snapshot_count,
)?;
self.update_scratch_bookkeeping_item(
SCRATCH_TOP_G2H_QUEUE_SIZE_OFFSET,
u64::try_from(self.layout.get_g2h_queue_size())?,
)?;
self.update_scratch_bookkeeping_item(
SCRATCH_TOP_G2H_POOL_PAGES_OFFSET,
u64::try_from(self.layout.get_g2h_pool_pages())?,
)?;
self.update_scratch_bookkeeping_item(
SCRATCH_TOP_G2H_BUFFER_SIZE_OFFSET,
u64::try_from(self.layout.get_g2h_buffer_size())?,
)?;
self.update_scratch_bookkeeping_item(
SCRATCH_TOP_H2G_QUEUE_SIZE_OFFSET,
u64::try_from(self.layout.get_h2g_queue_size())?,
)?;
self.update_scratch_bookkeeping_item(
SCRATCH_TOP_H2G_POOL_PAGES_OFFSET,
u64::try_from(self.layout.get_h2g_pool_pages())?,
)?;
self.update_scratch_bookkeeping_item(
SCRATCH_TOP_H2G_BUFFER_SIZE_OFFSET,
u64::try_from(self.layout.get_h2g_buffer_size())?,
)?;
let transport_arena = self.layout.get_transport_arena();
self.update_scratch_bookkeeping_item(
SCRATCH_TOP_TRANSPORT_ARENA_GPA_OFFSET,
transport_arena.base_addr(),
)?;
let snapshot_pt_end = self.shared_mem.mem_size();
let snapshot_pt_size = self.layout.get_pt_size();
let snapshot_pt_start =
snapshot_pt_end - snapshot_pt_size.next_multiple_of(page_size::get());
self.scratch_mem.with_exclusivity(|scratch| {
#[cfg(not(unshared_snapshot_mem))]
let bytes = &self.shared_mem.as_slice()[snapshot_pt_start..snapshot_pt_end];
#[cfg(unshared_snapshot_mem)]
let bytes = {
let mut bytes = vec![0u8; snapshot_pt_size];
self.shared_mem
.copy_to_slice(&mut bytes, snapshot_pt_start)?;
bytes
};
#[allow(clippy::needless_borrow)]
scratch.copy_from_slice(&bytes, self.layout.get_pt_base_scratch_offset())
})??;
Ok(())
}
#[cfg(crashdump)]
pub(crate) fn get_guest_memory_regions(
&mut self,
root_pt: u64,
mmap_regions: &[MemoryRegion],
) -> Result<Vec<CrashDumpRegion>> {
use crate::sandbox::snapshot::SharedMemoryPageTableBuffer;
let len = hyperlight_common::layout::SCRATCH_TOP_GVA;
let regions = self.shared_mem.with_contents(|snapshot| {
self.scratch_mem.with_contents(|scratch| {
let pt_buf =
SharedMemoryPageTableBuffer::new(snapshot, scratch, self.layout, root_pt);
let mappings: Vec<_> =
unsafe { hyperlight_common::vmem::virt_to_phys(&pt_buf, 0, len as u64) }
.collect();
if mappings.is_empty() {
return Err(new_error!("No page table mappings found (len {len})",));
}
let mut regions: Vec<CrashDumpRegion> = Vec::new();
for mapping in &mappings {
let virt_base = mapping.virt_base as usize;
let virt_end = (mapping.virt_base + mapping.len) as usize;
if let Some(resolved) = self.layout.resolve_gpa(mapping.phys_base, mmap_regions)
{
let (flags, region_type) = mapping_kind_to_flags(&mapping.kind);
let resolved = resolved.with_memories(snapshot, scratch);
let contents = resolved.as_ref();
let host_base = contents.as_ptr() as usize;
let host_len = (mapping.len as usize).min(contents.len());
if try_coalesce_region(&mut regions, virt_base, virt_end, host_base, flags)
{
continue;
}
regions.push(CrashDumpRegion {
guest_region: virt_base..virt_end,
host_region: host_base..host_base + host_len,
flags,
region_type,
});
}
}
Ok(regions)
})
})???;
Ok(regions)
}
#[cfg(feature = "trace_guest")]
pub(crate) fn read_guest_memory_by_gva(
&mut self,
gva: u64,
len: usize,
root_pt: u64,
) -> Result<Vec<u8>> {
use hyperlight_common::vmem::PAGE_SIZE;
use crate::sandbox::snapshot::{SharedMemoryPageTableBuffer, access_gpa};
self.shared_mem.with_contents(|snap| {
self.scratch_mem.with_contents(|scratch| {
let pt_buf = SharedMemoryPageTableBuffer::new(snap, scratch, self.layout, root_pt);
let mappings: Vec<_> = unsafe {
hyperlight_common::vmem::virt_to_phys(&pt_buf, gva, len as u64)
}
.collect();
if mappings.is_empty() {
return Err(new_error!(
"No page table mappings found for GVA {:#x} (len {})",
gva,
len,
));
}
let mut result = Vec::with_capacity(len);
let mut current_gva = gva;
for mapping in &mappings {
if mapping.virt_base > current_gva {
return Err(new_error!(
"Page table walker returned mapping with virt_base {:#x} > current read position {:#x}",
mapping.virt_base,
current_gva,
));
}
let page_offset = (current_gva - mapping.virt_base) as usize;
let bytes_remaining = len - result.len();
let available_in_page = PAGE_SIZE - page_offset;
let bytes_to_copy = bytes_remaining.min(available_in_page);
let gpa = mapping.phys_base + page_offset as u64;
let (mem, offset) = access_gpa(snap, scratch, self.layout, gpa)
.ok_or_else(|| {
new_error!(
"Failed to resolve GPA {:#x} to host memory (GVA {:#x})",
gpa,
gva
)
})?;
let slice = mem
.get(offset..offset + bytes_to_copy)
.ok_or_else(|| {
new_error!(
"GPA {:#x} resolved to out-of-bounds host offset {} (need {} bytes)",
gpa,
offset,
bytes_to_copy
)
})?;
result.extend_from_slice(slice);
current_gva += bytes_to_copy as u64;
}
if result.len() != len {
tracing::error!(
"Page table walker returned mappings that don't cover the full requested length: got {}, expected {}",
result.len(),
len,
);
return Err(new_error!(
"Could not read full GVA range: got {} of {} bytes {:?}",
result.len(),
len,
mappings
));
}
Ok(result)
})
})??
}
}
#[cfg(test)]
mod tests {
use hyperlight_common::flatbuffer_wrappers::function_call::FunctionCallType;
use hyperlight_common::flatbuffer_wrappers::function_types::{ParameterValue, ReturnType};
use hyperlight_common::transport::{
MsgHeader, SIZE_PREFIX_LEN, size_prefix_payload_len, size_prefixed_len,
};
use hyperlight_common::virtq::DescFlags;
#[cfg(target_arch = "x86_64")]
use hyperlight_testing::sandbox_sizes::{LARGE_HEAP_SIZE, MEDIUM_HEAP_SIZE, SMALL_HEAP_SIZE};
#[cfg(target_arch = "x86_64")]
use hyperlight_testing::simple_guest_as_pathbuf;
use super::*;
#[cfg(target_arch = "x86_64")]
use crate::GuestBinary;
use crate::mem::virtq::tests::{H2G_BUFFER_SIZE, SCRATCH_SIZE, TestCase, memory_layout};
#[cfg(target_arch = "x86_64")]
use crate::sandbox::SandboxConfiguration;
fn manager(case: &TestCase) -> SandboxMemoryManager<HostSharedMemory> {
let host_page_size = page_size::get();
#[cfg(not(unshared_snapshot_mem))]
let shared_mem =
ReadonlySharedMemory::from_bytes(&vec![0; host_page_size], host_page_size).unwrap();
#[cfg(unshared_snapshot_mem)]
let shared_mem = ExclusiveSharedMemory::new(host_page_size)
.unwrap()
.build()
.0;
let mut mgr = SandboxMemoryManager::new(
memory_layout(),
shared_mem,
case.scratch.clone(),
NextAction::None,
);
mgr.h2g_consumer = Some(case.h2g_consumer());
mgr
}
fn h2g_call(bytes: usize) -> FunctionCall {
let params = (bytes != 0).then(|| vec![ParameterValue::VecBytes(vec![0xa5; bytes])]);
FunctionCall::new(
"call".to_string(),
params,
FunctionCallType::Guest,
ReturnType::Void,
)
}
#[test]
fn rejects_invalid_h2g_descriptors() {
for (len, expected) in [
(H2G_BUFFER_SIZE as u32, "Payload data too large"),
(0, "not writable"),
] {
let case = TestCase::new();
let mut mgr = manager(&case);
let mut desc = case.h2g_desc(0);
desc.flags &= !DescFlags::WRITE.bits();
desc.len = len;
case.set_h2g_desc(0, desc);
let error = mgr.write_guest_function_call(&h2g_call(0)).unwrap_err();
assert!(error.to_string().contains(expected), "{error:#}");
assert!(error.is_poison_error());
assert!(matches!(error, HyperlightError::TransportError(_)));
}
}
#[test]
fn partial_h2g_write_is_fatal() {
let case = TestCase::new();
let mut mgr = manager(&case);
let mut desc = case.h2g_desc(1);
desc.addr = hyperlight_common::layout::scratch_base_gva(SCRATCH_SIZE) + SCRATCH_SIZE as u64;
case.set_h2g_desc(1, desc);
let error = mgr
.write_guest_function_call(&h2g_call(H2G_BUFFER_SIZE + 1024))
.unwrap_err();
assert!(error.to_string().contains("Memory write"), "{error:#}");
assert!(error.is_poison_error());
assert!(matches!(error, HyperlightError::TransportError(_)));
assert_eq!(mgr.h2g_consumer.as_ref().unwrap().used_cursor().head(), 1);
}
#[test]
fn insufficient_h2g_capacity_rolls_back() {
let case = TestCase::new();
let mut mgr = manager(&case);
let cursor = mgr.h2g_consumer.as_ref().unwrap().avail_cursor();
let error = mgr
.write_guest_function_call(&h2g_call(H2G_BUFFER_SIZE * 4))
.unwrap_err();
assert!(error.to_string().contains("H2G capacity"), "{error:#}");
assert!(!error.is_poison_error());
assert_eq!(mgr.h2g_consumer.as_ref().unwrap().avail_cursor(), cursor);
assert_eq!(mgr.write_guest_function_call(&h2g_call(0)).unwrap(), 1);
}
#[test]
fn missing_g2h_result_is_fatal() {
let case = TestCase::new();
let mut mgr = manager(&case);
mgr.g2h_consumer = Some(case.g2h_consumer());
let Err(error) = mgr.read_h2g_result_from_g2h(1) else {
panic!("expected missing G2H result");
};
assert!(
error
.to_string()
.contains("G2H has no guest function result")
);
assert!(error.is_poison_error());
assert!(matches!(error, HyperlightError::TransportError(_)));
}
#[test]
fn writes_dense_h2g_request_and_reserves_control_buffer() {
let case = TestCase::new();
let mut mgr = manager(&case);
let external_len = H2G_BUFFER_SIZE * 2;
let buffers: Vec<_> = (0..4).map(|index| case.h2g_desc(index).addr).collect();
let cid = mgr
.write_guest_function_call(&h2g_call(external_len))
.unwrap();
let used = mgr.h2g_consumer.as_ref().unwrap().avail_cursor().head();
let wire: Vec<u8> = (0..used)
.flat_map(|index| case.h2g_buffer(index, buffers[index as usize]))
.collect();
let header = MsgHeader::from_bytes(&wire[..MsgHeader::SIZE]).unwrap();
assert_eq!(header.kind, MsgKind::Request);
assert_eq!(header.cid, cid);
assert_eq!(header.payload_len as usize, wire.len() - MsgHeader::SIZE);
let control =
size_prefix_payload_len(&wire[MsgHeader::SIZE..MsgHeader::SIZE + SIZE_PREFIX_LEN])
.unwrap();
let control_len = size_prefixed_len(control).unwrap();
let external = &wire[MsgHeader::SIZE + control_len..];
assert_eq!(external, vec![0xa5; external_len]);
let cursor = mgr.h2g_consumer.as_ref().unwrap().avail_cursor();
let error = mgr.write_guest_function_call(&h2g_call(1)).unwrap_err();
assert!(error.to_string().contains("H2G capacity"), "{error:#}");
assert_eq!(mgr.h2g_consumer.as_ref().unwrap().avail_cursor(), cursor);
assert_eq!(mgr.write_guest_function_call(&h2g_call(0)).unwrap(), 2);
}
#[test]
fn writes_header_only_snapshot_checkpoint() {
let case = TestCase::new();
let mut mgr = manager(&case);
let buffer = case.h2g_desc(0).addr;
let mbx = mgr.layout.get_transport_arena().mbx_offset();
let cp = MailboxValue::CheckpointComplete.as_bytes();
mgr.scratch_mem.copy_from_slice(cp, mbx).unwrap();
mgr.begin_snapshot_checkpoint().unwrap();
let wire = case.h2g_buffer(0, buffer);
let header = MsgHeader::from_bytes(&wire).unwrap();
assert_eq!(wire.len(), MsgHeader::SIZE);
assert_eq!(header.kind, MsgKind::SnapshotCheckpoint);
assert_eq!(header.cid, 0);
assert_eq!(header.payload_len, 0);
assert_eq!(mgr.next_guest_cid, 1);
assert_eq!(
MailboxValue::try_from(mgr.scratch_mem.read::<u64>(mbx).unwrap()).unwrap(),
MailboxValue::CheckpointPending
);
}
#[test]
fn incomplete_snapshot_checkpoint_is_fatal() {
let case = TestCase::new();
let mut mgr = manager(&case);
mgr.g2h_consumer = Some(case.g2h_consumer());
mgr.begin_snapshot_checkpoint().unwrap();
let error = mgr.finish_snapshot_checkpoint().unwrap_err();
assert!(matches!(error, HyperlightError::TransportError(_)));
assert!(error.is_poison_error());
}
#[test]
fn reads_completed_snapshot_checkpoint_status() {
let case = TestCase::new();
let mut mgr = manager(&case);
mgr.g2h_consumer = Some(case.g2h_consumer());
mgr.begin_snapshot_checkpoint().unwrap();
let mbx = mgr.layout.get_transport_arena().mbx_offset();
mgr.scratch_mem
.copy_from_slice(MailboxValue::CheckpointComplete.as_bytes(), mbx)
.unwrap();
mgr.finish_snapshot_checkpoint().unwrap();
let consumer = mgr.h2g_consumer.as_ref().unwrap();
assert_eq!(consumer.avail_cursor().head(), 0);
assert_eq!(consumer.used_cursor().head(), 0);
}
#[test]
fn invalid_checkpoint_status_is_fatal() {
for status in [2, 3, 0xa000, u64::MAX - 1, u64::MAX] {
let queue = TestCase::new();
let mut mgr = manager(&queue);
mgr.g2h_consumer = Some(queue.g2h_consumer());
let mbx = mgr.layout.get_transport_arena().mbx_offset();
mgr.scratch_mem.write::<u64>(mbx, status).unwrap();
let error = mgr.finish_snapshot_checkpoint().unwrap_err();
assert!(matches!(error, HyperlightError::TransportError(_)));
assert!(error.is_poison_error(), "{error}");
}
}
#[test]
fn guest_cid_wraps_without_zero() {
let case = TestCase::new();
let mut mgr = manager(&case);
mgr.next_guest_cid = u32::MAX;
assert_eq!(
mgr.write_guest_function_call(&h2g_call(0)).unwrap(),
u32::MAX
);
assert_eq!(mgr.write_guest_function_call(&h2g_call(0)).unwrap(), 1);
}
#[test]
fn cloning_preserves_guest_cids_without_consumers() {
let case = TestCase::new();
let mut mgr = manager(&case);
mgr.g2h_consumer = Some(case.g2h_consumer());
mgr.write_guest_function_call(&h2g_call(0)).unwrap();
let cloned = mgr.clone();
assert!(cloned.g2h_consumer.is_none());
assert!(cloned.h2g_consumer.is_none());
assert_eq!(cloned.next_guest_cid, 2);
assert!(mgr.g2h_consumer.is_some());
assert!(mgr.h2g_consumer.is_some());
}
#[test]
fn restoring_transport_preserves_guest_cids() {
let case = TestCase::new();
let mut mgr = manager(&case);
mgr.g2h_consumer = Some(case.g2h_consumer());
let captured = VirtqSnapshot::capture(&mgr.layout, &mgr.scratch_mem).unwrap();
assert_eq!(mgr.write_guest_function_call(&h2g_call(0)).unwrap(), 1);
mgr.g2h_consumer = None;
mgr.h2g_consumer = None;
mgr.restore_virtq(&captured).unwrap();
assert_eq!(mgr.write_guest_function_call(&h2g_call(0)).unwrap(), 2);
}
#[cfg(target_arch = "x86_64")]
fn verify_page_tables(name: &str, config: SandboxConfiguration) {
let path = simple_guest_as_pathbuf();
let snapshot = Snapshot::from_env(GuestBinary::FilePath(path), config)
.unwrap_or_else(|e| panic!("{}: failed to create snapshot: {}", name, e));
assert!(
unsafe { hyperlight_common::vmem::virt_to_phys(&snapshot, 0, 1) }
.next()
.is_none(),
"{}: NULL page (0x0) should NOT be mapped",
name
);
}
#[test]
#[cfg(target_arch = "x86_64")]
fn test_page_tables_for_various_configurations() {
let test_cases: [(&str, SandboxConfiguration); 4] = [
("default", { SandboxConfiguration::default() }),
("small (8MB heap)", {
let mut cfg = SandboxConfiguration::default();
cfg.set_heap_size(SMALL_HEAP_SIZE);
cfg
}),
("medium (64MB heap)", {
let mut cfg = SandboxConfiguration::default();
cfg.set_heap_size(MEDIUM_HEAP_SIZE);
cfg
}),
("large (256MB heap)", {
let mut cfg = SandboxConfiguration::default();
cfg.set_heap_size(LARGE_HEAP_SIZE);
cfg.set_scratch_size(0x100000);
cfg
}),
];
for (name, config) in test_cases {
verify_page_tables(name, config);
}
}
#[test]
#[cfg(target_arch = "x86_64")]
fn build_creates_virtq_consumers_before_initialization() {
let path = simple_guest_as_pathbuf();
let bin = GuestBinary::FilePath(path);
let config = SandboxConfiguration::default();
let snapshot = Snapshot::from_env(bin, config).unwrap();
let mgr = SandboxMemoryManager::from_snapshot(&snapshot).unwrap();
let (mgr, _) = mgr.build().unwrap();
assert!(mgr.g2h_consumer.is_some());
assert!(mgr.h2g_consumer.is_some());
}
}