1use flatbuffers::FlatBufferBuilder;
5use hyperlight_common::flatbuffer_wrappers::function_call::FunctionCall;
6use hyperlight_common::flatbuffer_wrappers::function_types::FunctionCallResult;
7use hyperlight_common::flatbuffer_wrappers::host_function_details::HostFunctionDetails;
8use hyperlight_common::flatbuffer_wrappers::util::estimate_flatbuffer_capacity;
9use hyperlight_common::log_level::GuestLogFilter;
10use hyperlight_common::transport::{Buf, EncodedMessage, ExternalValues, MailboxValue, MsgKind};
11use hyperlight_common::virtq::ReplyChain;
12use hyperlight_common::vmem::{self, PAGE_TABLE_SIZE};
13#[cfg(crashdump)]
14use hyperlight_common::vmem::{BasicMapping, MappingKind};
15use tracing::{Span, instrument};
16
17use super::layout::SandboxMemoryLayout;
18use super::shared_mem::{
19 ExclusiveSharedMemory, GuestSharedMemory, HostSharedMemory, ReadonlySharedMemory, SharedMemory,
20};
21use super::virtq::{self, G2hConsumer, H2gConsumer, VirtqSnapshot};
22use crate::hypervisor::regs::CommonSpecialRegisters;
23use crate::mem::memory_region::MemoryRegion;
24#[cfg(crashdump)]
25use crate::mem::memory_region::{CrashDumpRegion, MemoryRegionFlags, MemoryRegionType};
26use crate::sandbox::snapshot::{NextAction, Snapshot};
27use crate::{HyperlightError, Result, new_error};
28
29#[cfg(crashdump)]
30fn mapping_kind_to_flags(kind: &MappingKind) -> (MemoryRegionFlags, MemoryRegionType) {
31 match kind {
32 MappingKind::Basic(BasicMapping {
33 readable,
34 writable,
35 executable,
36 }) => {
37 let mut flags = MemoryRegionFlags::empty();
38 if *readable {
39 flags |= MemoryRegionFlags::READ;
40 }
41 if *writable {
42 flags |= MemoryRegionFlags::WRITE;
43 }
44 if *executable {
45 flags |= MemoryRegionFlags::EXECUTE;
46 }
47 (flags, MemoryRegionType::Snapshot)
48 }
49 MappingKind::Cow(cow) => {
50 let mut flags = MemoryRegionFlags::empty();
51 if cow.readable {
52 flags |= MemoryRegionFlags::READ;
53 }
54 if cow.executable {
55 flags |= MemoryRegionFlags::EXECUTE;
56 }
57 (flags, MemoryRegionType::Scratch)
58 }
59 MappingKind::Unmapped => (MemoryRegionFlags::empty(), MemoryRegionType::Snapshot),
60 MappingKind::ZeroInit(bm) => {
61 let mut flags = MemoryRegionFlags::empty();
62 if bm.readable {
63 flags |= MemoryRegionFlags::READ;
64 }
65 if bm.writable {
66 flags |= MemoryRegionFlags::WRITE;
67 }
68 if bm.executable {
69 flags |= MemoryRegionFlags::EXECUTE;
70 }
71 (flags, MemoryRegionType::Snapshot)
72 }
73 }
74}
75
76#[cfg(crashdump)]
81fn try_coalesce_region(
82 regions: &mut [CrashDumpRegion],
83 virt_base: usize,
84 virt_end: usize,
85 host_base: usize,
86 flags: MemoryRegionFlags,
87) -> bool {
88 if let Some(last) = regions.last_mut()
89 && last.guest_region.end == virt_base
90 && last.host_region.end == host_base
91 && last.flags == flags
92 {
93 last.guest_region.end = virt_end;
94 last.host_region.end = host_base + (virt_end - virt_base);
95 return true;
96 }
97 false
98}
99
100mod unused_hack {
108 #[cfg(not(unshared_snapshot_mem))]
109 use crate::mem::shared_mem::ReadonlySharedMemory;
110 use crate::mem::shared_mem::SharedMemory;
111 pub trait SnapshotSharedMemoryT {
112 type T<S: SharedMemory>;
113 }
114 pub struct SnapshotSharedMemory_;
115 impl SnapshotSharedMemoryT for SnapshotSharedMemory_ {
116 #[cfg(not(unshared_snapshot_mem))]
117 type T<S: SharedMemory> = ReadonlySharedMemory;
118 #[cfg(unshared_snapshot_mem)]
119 type T<S: SharedMemory> = S;
120 }
121 pub type SnapshotSharedMemory<S> = <SnapshotSharedMemory_ as SnapshotSharedMemoryT>::T<S>;
122}
123impl ReadonlySharedMemory {
124 pub(crate) fn to_mgr_snapshot_mem(
125 &self,
126 ) -> Result<SnapshotSharedMemory<ExclusiveSharedMemory>> {
127 #[cfg(not(unshared_snapshot_mem))]
128 let ret = self.clone();
129 #[cfg(unshared_snapshot_mem)]
130 let ret = self.copy_to_writable()?;
131 Ok(ret)
132 }
133}
134pub(crate) use unused_hack::SnapshotSharedMemory;
135
136#[derive(Debug, Clone, Copy, PartialEq, Eq)]
138pub(crate) enum ExecutionMode {
139 Unrestricted,
141 Restricted,
143}
144
145pub(crate) struct SandboxMemoryManager<S: SharedMemory> {
148 pub(crate) shared_mem: SnapshotSharedMemory<S>,
150 pub(crate) scratch_mem: S,
152 pub(crate) layout: SandboxMemoryLayout,
154 pub(crate) next_action: NextAction,
157 pub(crate) original_entrypoint: u64,
161 pub(crate) abort_buffer: Vec<u8>,
163 pub(crate) execution_mode: ExecutionMode,
165 pub(crate) snapshot_count: u64,
171 pub(crate) g2h_consumer: Option<G2hConsumer>,
173 pub(crate) h2g_consumer: Option<H2gConsumer>,
175 next_guest_cid: u32,
177}
178
179impl<S: Clone + SharedMemory> Clone for SandboxMemoryManager<S> {
180 fn clone(&self) -> Self {
181 Self {
182 shared_mem: self.shared_mem.clone(),
183 scratch_mem: self.scratch_mem.clone(),
184 layout: self.layout,
185 next_action: self.next_action,
186 original_entrypoint: self.original_entrypoint,
187 abort_buffer: self.abort_buffer.clone(),
188 execution_mode: self.execution_mode,
189 snapshot_count: self.snapshot_count,
190 g2h_consumer: None,
191 h2g_consumer: None,
192 next_guest_cid: self.next_guest_cid,
193 }
194 }
195}
196
197pub(crate) struct GuestPageTableBuffer {
201 buffer: std::cell::RefCell<Vec<u8>>,
202 phys_base: usize,
203 root: std::cell::Cell<u64>,
208}
209
210impl vmem::TableReadOps for GuestPageTableBuffer {
211 type TableAddr = u64;
212
213 fn entry_addr(addr: u64, offset: u64) -> u64 {
214 addr + offset
215 }
216
217 unsafe fn read_entry(&self, addr: u64) -> vmem::PageTableEntry {
218 let buffer = self.buffer.borrow();
219 let byte_offset = addr as usize - self.phys_base;
220 let pte_size = core::mem::size_of::<vmem::PageTableEntry>();
221 let Some(bytes) = buffer.get(byte_offset..byte_offset + pte_size) else {
222 return 0;
223 };
224 let mut buf = [0u8; 8];
225 buf[..pte_size].copy_from_slice(bytes);
226 vmem::PageTableEntry::from_le_bytes(buf[..pte_size].try_into().unwrap_or_default())
227 }
228
229 fn to_phys(addr: u64) -> vmem::PhysAddr {
230 addr as vmem::PhysAddr
231 }
232
233 fn from_phys(addr: vmem::PhysAddr) -> u64 {
234 #[allow(clippy::unnecessary_cast)]
235 {
236 addr as u64
237 }
238 }
239
240 fn root_table(&self) -> u64 {
241 self.root.get()
242 }
243}
244
245impl vmem::TableOps for GuestPageTableBuffer {
246 type TableMovability = vmem::MayNotMoveTable;
247
248 unsafe fn alloc_table(&self) -> u64 {
249 let mut b = self.buffer.borrow_mut();
250 let offset = b.len();
251 b.resize(offset + PAGE_TABLE_SIZE, 0);
252 (self.phys_base + offset) as u64
253 }
254
255 unsafe fn write_entry(&self, addr: u64, entry: vmem::PageTableEntry) -> Option<vmem::Void> {
256 let mut b = self.buffer.borrow_mut();
257 let byte_offset = addr as usize - self.phys_base;
258 let pte_size = core::mem::size_of::<vmem::PageTableEntry>();
259 if let Some(slice) = b.get_mut(byte_offset..byte_offset + pte_size) {
260 slice.copy_from_slice(&entry.to_le_bytes()[..pte_size]);
261 }
262 None
263 }
264
265 unsafe fn update_root(&self, impossible: vmem::Void) {
266 match impossible {}
267 }
268}
269
270impl core::convert::AsRef<GuestPageTableBuffer> for GuestPageTableBuffer {
271 fn as_ref(&self) -> &Self {
272 self
273 }
274}
275
276impl GuestPageTableBuffer {
277 pub(crate) fn new(phys_base: usize) -> Self {
281 GuestPageTableBuffer {
282 buffer: std::cell::RefCell::new(vec![0u8; PAGE_TABLE_SIZE]),
283 phys_base,
284 root: std::cell::Cell::new(phys_base as u64),
285 }
286 }
287
288 pub(crate) fn set_root(&self, addr: u64) {
291 self.root.set(addr);
292 }
293
294 pub(crate) fn initial_root(&self) -> u64 {
296 self.phys_base as u64
297 }
298
299 #[cfg(test)]
300 #[allow(dead_code)]
301 pub(crate) fn size(&self) -> usize {
302 self.buffer.borrow().len()
303 }
304
305 pub(crate) fn into_bytes(self) -> Box<[u8]> {
306 self.buffer.into_inner().into_boxed_slice()
307 }
308}
309
310impl<S> SandboxMemoryManager<S>
311where
312 S: SharedMemory,
313{
314 #[instrument(skip_all, parent = Span::current(), level= "Trace")]
316 pub(crate) fn new(
317 layout: SandboxMemoryLayout,
318 shared_mem: SnapshotSharedMemory<S>,
319 scratch_mem: S,
320 next_action: NextAction,
321 ) -> Self {
322 Self {
323 layout,
324 shared_mem,
325 scratch_mem,
326 next_action,
327 original_entrypoint: 0,
328 abort_buffer: Vec::new(),
329 execution_mode: ExecutionMode::Unrestricted,
330 snapshot_count: 0,
331 g2h_consumer: None,
332 h2g_consumer: None,
333 next_guest_cid: 1,
334 }
335 }
336
337 pub(crate) fn get_abort_buffer_mut(&mut self) -> &mut Vec<u8> {
339 &mut self.abort_buffer
340 }
341}
342
343impl SandboxMemoryManager<ExclusiveSharedMemory> {
344 pub(crate) fn from_snapshot(s: &Snapshot) -> Result<Self> {
345 let layout = *s.layout();
346 let shared_mem = s.memory().to_mgr_snapshot_mem()?;
347 let scratch_mem = ExclusiveSharedMemory::new(s.layout().get_scratch_size())?;
348 let next_action = s.next_action();
349 let mut mgr = Self::new(layout, shared_mem, scratch_mem, next_action);
350 mgr.original_entrypoint = s.original_entrypoint();
351 mgr.snapshot_count = s.snapshot_generation();
356 Ok(mgr)
357 }
358
359 pub fn build(
370 self,
371 ) -> Result<(
372 SandboxMemoryManager<HostSharedMemory>,
373 SandboxMemoryManager<GuestSharedMemory>,
374 )> {
375 let (hshm, gshm) = self.shared_mem.build();
376 let (hscratch, gscratch) = self.scratch_mem.build();
377 let mut host_mgr = SandboxMemoryManager {
378 shared_mem: hshm,
379 scratch_mem: hscratch,
380 layout: self.layout,
381 next_action: self.next_action,
382 original_entrypoint: self.original_entrypoint,
383 abort_buffer: self.abort_buffer,
384 execution_mode: self.execution_mode,
385 snapshot_count: self.snapshot_count,
386 g2h_consumer: None,
387 h2g_consumer: None,
388 next_guest_cid: self.next_guest_cid,
389 };
390 let guest_mgr = SandboxMemoryManager {
391 shared_mem: gshm,
392 scratch_mem: gscratch,
393 layout: self.layout,
394 next_action: self.next_action,
395 original_entrypoint: self.original_entrypoint,
396 abort_buffer: Vec::new(), execution_mode: self.execution_mode,
398 snapshot_count: self.snapshot_count,
399 g2h_consumer: None,
400 h2g_consumer: None,
401 next_guest_cid: self.next_guest_cid,
402 };
403 host_mgr.update_scratch_bookkeeping()?;
404
405 if matches!(host_mgr.next_action, NextAction::Initialise(_)) {
406 let cp = MailboxValue::CheckpointPending.as_bytes();
407 let mbx = host_mgr.layout.get_transport_arena().mbx_offset();
408
409 host_mgr.scratch_mem.copy_from_slice(cp, mbx)?;
410 host_mgr.create_virtq_consumers()?;
411 }
412
413 Ok((host_mgr, guest_mgr))
414 }
415}
416
417impl SandboxMemoryManager<HostSharedMemory> {
418 #[allow(clippy::too_many_arguments)]
420 pub(crate) fn snapshot(
421 &mut self,
422 mapped_regions: Vec<MemoryRegion>,
423 root_pt_gpas: &[u64],
424 rsp_gva: u64,
425 sregs: CommonSpecialRegisters,
426 #[cfg(target_arch = "x86_64")] msrs: Vec<crate::hypervisor::regs::MsrEntry>,
427 next_action: NextAction,
428 host_functions: HostFunctionDetails,
429 ) -> Result<Snapshot> {
430 let virtq = match (&self.g2h_consumer, &self.h2g_consumer) {
431 (Some(_), Some(_)) => Some(VirtqSnapshot::capture(&self.layout, &self.scratch_mem)?),
432 (None, None) => None,
433 _ => return Err(new_error!("virtqueue consumer ownership is incomplete")),
434 };
435
436 let snapshot = Snapshot::new(
437 &mut self.shared_mem,
438 &mut self.scratch_mem,
439 self.layout,
440 crate::mem::exe::LoadInfo::dummy(),
441 mapped_regions,
442 root_pt_gpas,
443 rsp_gva,
444 sregs,
445 #[cfg(target_arch = "x86_64")]
446 msrs,
447 next_action,
448 self.original_entrypoint,
449 self.snapshot_count + 1,
450 host_functions,
451 virtq,
452 )?;
453 self.snapshot_count = snapshot.snapshot_generation();
454 Ok(snapshot)
455 }
456
457 fn create_virtq_consumers(&mut self) -> Result<()> {
462 if self.g2h_consumer.is_some() || self.h2g_consumer.is_some() {
463 return Err(new_error!("virtqueue consumers already exist"));
464 }
465
466 let (g2h, h2g) = virtq::create_consumers(&self.layout, &self.scratch_mem)?;
467 self.g2h_consumer = Some(g2h);
468 self.h2g_consumer = Some(h2g);
469 Ok(())
470 }
471
472 pub(crate) fn restore_virtq(&mut self, snapshot: &VirtqSnapshot) -> Result<()> {
474 if self.g2h_consumer.is_some() || self.h2g_consumer.is_some() {
475 return Err(new_error!("virtqueue consumers are already attached"));
476 }
477
478 let (g2h, h2g) = snapshot.restore(&self.layout, &self.scratch_mem)?;
479 self.g2h_consumer = Some(g2h);
480 self.h2g_consumer = Some(h2g);
481 Ok(())
482 }
483
484 #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
486 pub(crate) fn write_guest_function_call(&mut self, call: &FunctionCall) -> Result<u32> {
487 let cid = self.next_guest_cid;
488 let params = call.parameters.as_deref().unwrap_or_default();
489 let cap = estimate_flatbuffer_capacity(&call.function_name, params);
490
491 let mut builder = FlatBufferBuilder::with_capacity(cap);
492 let mut externals = ExternalValues::new();
493
494 let control = call.encode(&mut builder, &mut externals)?;
495
496 let Some(msg) = EncodedMessage::new(MsgKind::Request, cid, control, externals) else {
497 return Err(new_error!("H2G request exceeds the wire payload limit"));
498 };
499
500 self.write_h2g_message(&msg)?;
501
502 self.next_guest_cid = cid.wrapping_add(1);
503 if self.next_guest_cid == 0 {
504 self.next_guest_cid = 1;
505 }
506
507 Ok(cid)
508 }
509
510 fn write_h2g_message(&mut self, message: &EncodedMessage<'_>) -> Result<()> {
511 let consumer = self.h2g_consumer.as_mut().ok_or_else(|| {
512 HyperlightError::TransportError("H2G consumer is not attached".into())
513 })?;
514
515 let buffer_size = self.layout.get_h2g_buffer_size();
516 let buffer_count = message.total_len().div_ceil(buffer_size);
517
518 let control_reserve = usize::from(message.external_len() != 0);
520
521 let maybe_buffers = consumer
523 .poll_batch(buffer_count, control_reserve, 0)
524 .map_err(|error| {
525 HyperlightError::TransportError(format!("H2G poll failed: {error}"))
526 })?;
527
528 let Some(buffers) = maybe_buffers else {
529 return Err(new_error!(
530 "H2G capacity cannot provide {buffer_count} buffers with {control_reserve} spare"
531 ));
532 };
533
534 let mut message = message.as_buf();
538
539 for (recv, reply) in buffers {
540 let ReplyChain::Writable(mut buffer) = reply else {
541 return Err(HyperlightError::TransportError(
542 "H2G receive buffer is not writable".into(),
543 ));
544 };
545
546 if buffer.desc_count() != 1 || buffer.capacity() != buffer_size {
547 return Err(HyperlightError::TransportError(
548 "H2G receive buffer has an invalid shape".into(),
549 ));
550 }
551
552 while message.has_remaining() && buffer.remaining() != 0 {
553 let written = buffer.write(message.chunk()).map_err(|err| {
554 HyperlightError::TransportError(format!("H2G write failed: {err}"))
555 })?;
556
557 message.advance(written);
558 }
559
560 consumer.complete(recv, buffer).map_err(|err| {
561 HyperlightError::TransportError(format!("H2G completion failed: {err}"))
562 })?;
563 }
564
565 debug_assert!(!message.has_remaining());
566 Ok(())
567 }
568
569 #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
571 pub(crate) fn read_h2g_result_from_g2h(&mut self, cid: u32) -> Result<FunctionCallResult> {
572 let max_recv_len = self.layout.get_g2h_queue_dims().pool_len();
573
574 let Some(consumer) = self.g2h_consumer.as_mut() else {
575 return Err(HyperlightError::TransportError(
576 "G2H consumer is not attached".into(),
577 ));
578 };
579
580 loop {
581 let maybe_next = consumer.poll(max_recv_len).map_err(|err| {
582 HyperlightError::TransportError(format!("G2H poll failed: {err}"))
583 })?;
584
585 let Some((mut recv, reply)) = maybe_next else {
586 return Err(HyperlightError::TransportError(
587 "G2H has no guest function result after halt".into(),
588 ));
589 };
590
591 let header = virtq::read_message_header(&mut recv).map_err(|err| {
592 HyperlightError::TransportError(format!("Failed to read G2H result header: {err}"))
593 })?;
594
595 if !matches!(&reply, ReplyChain::Ack(_)) {
596 return Err(HyperlightError::TransportError(
597 "G2H result entry has writable buffers".into(),
598 ));
599 }
600
601 match header.kind {
602 MsgKind::Log => {
603 if header.cid != 0 {
604 return Err(HyperlightError::TransportError(
605 "G2H log has a correlation ID".into(),
606 ));
607 }
608
609 let log = virtq::read_guest_log_data(&mut recv).map_err(|err| {
610 HyperlightError::TransportError(format!("Failed to read G2H log: {err}"))
611 })?;
612
613 consumer.complete(recv, reply).map_err(|err| {
614 HyperlightError::TransportError(format!(
615 "Failed to complete G2H log: {err}"
616 ))
617 })?;
618
619 crate::sandbox::outb::emit_guest_log(&log);
620 }
621 MsgKind::Response => {
622 if header.cid != cid {
623 return Err(HyperlightError::TransportError(
624 "G2H guest function result correlation ID mismatch".into(),
625 ));
626 }
627
628 let result = virtq::read_guest_function_call_result(&mut recv);
629 consumer.complete(recv, reply).map_err(|err| {
630 HyperlightError::TransportError(format!(
631 "Failed to complete G2H guest function result: {err}"
632 ))
633 })?;
634
635 return result.map_err(|err| {
636 HyperlightError::TransportError(format!(
637 "Failed to decode G2H guest function result: {err}"
638 ))
639 });
640 }
641 kind => {
642 return Err(HyperlightError::TransportError(format!(
643 "Expected G2H guest function result, got {kind:?}"
644 )));
645 }
646 }
647 }
648 }
649
650 pub(crate) fn begin_snapshot_checkpoint(&mut self) -> Result<()> {
654 let offset = self.layout.get_transport_arena().mbx_offset();
655 let cp = MailboxValue::CheckpointPending.as_bytes();
656 self.scratch_mem.copy_from_slice(cp, offset)?;
657
658 let message = EncodedMessage::new_snapshot_cp();
659 self.write_h2g_message(&message)
660 }
661
662 pub(crate) fn finish_snapshot_checkpoint(&mut self) -> Result<()> {
664 let Some(g2h) = self.g2h_consumer.as_mut() else {
665 return Err(new_error!("G2H consumer is not attached"));
666 };
667
668 let Some(h2g) = self.h2g_consumer.as_mut() else {
669 return Err(new_error!("H2G consumer is not attached"));
670 };
671
672 g2h.reset()?;
673 h2g.reset()?;
674
675 let status = self
676 .scratch_mem
677 .read::<u64>(self.layout.get_transport_arena().mbx_offset())
678 .map_err(|error| HyperlightError::TransportError(error.to_string()))?;
679
680 match MailboxValue::try_from(status) {
681 Ok(MailboxValue::CheckpointComplete) => Ok(()),
682 Ok(MailboxValue::CheckpointPending) => Err(HyperlightError::TransportError(
683 "Guest did not publish snapshot checkpoint status".into(),
684 )),
685 Err(_) => Err(HyperlightError::TransportError(format!(
686 "Guest published invalid snapshot checkpoint status {status:#x}"
687 ))),
688 }
689 }
690
691 pub(crate) fn restore_snapshot(
693 &mut self,
694 snapshot: &Snapshot,
695 ) -> Result<(
696 Option<SnapshotSharedMemory<GuestSharedMemory>>,
697 Option<GuestSharedMemory>,
698 )> {
699 let virtq = snapshot.virtq();
700
701 if virtq.is_none() && matches!(snapshot.next_action(), NextAction::Call(_)) {
702 return Err(new_error!(
703 "running snapshot has no canonical transport state"
704 ));
705 }
706
707 self.g2h_consumer = None;
708 self.h2g_consumer = None;
709 self.execution_mode = ExecutionMode::Unrestricted;
710
711 self.execution_mode = ExecutionMode::Unrestricted;
713
714 let gsnapshot = if *snapshot.memory() == self.shared_mem {
715 None
723 } else {
724 let new_snapshot_mem = snapshot.memory().to_mgr_snapshot_mem()?;
725 let (hsnapshot, gsnapshot) = new_snapshot_mem.build();
726 self.shared_mem = hsnapshot;
727 Some(gsnapshot)
728 };
729 let new_scratch_size = snapshot.layout().get_scratch_size();
730 let gscratch = if new_scratch_size == self.scratch_mem.mem_size() {
731 self.scratch_mem.zero_or_replace()?
734 } else {
735 let new_scratch_mem = ExclusiveSharedMemory::new(new_scratch_size)?;
736 let (hscratch, gscratch) = new_scratch_mem.build();
737 self.scratch_mem = hscratch;
743 Some(gscratch)
744 };
745 self.layout = *snapshot.layout();
746 self.snapshot_count = snapshot.snapshot_generation();
751 self.original_entrypoint = snapshot.original_entrypoint();
754 self.update_scratch_bookkeeping()?;
755 if let Some(virtq) = virtq {
756 self.restore_virtq(virtq)?;
757 } else if matches!(snapshot.next_action(), NextAction::Initialise(_)) {
758 self.scratch_mem.copy_from_slice(
759 MailboxValue::CheckpointPending.as_bytes(),
760 self.layout.get_transport_arena().mbx_offset(),
761 )?;
762 self.create_virtq_consumers()?;
763 }
764 Ok((gsnapshot, gscratch))
765 }
766
767 #[inline]
768 fn update_scratch_bookkeeping_item(&mut self, offset: u64, value: u64) -> Result<()> {
769 let scratch_size = self.scratch_mem.mem_size();
770 let base_offset = scratch_size - offset as usize;
771 self.scratch_mem
772 .write::<u64>(base_offset, value)
773 .map_err(From::from)
774 }
775
776 pub(crate) fn request_libc_rng_reseed(&mut self, seed: u32) -> Result<()> {
777 self.update_scratch_bookkeeping_item(
780 hyperlight_common::layout::SCRATCH_TOP_LIBC_RNG_SEED_OFFSET,
781 (1_u64 << 32) | u64::from(seed),
782 )
783 }
784
785 pub(crate) fn request_guest_log_level_update(
786 &mut self,
787 log_level: tracing_core::LevelFilter,
788 ) -> Result<()> {
789 self.update_scratch_bookkeeping_item(
790 hyperlight_common::layout::SCRATCH_TOP_GUEST_LOG_LEVEL_OFFSET,
791 (hyperlight_common::log_level::GUEST_LOG_FILTER_UPDATE_PENDING)
792 | u64::from(GuestLogFilter::from(log_level)),
793 )
794 }
795
796 fn update_scratch_bookkeeping(&mut self) -> Result<()> {
797 use hyperlight_common::layout::*;
798 let scratch_size = self.scratch_mem.mem_size();
799 self.update_scratch_bookkeeping_item(SCRATCH_TOP_SIZE_OFFSET, scratch_size as u64)?;
800 self.update_scratch_bookkeeping_item(
801 SCRATCH_TOP_ALLOCATOR_OFFSET,
802 self.layout.get_first_free_scratch_gpa(),
803 )?;
804 self.update_scratch_bookkeeping_item(
813 SCRATCH_TOP_SNAPSHOT_PT_GPA_BASE_OFFSET,
814 self.layout.get_pt_base_gpa(),
815 )?;
816 self.update_scratch_bookkeeping_item(
817 SCRATCH_TOP_SNAPSHOT_GENERATION_OFFSET,
818 self.snapshot_count,
819 )?;
820 self.update_scratch_bookkeeping_item(
822 SCRATCH_TOP_G2H_QUEUE_SIZE_OFFSET,
823 u64::try_from(self.layout.get_g2h_queue_size())?,
824 )?;
825 self.update_scratch_bookkeeping_item(
826 SCRATCH_TOP_G2H_POOL_PAGES_OFFSET,
827 u64::try_from(self.layout.get_g2h_pool_pages())?,
828 )?;
829 self.update_scratch_bookkeeping_item(
830 SCRATCH_TOP_G2H_BUFFER_SIZE_OFFSET,
831 u64::try_from(self.layout.get_g2h_buffer_size())?,
832 )?;
833 self.update_scratch_bookkeeping_item(
834 SCRATCH_TOP_H2G_QUEUE_SIZE_OFFSET,
835 u64::try_from(self.layout.get_h2g_queue_size())?,
836 )?;
837 self.update_scratch_bookkeeping_item(
838 SCRATCH_TOP_H2G_POOL_PAGES_OFFSET,
839 u64::try_from(self.layout.get_h2g_pool_pages())?,
840 )?;
841 self.update_scratch_bookkeeping_item(
842 SCRATCH_TOP_H2G_BUFFER_SIZE_OFFSET,
843 u64::try_from(self.layout.get_h2g_buffer_size())?,
844 )?;
845
846 let transport_arena = self.layout.get_transport_arena();
847 self.update_scratch_bookkeeping_item(
848 SCRATCH_TOP_TRANSPORT_ARENA_GPA_OFFSET,
849 transport_arena.base_addr(),
850 )?;
851
852 let snapshot_pt_end = self.shared_mem.mem_size();
859 let snapshot_pt_size = self.layout.get_pt_size();
860 let snapshot_pt_start =
861 snapshot_pt_end - snapshot_pt_size.next_multiple_of(page_size::get());
862 self.scratch_mem.with_exclusivity(|scratch| {
863 #[cfg(not(unshared_snapshot_mem))]
864 let bytes = &self.shared_mem.as_slice()[snapshot_pt_start..snapshot_pt_end];
865 #[cfg(unshared_snapshot_mem)]
866 let bytes = {
867 let mut bytes = vec![0u8; snapshot_pt_size];
868 self.shared_mem
869 .copy_to_slice(&mut bytes, snapshot_pt_start)?;
870 bytes
871 };
872 #[allow(clippy::needless_borrow)]
873 scratch.copy_from_slice(&bytes, self.layout.get_pt_base_scratch_offset())
874 })??;
875
876 Ok(())
877 }
878
879 #[cfg(crashdump)]
884 pub(crate) fn get_guest_memory_regions(
885 &mut self,
886 root_pt: u64,
887 mmap_regions: &[MemoryRegion],
888 ) -> Result<Vec<CrashDumpRegion>> {
889 use crate::sandbox::snapshot::SharedMemoryPageTableBuffer;
890
891 let len = hyperlight_common::layout::SCRATCH_TOP_GVA;
892
893 let regions = self.shared_mem.with_contents(|snapshot| {
894 self.scratch_mem.with_contents(|scratch| {
895 let pt_buf =
896 SharedMemoryPageTableBuffer::new(snapshot, scratch, self.layout, root_pt);
897
898 let mappings: Vec<_> =
899 unsafe { hyperlight_common::vmem::virt_to_phys(&pt_buf, 0, len as u64) }
900 .collect();
901
902 if mappings.is_empty() {
903 return Err(new_error!("No page table mappings found (len {len})",));
904 }
905
906 let mut regions: Vec<CrashDumpRegion> = Vec::new();
907 for mapping in &mappings {
908 let virt_base = mapping.virt_base as usize;
909 let virt_end = (mapping.virt_base + mapping.len) as usize;
910
911 if let Some(resolved) = self.layout.resolve_gpa(mapping.phys_base, mmap_regions)
912 {
913 let (flags, region_type) = mapping_kind_to_flags(&mapping.kind);
914 let resolved = resolved.with_memories(snapshot, scratch);
915 let contents = resolved.as_ref();
916 let host_base = contents.as_ptr() as usize;
917 let host_len = (mapping.len as usize).min(contents.len());
918
919 if try_coalesce_region(&mut regions, virt_base, virt_end, host_base, flags)
920 {
921 continue;
922 }
923
924 regions.push(CrashDumpRegion {
925 guest_region: virt_base..virt_end,
926 host_region: host_base..host_base + host_len,
927 flags,
928 region_type,
929 });
930 }
931 }
932
933 Ok(regions)
934 })
935 })???;
936
937 Ok(regions)
938 }
939
940 #[cfg(feature = "trace_guest")]
953 pub(crate) fn read_guest_memory_by_gva(
954 &mut self,
955 gva: u64,
956 len: usize,
957 root_pt: u64,
958 ) -> Result<Vec<u8>> {
959 use hyperlight_common::vmem::PAGE_SIZE;
960
961 use crate::sandbox::snapshot::{SharedMemoryPageTableBuffer, access_gpa};
962
963 self.shared_mem.with_contents(|snap| {
964 self.scratch_mem.with_contents(|scratch| {
965 let pt_buf = SharedMemoryPageTableBuffer::new(snap, scratch, self.layout, root_pt);
966
967 let mappings: Vec<_> = unsafe {
969 hyperlight_common::vmem::virt_to_phys(&pt_buf, gva, len as u64)
970 }
971 .collect();
972
973 if mappings.is_empty() {
974 return Err(new_error!(
975 "No page table mappings found for GVA {:#x} (len {})",
976 gva,
977 len,
978 ));
979 }
980
981 let mut result = Vec::with_capacity(len);
983 let mut current_gva = gva;
984
985 for mapping in &mappings {
986 if mapping.virt_base > current_gva {
989 return Err(new_error!(
990 "Page table walker returned mapping with virt_base {:#x} > current read position {:#x}",
991 mapping.virt_base,
992 current_gva,
993 ));
994 }
995
996 let page_offset = (current_gva - mapping.virt_base) as usize;
998
999 let bytes_remaining = len - result.len();
1000 let available_in_page = PAGE_SIZE - page_offset;
1001 let bytes_to_copy = bytes_remaining.min(available_in_page);
1002
1003 let gpa = mapping.phys_base + page_offset as u64;
1005 let (mem, offset) = access_gpa(snap, scratch, self.layout, gpa)
1006 .ok_or_else(|| {
1007 new_error!(
1008 "Failed to resolve GPA {:#x} to host memory (GVA {:#x})",
1009 gpa,
1010 gva
1011 )
1012 })?;
1013
1014 let slice = mem
1015 .get(offset..offset + bytes_to_copy)
1016 .ok_or_else(|| {
1017 new_error!(
1018 "GPA {:#x} resolved to out-of-bounds host offset {} (need {} bytes)",
1019 gpa,
1020 offset,
1021 bytes_to_copy
1022 )
1023 })?;
1024
1025 result.extend_from_slice(slice);
1026 current_gva += bytes_to_copy as u64;
1027 }
1028
1029 if result.len() != len {
1030 tracing::error!(
1031 "Page table walker returned mappings that don't cover the full requested length: got {}, expected {}",
1032 result.len(),
1033 len,
1034 );
1035 return Err(new_error!(
1036 "Could not read full GVA range: got {} of {} bytes {:?}",
1037 result.len(),
1038 len,
1039 mappings
1040 ));
1041 }
1042
1043 Ok(result)
1044 })
1045 })??
1046 }
1047}
1048
1049#[cfg(test)]
1050mod tests {
1051 use hyperlight_common::flatbuffer_wrappers::function_call::FunctionCallType;
1052 use hyperlight_common::flatbuffer_wrappers::function_types::{ParameterValue, ReturnType};
1053 use hyperlight_common::transport::{
1054 MsgHeader, SIZE_PREFIX_LEN, size_prefix_payload_len, size_prefixed_len,
1055 };
1056 use hyperlight_common::virtq::DescFlags;
1057 #[cfg(target_arch = "x86_64")]
1058 use hyperlight_testing::sandbox_sizes::{LARGE_HEAP_SIZE, MEDIUM_HEAP_SIZE, SMALL_HEAP_SIZE};
1059 #[cfg(target_arch = "x86_64")]
1060 use hyperlight_testing::simple_guest_as_pathbuf;
1061
1062 use super::*;
1063 #[cfg(target_arch = "x86_64")]
1064 use crate::GuestBinary;
1065 use crate::mem::virtq::tests::{H2G_BUFFER_SIZE, SCRATCH_SIZE, TestCase, memory_layout};
1066 #[cfg(target_arch = "x86_64")]
1067 use crate::sandbox::SandboxConfiguration;
1068
1069 fn manager(case: &TestCase) -> SandboxMemoryManager<HostSharedMemory> {
1070 let host_page_size = page_size::get();
1071
1072 #[cfg(not(unshared_snapshot_mem))]
1073 let shared_mem =
1074 ReadonlySharedMemory::from_bytes(&vec![0; host_page_size], host_page_size).unwrap();
1075
1076 #[cfg(unshared_snapshot_mem)]
1077 let shared_mem = ExclusiveSharedMemory::new(host_page_size)
1078 .unwrap()
1079 .build()
1080 .0;
1081
1082 let mut mgr = SandboxMemoryManager::new(
1083 memory_layout(),
1084 shared_mem,
1085 case.scratch.clone(),
1086 NextAction::None,
1087 );
1088
1089 mgr.h2g_consumer = Some(case.h2g_consumer());
1090 mgr
1091 }
1092
1093 fn h2g_call(bytes: usize) -> FunctionCall {
1094 let params = (bytes != 0).then(|| vec![ParameterValue::VecBytes(vec![0xa5; bytes])]);
1095 FunctionCall::new(
1096 "call".to_string(),
1097 params,
1098 FunctionCallType::Guest,
1099 ReturnType::Void,
1100 )
1101 }
1102
1103 #[test]
1104 fn rejects_invalid_h2g_descriptors() {
1105 for (len, expected) in [
1106 (H2G_BUFFER_SIZE as u32, "Payload data too large"),
1107 (0, "not writable"),
1108 ] {
1109 let case = TestCase::new();
1110 let mut mgr = manager(&case);
1111 let mut desc = case.h2g_desc(0);
1112
1113 desc.flags &= !DescFlags::WRITE.bits();
1114 desc.len = len;
1115 case.set_h2g_desc(0, desc);
1116
1117 let error = mgr.write_guest_function_call(&h2g_call(0)).unwrap_err();
1118
1119 assert!(error.to_string().contains(expected), "{error:#}");
1120 assert!(error.is_poison_error());
1121 assert!(matches!(error, HyperlightError::TransportError(_)));
1122 }
1123 }
1124
1125 #[test]
1126 fn partial_h2g_write_is_fatal() {
1127 let case = TestCase::new();
1128 let mut mgr = manager(&case);
1129 let mut desc = case.h2g_desc(1);
1130
1131 desc.addr = hyperlight_common::layout::scratch_base_gva(SCRATCH_SIZE) + SCRATCH_SIZE as u64;
1132 case.set_h2g_desc(1, desc);
1133
1134 let error = mgr
1135 .write_guest_function_call(&h2g_call(H2G_BUFFER_SIZE + 1024))
1136 .unwrap_err();
1137
1138 assert!(error.to_string().contains("Memory write"), "{error:#}");
1139 assert!(error.is_poison_error());
1140 assert!(matches!(error, HyperlightError::TransportError(_)));
1141 assert_eq!(mgr.h2g_consumer.as_ref().unwrap().used_cursor().head(), 1);
1142 }
1143
1144 #[test]
1145 fn insufficient_h2g_capacity_rolls_back() {
1146 let case = TestCase::new();
1147 let mut mgr = manager(&case);
1148 let cursor = mgr.h2g_consumer.as_ref().unwrap().avail_cursor();
1149
1150 let error = mgr
1151 .write_guest_function_call(&h2g_call(H2G_BUFFER_SIZE * 4))
1152 .unwrap_err();
1153
1154 assert!(error.to_string().contains("H2G capacity"), "{error:#}");
1155 assert!(!error.is_poison_error());
1156 assert_eq!(mgr.h2g_consumer.as_ref().unwrap().avail_cursor(), cursor);
1157 assert_eq!(mgr.write_guest_function_call(&h2g_call(0)).unwrap(), 1);
1158 }
1159
1160 #[test]
1161 fn missing_g2h_result_is_fatal() {
1162 let case = TestCase::new();
1163 let mut mgr = manager(&case);
1164 mgr.g2h_consumer = Some(case.g2h_consumer());
1165
1166 let Err(error) = mgr.read_h2g_result_from_g2h(1) else {
1167 panic!("expected missing G2H result");
1168 };
1169
1170 assert!(
1171 error
1172 .to_string()
1173 .contains("G2H has no guest function result")
1174 );
1175 assert!(error.is_poison_error());
1176 assert!(matches!(error, HyperlightError::TransportError(_)));
1177 }
1178
1179 #[test]
1180 fn writes_dense_h2g_request_and_reserves_control_buffer() {
1181 let case = TestCase::new();
1182 let mut mgr = manager(&case);
1183 let external_len = H2G_BUFFER_SIZE * 2;
1184 let buffers: Vec<_> = (0..4).map(|index| case.h2g_desc(index).addr).collect();
1185
1186 let cid = mgr
1187 .write_guest_function_call(&h2g_call(external_len))
1188 .unwrap();
1189
1190 let used = mgr.h2g_consumer.as_ref().unwrap().avail_cursor().head();
1191 let wire: Vec<u8> = (0..used)
1192 .flat_map(|index| case.h2g_buffer(index, buffers[index as usize]))
1193 .collect();
1194
1195 let header = MsgHeader::from_bytes(&wire[..MsgHeader::SIZE]).unwrap();
1196 assert_eq!(header.kind, MsgKind::Request);
1197 assert_eq!(header.cid, cid);
1198 assert_eq!(header.payload_len as usize, wire.len() - MsgHeader::SIZE);
1199
1200 let control =
1201 size_prefix_payload_len(&wire[MsgHeader::SIZE..MsgHeader::SIZE + SIZE_PREFIX_LEN])
1202 .unwrap();
1203
1204 let control_len = size_prefixed_len(control).unwrap();
1205 let external = &wire[MsgHeader::SIZE + control_len..];
1206 assert_eq!(external, vec![0xa5; external_len]);
1207
1208 let cursor = mgr.h2g_consumer.as_ref().unwrap().avail_cursor();
1209 let error = mgr.write_guest_function_call(&h2g_call(1)).unwrap_err();
1210
1211 assert!(error.to_string().contains("H2G capacity"), "{error:#}");
1212 assert_eq!(mgr.h2g_consumer.as_ref().unwrap().avail_cursor(), cursor);
1213 assert_eq!(mgr.write_guest_function_call(&h2g_call(0)).unwrap(), 2);
1214 }
1215
1216 #[test]
1217 fn writes_header_only_snapshot_checkpoint() {
1218 let case = TestCase::new();
1219 let mut mgr = manager(&case);
1220 let buffer = case.h2g_desc(0).addr;
1221
1222 let mbx = mgr.layout.get_transport_arena().mbx_offset();
1223 let cp = MailboxValue::CheckpointComplete.as_bytes();
1224
1225 mgr.scratch_mem.copy_from_slice(cp, mbx).unwrap();
1226 mgr.begin_snapshot_checkpoint().unwrap();
1227
1228 let wire = case.h2g_buffer(0, buffer);
1229 let header = MsgHeader::from_bytes(&wire).unwrap();
1230
1231 assert_eq!(wire.len(), MsgHeader::SIZE);
1232 assert_eq!(header.kind, MsgKind::SnapshotCheckpoint);
1233 assert_eq!(header.cid, 0);
1234 assert_eq!(header.payload_len, 0);
1235 assert_eq!(mgr.next_guest_cid, 1);
1236
1237 assert_eq!(
1238 MailboxValue::try_from(mgr.scratch_mem.read::<u64>(mbx).unwrap()).unwrap(),
1239 MailboxValue::CheckpointPending
1240 );
1241 }
1242
1243 #[test]
1244 fn incomplete_snapshot_checkpoint_is_fatal() {
1245 let case = TestCase::new();
1246 let mut mgr = manager(&case);
1247 mgr.g2h_consumer = Some(case.g2h_consumer());
1248 mgr.begin_snapshot_checkpoint().unwrap();
1249
1250 let error = mgr.finish_snapshot_checkpoint().unwrap_err();
1251
1252 assert!(matches!(error, HyperlightError::TransportError(_)));
1253 assert!(error.is_poison_error());
1254 }
1255
1256 #[test]
1257 fn reads_completed_snapshot_checkpoint_status() {
1258 let case = TestCase::new();
1259 let mut mgr = manager(&case);
1260 mgr.g2h_consumer = Some(case.g2h_consumer());
1261 mgr.begin_snapshot_checkpoint().unwrap();
1262 let mbx = mgr.layout.get_transport_arena().mbx_offset();
1263 mgr.scratch_mem
1264 .copy_from_slice(MailboxValue::CheckpointComplete.as_bytes(), mbx)
1265 .unwrap();
1266
1267 mgr.finish_snapshot_checkpoint().unwrap();
1268 let consumer = mgr.h2g_consumer.as_ref().unwrap();
1269 assert_eq!(consumer.avail_cursor().head(), 0);
1270 assert_eq!(consumer.used_cursor().head(), 0);
1271 }
1272
1273 #[test]
1274 fn invalid_checkpoint_status_is_fatal() {
1275 for status in [2, 3, 0xa000, u64::MAX - 1, u64::MAX] {
1276 let queue = TestCase::new();
1277 let mut mgr = manager(&queue);
1278 mgr.g2h_consumer = Some(queue.g2h_consumer());
1279 let mbx = mgr.layout.get_transport_arena().mbx_offset();
1280 mgr.scratch_mem.write::<u64>(mbx, status).unwrap();
1281 let error = mgr.finish_snapshot_checkpoint().unwrap_err();
1282
1283 assert!(matches!(error, HyperlightError::TransportError(_)));
1284 assert!(error.is_poison_error(), "{error}");
1285 }
1286 }
1287
1288 #[test]
1289 fn guest_cid_wraps_without_zero() {
1290 let case = TestCase::new();
1291 let mut mgr = manager(&case);
1292 mgr.next_guest_cid = u32::MAX;
1293
1294 assert_eq!(
1295 mgr.write_guest_function_call(&h2g_call(0)).unwrap(),
1296 u32::MAX
1297 );
1298 assert_eq!(mgr.write_guest_function_call(&h2g_call(0)).unwrap(), 1);
1299 }
1300
1301 #[test]
1302 fn cloning_preserves_guest_cids_without_consumers() {
1303 let case = TestCase::new();
1304 let mut mgr = manager(&case);
1305 mgr.g2h_consumer = Some(case.g2h_consumer());
1306 mgr.write_guest_function_call(&h2g_call(0)).unwrap();
1307
1308 let cloned = mgr.clone();
1309
1310 assert!(cloned.g2h_consumer.is_none());
1311 assert!(cloned.h2g_consumer.is_none());
1312 assert_eq!(cloned.next_guest_cid, 2);
1313 assert!(mgr.g2h_consumer.is_some());
1314 assert!(mgr.h2g_consumer.is_some());
1315 }
1316
1317 #[test]
1318 fn restoring_transport_preserves_guest_cids() {
1319 let case = TestCase::new();
1320 let mut mgr = manager(&case);
1321 mgr.g2h_consumer = Some(case.g2h_consumer());
1322 let captured = VirtqSnapshot::capture(&mgr.layout, &mgr.scratch_mem).unwrap();
1323 assert_eq!(mgr.write_guest_function_call(&h2g_call(0)).unwrap(), 1);
1324
1325 mgr.g2h_consumer = None;
1326 mgr.h2g_consumer = None;
1327 mgr.restore_virtq(&captured).unwrap();
1328
1329 assert_eq!(mgr.write_guest_function_call(&h2g_call(0)).unwrap(), 2);
1330 }
1331
1332 #[cfg(target_arch = "x86_64")]
1335 fn verify_page_tables(name: &str, config: SandboxConfiguration) {
1336 let path = simple_guest_as_pathbuf();
1337 let snapshot = Snapshot::from_env(GuestBinary::FilePath(path), config)
1338 .unwrap_or_else(|e| panic!("{}: failed to create snapshot: {}", name, e));
1339
1340 assert!(
1342 unsafe { hyperlight_common::vmem::virt_to_phys(&snapshot, 0, 1) }
1343 .next()
1344 .is_none(),
1345 "{}: NULL page (0x0) should NOT be mapped",
1346 name
1347 );
1348 }
1349
1350 #[test]
1351 #[cfg(target_arch = "x86_64")]
1352 fn test_page_tables_for_various_configurations() {
1353 let test_cases: [(&str, SandboxConfiguration); 4] = [
1354 ("default", { SandboxConfiguration::default() }),
1355 ("small (8MB heap)", {
1356 let mut cfg = SandboxConfiguration::default();
1357 cfg.set_heap_size(SMALL_HEAP_SIZE);
1358 cfg
1359 }),
1360 ("medium (64MB heap)", {
1361 let mut cfg = SandboxConfiguration::default();
1362 cfg.set_heap_size(MEDIUM_HEAP_SIZE);
1363 cfg
1364 }),
1365 ("large (256MB heap)", {
1366 let mut cfg = SandboxConfiguration::default();
1367 cfg.set_heap_size(LARGE_HEAP_SIZE);
1368 cfg.set_scratch_size(0x100000);
1369 cfg
1370 }),
1371 ];
1372
1373 for (name, config) in test_cases {
1374 verify_page_tables(name, config);
1375 }
1376 }
1377
1378 #[test]
1379 #[cfg(target_arch = "x86_64")]
1380 fn build_creates_virtq_consumers_before_initialization() {
1381 let path = simple_guest_as_pathbuf();
1382 let bin = GuestBinary::FilePath(path);
1383 let config = SandboxConfiguration::default();
1384
1385 let snapshot = Snapshot::from_env(bin, config).unwrap();
1386
1387 let mgr = SandboxMemoryManager::from_snapshot(&snapshot).unwrap();
1388 let (mgr, _) = mgr.build().unwrap();
1389
1390 assert!(mgr.g2h_consumer.is_some());
1391 assert!(mgr.h2g_consumer.is_some());
1392 }
1393}