1use std::fmt::Debug;
5use std::sync::OnceLock;
6
7use tracing::{Span, instrument};
8
9#[cfg(gdb)]
10use crate::hypervisor::gdb::DebugError;
11use crate::hypervisor::regs::{
12 CommonDebugRegs, CommonFpu, CommonRegisters, CommonSpecialRegisters,
13};
14#[cfg(all(target_arch = "x86_64", any(mshv3, target_os = "windows")))]
15use crate::hypervisor::regs::{
16 MSR_MTRR_CAP, filterless_core_reset_candidates, hyperv_mtrr_reset_indices,
17};
18#[cfg(target_arch = "x86_64")]
19use crate::hypervisor::regs::{MsrEntry, is_resettable_msr};
20use crate::mem::memory_region::MemoryRegion;
21#[cfg(feature = "trace_guest")]
22use crate::sandbox::trace::TraceContext as SandboxTraceContext;
23
24#[cfg(hvf)]
26pub(crate) mod hvf;
27#[cfg(kvm)]
29pub(crate) mod kvm;
30#[cfg(mshv3)]
32pub(crate) mod mshv;
33#[cfg(target_os = "windows")]
35pub(crate) mod whp;
36
37#[cfg(feature = "hw-interrupts")]
39pub(crate) mod x86_64;
40
41static AVAILABLE_HYPERVISOR: OnceLock<Option<HypervisorType>> = OnceLock::new();
42
43pub fn get_available_hypervisor() -> &'static Option<HypervisorType> {
45 AVAILABLE_HYPERVISOR.get_or_init(|| {
46 cfg_if::cfg_if! {
47 if #[cfg(all(kvm, mshv3))] {
48 if mshv::is_hypervisor_present() {
52 Some(HypervisorType::Mshv)
53 } else if kvm::is_hypervisor_present() {
54 Some(HypervisorType::Kvm)
55 } else {
56 None
57 }
58 } else if #[cfg(kvm)] {
59 if kvm::is_hypervisor_present() {
60 Some(HypervisorType::Kvm)
61 } else {
62 None
63 }
64 } else if #[cfg(mshv3)] {
65 if mshv::is_hypervisor_present() {
66 Some(HypervisorType::Mshv)
67 } else {
68 None
69 }
70 } else if #[cfg(target_os = "windows")] {
71 if whp::is_hypervisor_present() {
72 Some(HypervisorType::Whp)
73 } else {
74 None
75 }
76 } else if #[cfg(hvf)] {
77 if hvf::is_hypervisor_present() {
78 Some(HypervisorType::Hvf)
79 } else {
80 None
81 }
82 } else {
83 None
84 }
85 }
86 })
87}
88
89#[instrument(skip_all, parent = Span::current())]
92pub fn is_hypervisor_present() -> bool {
93 get_available_hypervisor().is_some()
94}
95
96#[derive(PartialEq, Eq, Debug, Copy, Clone)]
98pub(crate) enum HypervisorType {
99 #[cfg(kvm)]
100 Kvm,
101
102 #[cfg(mshv3)]
103 Mshv,
104
105 #[cfg(target_os = "windows")]
106 Whp,
107
108 #[cfg(hvf)]
109 Hvf,
110}
111
112#[cfg(all(target_arch = "x86_64", target_os = "windows"))]
116pub(crate) const XSAVE_MIN_SIZE: usize = 576;
117
118#[cfg(all(any(kvm, mshv3), test, not(target_arch = "aarch64")))]
121pub(crate) const XSAVE_BUFFER_SIZE: usize = 4096;
122
123#[cfg(target_arch = "x86_64")]
125pub(crate) const XCR0_RESET: u64 = 1;
126
127#[cfg(not(any(kvm, mshv3, target_os = "windows", target_arch = "aarch64")))]
129compile_error!(
130 "No hypervisor type is available for the current platform. Please enable either the `kvm` or `mshv3` cargo feature."
131);
132
133#[cfg_attr(target_os = "macos", allow(unused))]
135pub(crate) enum VmExit {
136 #[cfg(gdb)]
138 Debug {
139 #[cfg(target_arch = "x86_64")]
140 dr6: u64,
141 #[cfg(target_arch = "x86_64")]
142 exception: u32,
143 },
144 Halt(),
146 IoOut(u16, Vec<u8>),
148 MmioRead(u64),
150 MmioWrite(u64),
152 Cancelled(),
154 Unknown(String),
156 #[cfg_attr(
158 any(target_os = "windows", feature = "hw-interrupts"),
159 expect(
160 dead_code,
161 reason = "Retry() is never constructed on Windows or with hw-interrupts (EAGAIN causes continue instead)"
162 )
163 )]
164 Retry(),
165}
166
167#[derive(Debug, thiserror::Error)]
169pub enum VmError {
170 #[error("Failed to create vm: {0}")]
171 CreateVm(#[from] CreateVmError),
172 #[cfg(gdb)]
173 #[error("Debug operation failed: {0}")]
174 Debug(#[from] DebugError),
175 #[error("Map memory operation failed: {0}")]
176 MapMemory(#[from] MapMemoryError),
177 #[error("Register operation failed: {0}")]
178 Register(#[from] RegisterError),
179 #[error("Failed to run vcpu: {0}")]
180 RunVcpu(#[from] RunVcpuError),
181 #[error("Unmap memory operation failed: {0}")]
182 UnmapMemory(#[from] UnmapMemoryError),
183}
184
185#[derive(Debug, Clone, thiserror::Error)]
187pub enum CreateVmError {
188 #[error("VCPU creation failed: {0}")]
189 CreateVcpuFd(HypervisorError),
190 #[error("VM creation failed: {0}")]
191 CreateVmFd(HypervisorError),
192 #[error("Hypervisor is not available: {0}")]
193 HypervisorNotAvailable(HypervisorError),
194 #[error("Initialize VM failed: {0}")]
195 InitializeVm(HypervisorError),
196 #[cfg(all(kvm, target_arch = "x86_64"))]
197 #[error("KVM MSR filtering requires KVM_CAP_X86_MSR_FILTER")]
198 MsrFilterNotSupported,
199 #[cfg(target_arch = "x86_64")]
200 #[error("MSR {msr:#x} cannot be declared as a guest MSR: {reason}")]
201 MsrNotDeclarable { msr: u32, reason: String },
202 #[cfg(target_arch = "x86_64")]
203 #[error("Failed to read IA32_MTRRCAP: {0}")]
204 GetMtrrCap(RegisterError),
205 #[cfg(target_arch = "x86_64")]
206 #[error("Guest-visible MTRRs cannot be reset: {0}")]
207 RequiredMtrrsNotResettable(RegisterError),
208 #[cfg(all(target_arch = "x86_64", any(mshv3, target_os = "windows")))]
209 #[error("Core reset MSR {msr:#x} is readable but not writable on this host")]
210 MsrNotResettable { msr: u32 },
211 #[cfg(target_arch = "x86_64")]
212 #[error("Guest exposes {advertised} variable MTRR pairs, expected at most {maximum}")]
213 UnexpectedVariableMtrrCount { advertised: u8, maximum: u8 },
214 #[cfg(all(kvm, target_arch = "x86_64"))]
215 #[error("Too many guest MSR filter ranges: {0}. Maximum is 16")]
216 TooManyMsrRanges(usize),
217 #[cfg(target_os = "windows")]
218 #[error("Get Partition Property failed: {0}")]
219 GetPartitionProperty(HypervisorError),
220 #[cfg(target_os = "windows")]
221 #[error("WHP exposes {advertised} processor feature banks, expected {expected}")]
222 UnexpectedProcessorFeatureBankCount { advertised: u32, expected: u32 },
223 #[error("Set Partition Property failed: {0}")]
224 SetPartitionProperty(HypervisorError),
225 #[cfg(target_os = "windows")]
226 #[error("Surrogate process creation failed: {0}")]
227 SurrogateProcess(String),
228 #[cfg(all(target_os = "windows", target_arch = "aarch64"))]
229 #[error("Timed out waiting for an ARM64 WHP partition slot")]
230 WhpPartitionLimit,
231}
232
233#[derive(Debug, thiserror::Error)]
235pub enum RunVcpuError {
236 #[error("Failed to decode message type: {0}")]
237 DecodeIOMessage(u32),
238 #[cfg(gdb)]
239 #[error("Failed to get DR6 debug register: {0}")]
240 GetDr6(HypervisorError),
241 #[error("Increment RIP failed: {0}")]
242 IncrementRip(HypervisorError),
243 #[error("Parse GPA access info failed")]
244 ParseGpaAccessInfo,
245 #[cfg(target_arch = "aarch64")]
246 #[error("Flush MMIO pending state failed: {0}")]
247 FlushMmioPending(String),
248 #[cfg(hvf)]
249 #[error("HVF sync error: {0}")]
250 HvfSync(HvfSyncError),
251 #[error("Unknown error: {0}")]
252 Unknown(HypervisorError),
253}
254
255#[derive(Debug, Clone, thiserror::Error)]
257pub enum RegisterError {
258 #[error("Failed to get registers: {0}")]
259 GetRegs(HypervisorError),
260 #[error("Failed to set registers: {0}")]
261 SetRegs(HypervisorError),
262 #[error("Failed to get FPU registers: {0}")]
263 GetFpu(HypervisorError),
264 #[error("Failed to set FPU registers: {0}")]
265 SetFpu(HypervisorError),
266 #[error("Failed to get special registers: {0}")]
267 GetSregs(HypervisorError),
268 #[error("Failed to set special registers: {0}")]
269 SetSregs(HypervisorError),
270 #[cfg(target_arch = "x86_64")]
271 #[error("Snapshot APIC_BASE {value:#x} enables unsupported x2APIC mode")]
272 InvalidSnapshotApicBase {
273 value: u64,
275 },
276 #[error("Failed to get debug registers: {0}")]
277 GetDebugRegs(HypervisorError),
278 #[error("Failed to set debug registers: {0}")]
279 SetDebugRegs(HypervisorError),
280 #[error("Failed to get xsave: {0}")]
281 GetXsave(HypervisorError),
282 #[error("Failed to set xsave: {0}")]
283 SetXsave(HypervisorError),
284 #[cfg(target_arch = "x86_64")]
285 #[error("Failed to get XCRs: {0}")]
286 GetXcrs(HypervisorError),
287 #[cfg(target_arch = "x86_64")]
288 #[error("Failed to set XCRs: {0}")]
289 SetXcrs(HypervisorError),
290 #[cfg(target_arch = "x86_64")]
291 #[error("Hypervisor did not return XCR0")]
292 MissingXcr0,
293 #[error("Xsave size mismatch: expected {expected} bytes, got {actual}")]
294 XsaveSizeMismatch {
295 expected: u32,
297 actual: u32,
299 },
300 #[error("Invalid xsave alignment")]
301 InvalidXsaveAlignment,
302 #[cfg(target_arch = "x86_64")]
303 #[error("MSR operation not supported on this hypervisor")]
304 MsrsUnsupported,
305 #[cfg(target_arch = "x86_64")]
306 #[error("Failed to build MSR list: {0}")]
307 MsrBuild(String),
308 #[cfg(target_arch = "x86_64")]
309 #[error("Failed to get MSRs: {0}")]
310 GetMsrs(HypervisorError),
311 #[cfg(target_arch = "x86_64")]
312 #[error("Failed to set MSRs: {0}")]
313 SetMsrs(HypervisorError),
314 #[cfg(target_arch = "x86_64")]
315 #[error("Failed to set batched registers: {0}")]
316 SetBatchedRegisters(HypervisorError),
317 #[cfg(target_arch = "x86_64")]
318 #[error("Batched register writes are not supported")]
319 BatchedSetRegistersUnsupported,
320 #[cfg(target_arch = "x86_64")]
321 #[error("Snapshot MSR index {index:#x} is not in this VM's reset set")]
322 InvalidSnapshotMsrIndex {
323 index: u32,
325 },
326 #[cfg(all(kvm, target_arch = "x86_64"))]
327 #[error("MSR batch short count: expected {expected}, applied {actual}")]
328 MsrShortCount {
329 expected: usize,
331 actual: usize,
333 },
334 #[cfg(target_os = "windows")]
335 #[error("Failed to get xsave size: {0}")]
336 GetXsaveSize(#[from] HypervisorError),
337 #[cfg(target_os = "windows")]
338 #[error("Failed to convert WHP registers: {0}")]
339 ConversionFailed(String),
340}
341
342#[derive(Debug, thiserror::Error)]
343pub enum ResetVcpuError {
344 #[error("Single-operation vcpu reset not supported on this hypervisor")]
345 NotSupported,
346 #[error("Hypervisor operation failed: {0}")]
347 Hypervisor(HypervisorError),
348 #[error("Register operation failed: {0}")]
349 Register(#[from] RegisterError),
350 #[error("Operation failed: {0}")]
351 Unknown(String),
352}
353
354#[derive(Debug, thiserror::Error)]
356pub enum MapMemoryError {
357 #[cfg(target_os = "windows")]
358 #[error("Address conversion failed: {0}")]
359 AddressConversion(std::num::TryFromIntError),
360 #[error("Hypervisor error: {0}")]
361 Hypervisor(HypervisorError),
362 #[cfg(target_os = "windows")]
363 #[error("Invalid memory region flags: {0}")]
364 InvalidFlags(String),
365 #[cfg(target_os = "windows")]
366 #[error("Failed to load API '{api_name}': {source}")]
367 LoadApi {
368 api_name: &'static str,
369 source: windows_result::Error,
370 },
371 #[cfg(target_os = "windows")]
372 #[error("Operation not supported: {0}")]
373 NotSupported(String),
374 #[cfg(target_os = "windows")]
375 #[error("Surrogate process creation failed: {0}")]
376 SurrogateProcess(String),
377}
378
379#[derive(Debug, thiserror::Error)]
381pub enum UnmapMemoryError {
382 #[error("Hypervisor error: {0}")]
383 Hypervisor(HypervisorError),
384}
385
386#[derive(Debug, Clone, thiserror::Error)]
388pub enum HypervisorError {
389 #[cfg(test)]
390 #[error("Injected hypervisor error")]
391 Injected,
392 #[cfg(kvm)]
393 #[error("KVM error: {0}")]
394 KvmError(#[from] kvm_ioctls::Error),
395 #[cfg(mshv3)]
396 #[error("MSHV error: {0}")]
397 MshvError(#[from] mshv_ioctls::MshvError),
398 #[cfg(target_os = "windows")]
399 #[error("Windows error: {0}")]
400 WindowsError(#[from] windows_result::Error),
401 #[cfg(hvf)]
402 #[error("HVF error: {0}")]
403 HvfError(hvf::bindings::hv_return_t),
404}
405
406#[cfg(hvf)]
408#[derive(Debug, thiserror::Error)]
409pub enum MemorySpaceInstallError {
410 #[error("Failed to update VM/VCPU state: {0}")]
411 Hypervisor(#[from] HypervisorError),
412 #[error("Unexpected VCPU exit: {0:?}")]
413 UnexpectedExit(hvf::bindings::hv_vcpu_exit_t),
414 #[error("Failed to allocate ReadonlySharedMemory: {0}")]
415 SharedMemoryCreation(#[from] crate::mem::shared_mem::SharedMemoryError),
416}
417#[cfg(hvf)]
418#[derive(Debug, thiserror::Error)]
419pub enum HvfSyncError {
420 #[error("Error creating VCPU: {0}")]
421 CreateVcpu(HypervisorError),
422 #[error("Error resetting VCPU: {0}")]
423 ResetVcpu(HypervisorError),
424 #[error("Error reading/writing registers: {0}")]
425 Register(#[from] RegisterError),
426 #[error("Error updating memory space: {0}")]
427 MemorySpace(#[from] MemorySpaceInstallError),
428 #[error("Invariant violation: vcpu in unexpected sync state: {0}")]
429 SyncInvariant(String),
430}
431
432pub(crate) trait VirtualMachine: Debug + Send {
435 unsafe fn map_memory(
444 &mut self,
445 region: (u32, &MemoryRegion),
446 ) -> std::result::Result<(), MapMemoryError>;
447
448 fn unmap_memory(
450 &mut self,
451 region: (u32, &MemoryRegion),
452 ) -> std::result::Result<(), UnmapMemoryError>;
453
454 fn run_vcpu(
458 &mut self,
459 #[cfg(feature = "trace_guest")] tc: &mut SandboxTraceContext,
460 ) -> std::result::Result<VmExit, RunVcpuError>;
461
462 #[allow(dead_code)]
464 fn regs(&self) -> std::result::Result<CommonRegisters, RegisterError>;
465 fn set_regs(&mut self, regs: &CommonRegisters) -> std::result::Result<(), RegisterError>;
467 #[allow(dead_code)]
469 fn fpu(&self) -> std::result::Result<CommonFpu, RegisterError>;
470 fn set_fpu(&mut self, fpu: &CommonFpu) -> std::result::Result<(), RegisterError>;
472 #[allow(dead_code)]
474 fn sregs(&self) -> std::result::Result<CommonSpecialRegisters, RegisterError>;
475 fn set_sregs(
477 &mut self,
478 sregs: &CommonSpecialRegisters,
479 ) -> std::result::Result<(), RegisterError>;
480 #[allow(dead_code)]
482 fn debug_regs(&self) -> std::result::Result<CommonDebugRegs, RegisterError>;
483 #[allow(dead_code)]
485 fn set_debug_regs(&self, drs: &CommonDebugRegs) -> std::result::Result<(), RegisterError>;
486
487 #[cfg(target_arch = "x86_64")]
489 fn msrs(&self, indices: &[u32]) -> std::result::Result<Vec<MsrEntry>, RegisterError>;
490 #[cfg(target_arch = "x86_64")]
492 fn set_msrs(&self, msrs: &[MsrEntry]) -> std::result::Result<(), RegisterError>;
493 #[cfg(target_arch = "x86_64")]
495 fn msr_reset_indices(&self, guest_msrs: &[u32])
496 -> std::result::Result<Vec<u32>, CreateVmError>;
497
498 #[allow(dead_code)]
500 #[cfg(not(target_arch = "aarch64"))]
501 fn xsave(&self) -> std::result::Result<Vec<u8>, RegisterError>;
502 #[cfg(not(target_arch = "aarch64"))]
504 fn reset_xsave(&self) -> std::result::Result<(), RegisterError>;
505 #[cfg(test)]
507 #[cfg(not(target_arch = "aarch64"))]
508 fn set_xsave(&self, xsave: &[u32]) -> std::result::Result<(), RegisterError>;
509
510 #[cfg(all(test, target_arch = "x86_64"))]
511 fn xcr0(&self) -> std::result::Result<u64, RegisterError>;
512 #[cfg(target_arch = "x86_64")]
513 fn set_xcr0(&self, value: u64) -> std::result::Result<(), RegisterError>;
514
515 #[cfg(target_arch = "x86_64")]
516 fn can_batch_registers(&self) -> bool {
517 false
518 }
519 #[cfg(target_arch = "x86_64")]
520 fn set_batched_registers(
521 &mut self,
522 _regs: &CommonRegisters,
523 _debug_regs: &CommonDebugRegs,
524 _sregs: &CommonSpecialRegisters,
525 _xcr0: u64,
526 _msrs: &[MsrEntry],
527 ) -> std::result::Result<(), RegisterError> {
528 Err(RegisterError::BatchedSetRegistersUnsupported)
529 }
530
531 #[cfg(target_arch = "aarch64")]
533 fn can_reset_vcpu(&self) -> bool {
534 false
535 }
536 #[cfg(target_arch = "aarch64")]
537 fn reset_vcpu(&mut self) -> std::result::Result<(), ResetVcpuError> {
538 Err(ResetVcpuError::NotSupported)
539 }
540 #[cfg(target_os = "windows")]
542 fn partition_handle(&self) -> windows::Win32::System::Hypervisor::WHV_PARTITION_HANDLE;
543}
544
545#[cfg(target_arch = "x86_64")]
547enum MsrProbe {
548 Unreadable,
550 Unwritable,
552}
553
554#[cfg(target_arch = "x86_64")]
557fn probe_resettable(vm: &dyn VirtualMachine, msr: u32) -> Result<(), MsrProbe> {
558 let captured = vm.msrs(&[msr]).map_err(|_| MsrProbe::Unreadable)?;
559 vm.set_msrs(&captured).map_err(|_| MsrProbe::Unwritable)
560}
561
562#[cfg(target_arch = "x86_64")]
566pub(crate) fn validate_guest_msrs(
567 vm: &dyn VirtualMachine,
568 guest_msrs: &[u32],
569) -> std::result::Result<(), CreateVmError> {
570 for &msr in guest_msrs {
571 if !is_resettable_msr(msr) {
572 return Err(CreateVmError::MsrNotDeclarable {
573 msr,
574 reason: "MSR is not a resettable MSR".to_string(),
575 });
576 }
577 probe_resettable(vm, msr).map_err(|probe| CreateVmError::MsrNotDeclarable {
579 msr,
580 reason: match probe {
581 MsrProbe::Unreadable => "MSR cannot be read on this host",
582 MsrProbe::Unwritable => "MSR cannot be written on this host",
583 }
584 .to_string(),
585 })?;
586 }
587 Ok(())
588}
589
590#[cfg(all(target_arch = "x86_64", any(mshv3, target_os = "windows")))]
592pub(crate) fn mtrr_reset_indices(
593 vm: &dyn VirtualMachine,
594) -> std::result::Result<Vec<u32>, CreateVmError> {
595 let mtrr_cap = vm
596 .msrs(&[MSR_MTRR_CAP])
597 .map_err(CreateVmError::GetMtrrCap)?[0]
598 .value;
599 let indices = hyperv_mtrr_reset_indices(mtrr_cap)?;
600 vm.msrs(&indices)
601 .map_err(CreateVmError::RequiredMtrrsNotResettable)?;
602 Ok(indices)
603}
604
605#[cfg(all(target_arch = "x86_64", any(mshv3, target_os = "windows")))]
607pub(crate) fn hyperv_msr_reset_indices(
608 vm: &dyn VirtualMachine,
609 guest_msrs: &[u32],
610) -> std::result::Result<Vec<u32>, CreateVmError> {
611 static DEFAULT: OnceLock<Vec<u32>> = OnceLock::new();
614
615 if guest_msrs.is_empty()
617 && let Some(indices) = DEFAULT.get()
618 {
619 return Ok(indices.clone());
620 }
621
622 validate_guest_msrs(vm, guest_msrs)?;
623 let mut indices = filterless_core_reset_candidates()
624 .filter_map(|index| match probe_resettable(vm, index) {
625 Ok(()) => Some(Ok(index)),
627 Err(MsrProbe::Unreadable) => None,
629 Err(MsrProbe::Unwritable) => Some(Err(CreateVmError::MsrNotResettable { msr: index })),
631 })
632 .collect::<Result<Vec<u32>, _>>()?;
633 indices.extend(mtrr_reset_indices(vm)?);
637 indices.extend(guest_msrs.iter().copied());
639 indices.sort_unstable();
640 indices.dedup();
641 if guest_msrs.is_empty() {
643 let _ = DEFAULT.set(indices.clone());
645 }
646 Ok(indices)
647}
648
649#[cfg(test)]
650mod tests {
651
652 #[test]
653 #[cfg(target_os = "linux")]
655 fn is_hypervisor_present() {
656 use std::path::Path;
657
658 cfg_if::cfg_if! {
659 if #[cfg(all(kvm, mshv3))] {
660 assert_eq!(Path::new("/dev/kvm").exists() || Path::new("/dev/mshv").exists(), super::is_hypervisor_present());
661 } else if #[cfg(kvm)] {
662 assert_eq!(Path::new("/dev/kvm").exists(), super::is_hypervisor_present());
663 } else if #[cfg(mshv3)] {
664 assert_eq!(Path::new("/dev/mshv").exists(), super::is_hypervisor_present());
665 } else {
666 assert!(!super::is_hypervisor_present());
667 }
668 }
669 }
670}