use alloc::{
boxed::Box,
collections::{BTreeMap, BTreeSet},
format,
string::String,
sync::Arc,
vec::Vec,
};
use core::{
alloc::Layout,
sync::atomic::{AtomicBool, AtomicUsize, Ordering},
};
use ax_cpumask::CpuMask;
use ax_kspin::SpinNoIrq as Mutex;
use ax_memory_addr::align_up_4k;
use axaddrspace::{AddrSpace, NestedPageTableOps};
use axdevice::{
DeviceRuntime, FwCfgPayloadConfig, FwCfgPlatformConfig, RuntimeAccessPorts, StopAccessPort,
TimerAccessPort, WakeAccessPort,
};
use axdevice_base::{AccessWidth, DeviceAccess, DeviceId, DeviceResult, DmaGrant};
use axvm_types::{
GuestPhysAddr, HostPhysAddr, HostVirtAddr, MappingFlags, NestedPagingConfig, VmVcpuState,
};
use crate::{
AxVmError, AxVmResult,
arch::ArchNestedPageTable,
ax_err, ax_err_type,
boot::{GuestBootDescription, GuestFdtBuilder},
config::{AxVMConfig, PhysCpuList, VMInterruptMode},
host::paging::virt_to_phys,
irq::{InterruptFabric, model::PendingVcpuInterrupt},
layout::VmAddressLayout,
lifecycle::{Machine, StopReason, VmStatus},
runtime::VcpuIrqDispatcher,
vcpu::AxVCpu,
};
pub(crate) mod boot;
pub(crate) mod memory;
pub(crate) mod prepare;
pub use memory::PreparedMemoryLayout;
const VM_ASPACE_BASE: usize = 0x0;
const VM_ASPACE_SIZE: usize = 0x7fff_ffff_f000;
type VCpu = AxVCpu<crate::arch::ArchVCpu>;
pub(crate) type AxVCpuRef<A = crate::arch::ArchVCpu> = Arc<AxVCpu<A>>;
pub type AxVMRef = Arc<AxVM>;
struct VmDmaAccess<'a> {
vm: &'a AxVM,
}
impl DeviceAccess for VmDmaAccess<'_> {
fn device_id(&self) -> DeviceId {
DeviceId::new(0)
}
fn read_guest_memory(
&mut self,
_grant: &DmaGrant,
addr: GuestPhysAddr,
data: &mut [u8],
) -> DeviceResult {
self.vm
.read_from_guest(addr, data)
.map_err(|error| axdevice_base::DeviceError::Backend {
operation: "read guest memory for DMA",
detail: alloc::format!("{error}"),
})
}
fn write_guest_memory(
&mut self,
_grant: &DmaGrant,
addr: GuestPhysAddr,
data: &[u8],
) -> DeviceResult {
self.vm
.write_to_guest(addr, data)
.map_err(|error| axdevice_base::DeviceError::Backend {
operation: "write guest memory for DMA",
detail: alloc::format!("{error}"),
})
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct VcpuSnapshot {
pub id: usize,
pub state: VmVcpuState,
pub phys_cpu_set: Option<usize>,
}
pub(crate) fn width_mask(width: AccessWidth) -> usize {
match width {
AccessWidth::Byte => 0xff,
AccessWidth::Word => 0xffff,
AccessWidth::Dword => 0xffff_ffff,
AccessWidth::Qword => usize::MAX,
}
}
pub(crate) fn sign_extend_value(value: usize, width: AccessWidth) -> usize {
match width {
AccessWidth::Byte => (value as i8) as isize as usize,
AccessWidth::Word => (value as i16) as isize as usize,
AccessWidth::Dword => (value as i32) as isize as usize,
AccessWidth::Qword => value,
}
}
fn write_guest_bytes_to_chunks(chunks: &mut [&mut [u8]], data: &[u8]) -> AxVmResult {
if data.is_empty() {
return Ok(());
}
let mut copied = 0;
for chunk in chunks {
let len = (data.len() - copied).min(chunk.len());
if len == 0 {
continue;
}
chunk[..len].copy_from_slice(&data[copied..copied + len]);
crate::clean_dcache_range((chunk.as_ptr() as usize).into(), len);
copied += len;
if copied == data.len() {
return Ok(());
}
}
ax_err!(
InvalidInput,
"Insufficient guest memory to write the requested buffer"
)
}
pub(crate) struct AxVMResources {
pub(crate) address_space: AddrSpace<ArchNestedPageTable>,
nested_paging: NestedPagingConfig,
memory_regions: Vec<VMMemoryRegion>,
config: AxVMConfig,
phys_cpu_ls: PhysCpuList,
vcpu_list: Option<Box<[AxVCpuRef]>>,
devices: Option<Arc<DeviceRuntime>>,
interrupt_fabric: Option<InterruptFabric>,
address_layout: Option<VmAddressLayout>,
boot_description: GuestBootDescription,
}
unsafe impl Send for AxVMResources {}
unsafe impl Sync for AxVMResources {}
#[allow(dead_code)]
#[derive(Clone, Copy, Debug)]
pub(crate) enum PendingInterrupt {
Normal(usize),
External { vector: usize, physical_irq: usize },
}
pub(crate) struct VmRuntimeHandle {
wait_queue: crate::WaitQueue,
vcpu_task_list: Mutex<BTreeMap<usize, crate::AxTaskRef>>,
cpu_on_start_acks: Mutex<BTreeMap<usize, Arc<crate::runtime::vcpus::CpuOnStartAck>>>,
cpu_off_exit_reservations: Mutex<BTreeSet<usize>>,
pending_interrupts: Mutex<BTreeMap<usize, Vec<PendingInterrupt>>>,
irq_dispatcher: crate::runtime::VcpuIrqDispatcher,
running_halting_vcpu_count: AtomicUsize,
deferred_reset_requested: AtomicBool,
}
pub(crate) fn dispatch_vcpu_interrupt_with(
enqueue: impl FnOnce() -> AxVmResult<usize>,
notify: impl FnOnce(),
send_ipi: impl FnOnce(usize),
) -> AxVmResult {
let pcpu_id = enqueue()?;
notify();
send_ipi(pcpu_id);
Ok(())
}
fn pulse_interrupt_with_snapshot(
snapshot: impl FnOnce() -> AxVmResult<InterruptFabric>,
irq_id: usize,
) -> AxVmResult {
snapshot()?.pulse(irq_id)
}
impl VmRuntimeHandle {
pub(crate) fn new() -> Self {
Self {
wait_queue: crate::WaitQueue::new(),
vcpu_task_list: Mutex::new(BTreeMap::new()),
cpu_on_start_acks: Mutex::new(BTreeMap::new()),
cpu_off_exit_reservations: Mutex::new(BTreeSet::new()),
pending_interrupts: Mutex::new(BTreeMap::new()),
irq_dispatcher: crate::runtime::VcpuIrqDispatcher::new(),
running_halting_vcpu_count: AtomicUsize::new(0),
deferred_reset_requested: AtomicBool::new(false),
}
}
#[allow(dead_code)]
pub(crate) fn has_vcpu_task(&self, vcpu_id: usize) -> bool {
self.vcpu_task_list.lock().contains_key(&vcpu_id)
}
pub(crate) fn add_vcpu_task(&self, vcpu_id: usize, vcpu_task: crate::AxTaskRef) -> AxVmResult {
let mut vcpu_task_list = self.vcpu_task_list.lock();
if vcpu_task_list.contains_key(&vcpu_id) {
return ax_err!(BadState, format!("vCPU {vcpu_id} task already exists"));
}
self.irq_dispatcher
.register_vcpu_task(vcpu_id, vcpu_task.clone());
vcpu_task_list.insert(vcpu_id, vcpu_task);
drop(vcpu_task_list);
self.pending_interrupts.lock().entry(vcpu_id).or_default();
Ok(())
}
pub(crate) fn remove_cpu_on_start_ack(
&self,
vcpu_id: usize,
) -> Option<Arc<crate::runtime::vcpus::CpuOnStartAck>> {
self.cpu_on_start_acks.lock().remove(&vcpu_id)
}
pub(crate) fn remove_vcpu_task(&self, vcpu_id: usize) -> Option<crate::AxTaskRef> {
self.pending_interrupts.lock().remove(&vcpu_id);
self.irq_dispatcher.unregister_vcpu_task(vcpu_id);
self.vcpu_task_list.lock().remove(&vcpu_id)
}
#[allow(dead_code)]
pub(crate) fn insert_cpu_on_start_ack(
&self,
vcpu_id: usize,
ack: Arc<crate::runtime::vcpus::CpuOnStartAck>,
) -> AxVmResult {
let mut acks = self.cpu_on_start_acks.lock();
if acks.contains_key(&vcpu_id) {
return ax_err!(
AlreadyExists,
format!("vCPU {vcpu_id} CPU_ON ack already exists")
);
}
acks.insert(vcpu_id, ack);
Ok(())
}
pub(crate) fn cpu_on_start_ack(
&self,
vcpu_id: usize,
) -> Option<Arc<crate::runtime::vcpus::CpuOnStartAck>> {
self.cpu_on_start_acks.lock().get(&vcpu_id).cloned()
}
pub(crate) fn queue_interrupt(&self, vcpu_id: usize, vector: usize) -> AxVmResult<usize> {
let task = self
.vcpu_task_list
.lock()
.get(&vcpu_id)
.cloned()
.ok_or_else(|| ax_err_type!(NotFound, format!("vCPU {vcpu_id} task not found")))?;
self.pending_interrupts
.lock()
.entry(vcpu_id)
.or_default()
.push(PendingInterrupt::Normal(vector));
Ok(task.cpu_id() as usize)
}
#[expect(
dead_code,
reason = "only the LoongArch IRQ backend queues physical interrupts"
)]
pub(crate) fn queue_external_interrupt(
&self,
vcpu_id: usize,
vector: usize,
physical_irq: usize,
) -> AxVmResult<usize> {
let task = self
.vcpu_task_list
.lock()
.get(&vcpu_id)
.cloned()
.ok_or_else(|| ax_err_type!(NotFound, format!("vCPU {vcpu_id} task not found")))?;
self.pending_interrupts
.lock()
.entry(vcpu_id)
.or_default()
.push(PendingInterrupt::External {
vector,
physical_irq,
});
Ok(task.cpu_id() as usize)
}
#[cfg_attr(
not(test),
expect(
dead_code,
reason = "architecture interrupt routers dispatch in later modules"
)
)]
pub(crate) fn dispatch_vcpu_interrupt(
&self,
vcpu_id: usize,
interrupt: PendingVcpuInterrupt,
) -> AxVmResult {
dispatch_vcpu_interrupt_with(
|| self.irq_dispatcher.enqueue(vcpu_id, interrupt),
|| self.notify_all(),
crate::host::task::send_ipi,
)
}
pub(crate) fn irq_dispatcher(&self) -> &VcpuIrqDispatcher {
&self.irq_dispatcher
}
pub(crate) fn drain_pending_interrupts(&self, vcpu_id: usize) -> Vec<PendingInterrupt> {
self.pending_interrupts
.lock()
.get_mut(&vcpu_id)
.map(core::mem::take)
.unwrap_or_default()
}
pub(crate) fn wait(&self) {
self.wait_queue.wait();
}
pub(crate) fn wait_until(&self, condition: impl Fn() -> bool) {
self.wait_queue.wait_until(condition);
}
pub(crate) fn notify_one(&self) {
self.wait_queue.notify_one(false);
}
pub(crate) fn notify_all(&self) {
self.wait_queue.notify_all(false);
}
pub(crate) fn mark_vcpu_running(&self) {
self.running_halting_vcpu_count
.fetch_add(1, Ordering::Relaxed);
}
pub(crate) fn publish_cpu_on_start_success(&self, ack: &crate::runtime::vcpus::CpuOnStartAck) {
self.mark_vcpu_running();
ack.complete(Ok(()));
}
pub(crate) fn mark_vcpu_exiting(&self) -> bool {
self.running_halting_vcpu_count
.try_update(Ordering::Relaxed, Ordering::Relaxed, |count| {
count.checked_sub(1)
})
== Ok(1)
}
pub(crate) fn request_deferred_reset(&self) -> bool {
!self.deferred_reset_requested.swap(true, Ordering::AcqRel)
}
pub(crate) fn take_deferred_reset_request(&self) -> bool {
self.deferred_reset_requested.swap(false, Ordering::AcqRel)
}
pub(crate) fn try_reserve_cpu_off(&self, vcpu_id: usize) -> bool {
let reserved = self
.running_halting_vcpu_count
.try_update(Ordering::AcqRel, Ordering::Acquire, |count| {
(count > 1).then_some(count - 1)
})
.is_ok();
if reserved {
self.cpu_off_exit_reservations.lock().insert(vcpu_id);
}
reserved
}
pub(crate) fn consume_cpu_off_reservation(&self, vcpu_id: usize) -> bool {
self.cpu_off_exit_reservations.lock().remove(&vcpu_id)
}
pub(crate) fn join_all_vcpu_tasks(&self, vm_id: usize) {
let current = crate::host::task::current_task();
let tasks: Vec<_> = self
.vcpu_task_list
.lock()
.values()
.filter(|task| !current.ptr_eq(task))
.cloned()
.collect();
let task_count = tasks.len();
info!("VM[{vm_id}] Joining {task_count} VCpu tasks...");
for (idx, task) in tasks.iter().enumerate() {
debug!(
"VM[{}] Joining VCpu task[{}]: {}",
vm_id,
idx,
task.id_name()
);
let exit_code = task.join();
debug!("VM[{vm_id}] VCpu task[{idx}] exited with code: {exit_code}");
}
info!("VM[{vm_id}] VCpu resources cleaned up, {task_count} VCpu tasks joined");
}
}
#[cfg(all(test, feature = "host-test"))]
mod runtime_handle_tests {
#[test]
fn runtime_cpu_on_success_publishes_online_count_before_ack() {
let runtime = VmRuntimeHandle::new();
let ack = crate::runtime::vcpus::CpuOnStartAck::new();
runtime.mark_vcpu_running();
assert!(ack.begin_startup());
runtime.publish_cpu_on_start_success(&ack);
assert!(ack.is_complete());
assert!(runtime.try_reserve_cpu_off(0));
}
#[test]
fn runtime_cpu_on_ack_rejects_duplicate_and_can_be_removed() {
let runtime = VmRuntimeHandle::new();
let first = Arc::new(crate::runtime::vcpus::CpuOnStartAck::new());
let second = Arc::new(crate::runtime::vcpus::CpuOnStartAck::new());
assert!(runtime.insert_cpu_on_start_ack(1, first.clone()).is_ok());
assert!(runtime.insert_cpu_on_start_ack(1, second).is_err());
let stored = runtime.cpu_on_start_ack(1).unwrap();
assert!(Arc::ptr_eq(&stored, &first));
assert!(runtime.remove_cpu_on_start_ack(1).is_some());
assert!(runtime.cpu_on_start_ack(1).is_none());
}
#[test]
fn runtime_deferred_reset_request_is_single_consumer() {
let runtime = VmRuntimeHandle::new();
assert!(!runtime.take_deferred_reset_request());
assert!(runtime.request_deferred_reset());
assert!(!runtime.request_deferred_reset());
assert!(runtime.take_deferred_reset_request());
assert!(!runtime.take_deferred_reset_request());
assert!(runtime.request_deferred_reset());
}
#[test]
fn runtime_cpu_off_reservation_rejects_second_parallel_last_vcpu() {
let runtime = VmRuntimeHandle::new();
runtime.mark_vcpu_running();
runtime.mark_vcpu_running();
assert!(runtime.try_reserve_cpu_off(0));
assert!(!runtime.try_reserve_cpu_off(1));
assert!(runtime.consume_cpu_off_reservation(0));
assert!(!runtime.consume_cpu_off_reservation(0));
assert!(runtime.mark_vcpu_exiting());
}
use super::*;
#[test]
fn remove_vcpu_task_clears_pending_interrupts_and_dispatcher_registration() {
let runtime = VmRuntimeHandle::new();
runtime.pending_interrupts.lock().entry(3).or_default();
runtime.irq_dispatcher.register_test_vcpu(3, 11);
assert!(runtime.pending_interrupts.lock().contains_key(&3));
assert_eq!(runtime.irq_dispatcher.test_lookup_cpu_id(3).unwrap(), 11);
runtime.remove_vcpu_task(3);
assert!(!runtime.pending_interrupts.lock().contains_key(&3));
assert!(runtime.irq_dispatcher.test_lookup_cpu_id(3).is_err());
runtime.remove_vcpu_task(3);
assert!(!runtime.pending_interrupts.lock().contains_key(&3));
assert!(runtime.irq_dispatcher.test_lookup_cpu_id(3).is_err());
}
}
impl AxVMResources {
pub(crate) fn from_page_table(
config: AxVMConfig,
page_table: ArchNestedPageTable,
build_nested_paging: impl FnOnce(HostPhysAddr) -> AxVmResult<NestedPagingConfig>,
) -> AxVmResult<Self> {
let address_space = AddrSpace::new_empty(
page_table,
GuestPhysAddr::from(VM_ASPACE_BASE),
VM_ASPACE_SIZE,
)
.map_err(|error| AxVmError::from_addrspace("create guest address space", error))?;
let nested_paging = build_nested_paging(address_space.page_table_root())?;
Ok(Self {
address_space,
nested_paging,
memory_regions: Vec::new(),
config,
phys_cpu_ls: PhysCpuList::default(),
vcpu_list: None,
devices: None,
interrupt_fabric: None,
address_layout: None,
boot_description: GuestBootDescription::none(),
})
}
pub(crate) const fn config(&self) -> &AxVMConfig {
&self.config
}
fn vcpu_list(&self) -> AxVmResult<&[AxVCpuRef]> {
self.vcpu_list
.as_deref()
.ok_or_else(|| ax_err_type!(BadState, "VM vCPU resources are not prepared"))
}
fn devices(&self) -> AxVmResult<Arc<DeviceRuntime>> {
self.devices
.clone()
.ok_or_else(|| ax_err_type!(BadState, "VM devices are not prepared"))
}
fn interrupt_fabric(&self) -> AxVmResult<&InterruptFabric> {
self.interrupt_fabric
.as_ref()
.ok_or_else(|| ax_err_type!(BadState, "VM interrupt fabric is not prepared"))
}
fn reset_transient_resources(&mut self) -> AxVmResult {
if let Some(devices) = self.devices.take() {
devices
.reset_lifecycle_devices()
.map_err(|error| AxVmError::device("reset device lifecycle", error))?;
}
let memory_regions = self.memory_regions.clone();
self.address_space.clear();
for region in &memory_regions {
self.address_space
.map_linear(
region.gpa,
region.host_paddr(),
region.size(),
MappingFlags::READ
| MappingFlags::WRITE
| MappingFlags::EXECUTE
| MappingFlags::USER,
)
.map_err(|error| {
AxVmError::from_addrspace("restore guest memory mapping", error)
})?;
}
self.vcpu_list = None;
self.interrupt_fabric = None;
self.address_layout = None;
Ok(())
}
}
#[allow(dead_code)]
struct PendingFwCfgPayload {
base: GuestPhysAddr,
size: usize,
kernel: &'static [u8],
initrd: Option<&'static [u8]>,
cmdline: Option<String>,
cpu_num: u16,
platform: FwCfgPlatformConfig,
}
pub struct FwCfgDeviceConfig {
pub base: GuestPhysAddr,
pub size: usize,
pub kernel: &'static [u8],
pub initrd: Option<&'static [u8]>,
pub cmdline: Option<String>,
pub cpu_num: u16,
pub platform: FwCfgPlatformConfig,
}
#[derive(Clone)]
struct AxVmDeviceAccessPorts {
vm_id: usize,
}
impl AxVmDeviceAccessPorts {
const fn new(vm_id: usize) -> Self {
Self { vm_id }
}
fn into_ports(self) -> RuntimeAccessPorts {
let this = Arc::new(self);
RuntimeAccessPorts::new()
.with_timer(this.clone())
.with_wake(this.clone())
.with_stop(this)
}
fn vm(&self, operation: &'static str) -> axdevice::DeviceManagerResult<AxVMRef> {
crate::get_vm_by_id(self.vm_id).ok_or_else(|| {
axdevice::DeviceManagerError::ResourceNotFound {
operation,
resource: format!("VM[{}]", self.vm_id),
}
})
}
}
impl TimerAccessPort for AxVmDeviceAccessPorts {
fn schedule_timer(
&self,
device_id: DeviceId,
deadline_ns: u64,
) -> axdevice::DeviceManagerResult {
let vm_id = self.vm_id;
trace!(
"VM[{vm_id}] device {device_id:?} scheduled access-scoped timer at {deadline_ns:#x} ns"
);
crate::timer::register_timer(
deadline_ns,
Box::new(move |_| crate::runtime::vcpus::notify_all_vcpus(vm_id)),
);
Ok(())
}
}
impl WakeAccessPort for AxVmDeviceAccessPorts {
fn wake_vcpu(&self, device_id: DeviceId, vcpu_id: usize) -> axdevice::DeviceManagerResult {
let vm = self.vm("wake vCPU from device access")?;
if vm.vcpu(vcpu_id).is_none() {
return Err(axdevice::DeviceManagerError::InvalidInput {
operation: "wake vCPU from device access",
detail: format!(
"device {device_id:?} requested nonexistent VM[{}] vCPU {}",
self.vm_id, vcpu_id
),
});
}
vm.with_runtime(|runtime| {
runtime.notify_all();
Ok(())
})
.map_err(|error| axdevice::DeviceManagerError::InvalidState {
operation: "wake vCPU from device access",
detail: format!("{error}"),
})
}
}
impl StopAccessPort for AxVmDeviceAccessPorts {
fn request_vm_stop(&self, device_id: DeviceId, reason: &str) -> axdevice::DeviceManagerResult {
let vm = self.vm("request VM stop from device access")?;
vm.stop(StopReason::Fault(format!(
"device {device_id:?} requested VM stop: {reason}"
)))
.map_err(|error| axdevice::DeviceManagerError::InvalidState {
operation: "request VM stop from device access",
detail: format!("{error}"),
})?;
if let Ok(()) = vm.with_runtime(|runtime| {
runtime.notify_all();
Ok(())
}) {}
Ok(())
}
}
#[derive(Debug, Clone)]
pub struct VMMemoryRegion {
pub gpa: GuestPhysAddr,
pub hva: HostVirtAddr,
pub layout: Layout,
pub needs_dealloc: bool,
}
impl VMMemoryRegion {
pub fn size(&self) -> usize {
self.layout.size()
}
pub fn host_paddr(&self) -> HostPhysAddr {
virt_to_phys(self.hva)
}
pub fn is_identical(&self) -> bool {
self.gpa.as_usize() == self.host_paddr().as_usize()
}
}
const TEMP_MAX_VCPU_NUM: usize = 64;
pub struct AxVM {
id: usize,
name: String,
machine: Mutex<Machine<AxVMResources, Arc<VmRuntimeHandle>>>,
pending_fw_cfg_payload: Mutex<Option<PendingFwCfgPayload>>,
}
impl AxVM {
pub fn new(config: AxVMConfig) -> AxVmResult<AxVMRef> {
let id = config.id();
let name = config.name();
let resources = crate::arch::CurrentArch::create_vm_resources(config)?;
let result = Arc::new(Self {
id,
name,
machine: Mutex::new(Machine::Ready(resources)),
pending_fw_cfg_payload: Mutex::new(None),
});
info!("VM created: id={}", result.id());
Ok(result)
}
#[inline]
pub fn id(&self) -> usize {
self.id
}
pub fn name(&self) -> String {
self.name.clone()
}
pub fn status(&self) -> VmStatus {
self.machine.lock().status()
}
pub fn interrupt_mode(&self) -> VMInterruptMode {
self.with_resources(|resources| Ok(resources.config.interrupt_mode()))
.unwrap_or(VMInterruptMode::NoIrq)
}
fn with_resources<F, R>(&self, f: F) -> AxVmResult<R>
where
F: FnOnce(&AxVMResources) -> AxVmResult<R>,
{
let machine = self.machine.lock();
let resources = machine
.resources()
.ok_or_else(|| ax_err_type!(BadState, "VM resources are not available"))?;
f(resources)
}
fn with_resources_mut<F, R>(&self, f: F) -> AxVmResult<R>
where
F: FnOnce(&mut AxVMResources) -> AxVmResult<R>,
{
let mut machine = self.machine.lock();
let resources = machine
.resources_mut()
.ok_or_else(|| ax_err_type!(BadState, "VM resources are not available"))?;
f(resources)
}
fn interrupt_fabric_snapshot(&self) -> AxVmResult<InterruptFabric> {
let machine = self.machine.lock();
match machine.status() {
VmStatus::Running | VmStatus::Paused => machine
.resources()
.ok_or_else(|| ax_err_type!(BadState, "VM resources are not available"))?
.interrupt_fabric()
.cloned(),
status => ax_err!(
BadState,
format!("VM[{}] cannot accept IRQ in {status:?}", self.id())
),
}
}
pub(crate) fn with_runtime<F, R>(&self, f: F) -> AxVmResult<R>
where
F: FnOnce(&Arc<VmRuntimeHandle>) -> AxVmResult<R>,
{
let machine = self.machine.lock();
let runtime = machine
.runtime()
.ok_or_else(|| ax_err_type!(BadState, "VM runtime is not available"))?;
f(runtime)
}
pub(crate) fn current_interrupt_runtime(&self) -> AxVmResult<Arc<VmRuntimeHandle>> {
let machine = self.machine.lock();
Ok(machine.interrupt_runtime()?.clone())
}
fn take_stopped_runtime(&self) -> Option<Arc<VmRuntimeHandle>> {
self.machine.lock().take_stopped_runtime()
}
#[inline]
pub(crate) fn vcpu(&self, vcpu_id: usize) -> Option<AxVCpuRef> {
self.vcpu_list().get(vcpu_id).cloned()
}
#[inline]
pub fn vcpu_num(&self) -> usize {
self.with_resources(|resources| Ok(resources.vcpu_list().map_or(0, <[_]>::len)))
.unwrap_or(0)
}
#[inline]
pub(crate) fn vcpu_list(&self) -> Vec<AxVCpuRef> {
self.with_resources(|resources| Ok(resources.vcpu_list()?.to_vec()))
.unwrap_or_default()
}
pub fn vcpu_snapshots(&self) -> Vec<VcpuSnapshot> {
self.vcpu_list()
.iter()
.map(|vcpu| VcpuSnapshot {
id: vcpu.id(),
state: vcpu.state(),
phys_cpu_set: vcpu.phys_cpu_set(),
})
.collect()
}
pub fn nested_page_table_root(&self) -> AxVmResult<HostPhysAddr> {
self.with_resources(|resources| Ok(resources.address_space.page_table_root()))
}
pub fn with_config<F, R>(&self, f: F) -> R
where
F: FnOnce(&mut AxVMConfig) -> R,
{
let mut machine = self.machine.lock();
let resources = machine
.resources_mut()
.expect("VM resources are not available for config access");
f(&mut resources.config)
}
pub fn set_guest_device_tree(&self, load_gpa: GuestPhysAddr, bytes: Vec<u8>) -> AxVmResult {
self.with_resources_mut(|resources| {
resources.config.set_dtb_load_gpa(load_gpa);
resources
.boot_description
.set_device_tree(GuestFdtBuilder::from_bytes(bytes).build(load_gpa));
Ok(())
})
}
pub fn get_image_load_region(
&self,
image_load_gpa: GuestPhysAddr,
image_size: usize,
) -> AxVmResult<Vec<&'static mut [u8]>> {
let image_load_hva = self.with_resources(|resources| {
resources
.address_space
.translated_byte_buffer(image_load_gpa, image_size)
.ok_or_else(|| {
ax_err_type!(BadState, "Failed to translate kernel image load address")
})
})?;
Ok(image_load_hva)
}
pub fn start(self: &Arc<Self>) -> AxVmResult {
if self.status() == VmStatus::Stopped {
if let Some(runtime) = self.take_stopped_runtime() {
runtime.join_all_vcpu_tasks(self.id());
}
self.prepare()?;
}
info!("Starting VM[{}]", self.id());
let primary_vcpu = self
.vcpu(0)
.ok_or_else(|| ax_err_type!(BadState, "VM primary vCPU is not prepared"))?;
let primary_task = crate::runtime::vcpus::build_vcpu_task(self, primary_vcpu);
let runtime = Arc::new(VmRuntimeHandle::new());
self.machine.lock().start_with(|resources| {
resources
.vcpu_list()
.map_err(|error| AxVmError::resource_unavailable("vCPU list", error))?;
resources
.devices()
.map_err(|error| AxVmError::resource_unavailable("devices", error))?;
resources
.interrupt_fabric()
.map_err(|error| AxVmError::resource_unavailable("interrupt fabric", error))?;
Ok(runtime.clone())
})?;
let task = crate::host::task::spawn_task(primary_task);
runtime.add_vcpu_task(0, task)?;
Ok(())
}
pub fn running(&self) -> bool {
self.status() == VmStatus::Running
}
pub fn stopping(&self) -> bool {
self.status() == VmStatus::Stopping
}
pub fn suspending(&self) -> bool {
matches!(self.status(), VmStatus::Pausing | VmStatus::Paused)
}
pub fn stopped(&self) -> bool {
self.status() == VmStatus::Stopped
}
pub fn pause(&self) -> AxVmResult {
let mut machine = self.machine.lock();
if machine.status() != VmStatus::Running {
return machine.pause();
}
let devices = machine
.resources()
.expect("running VM must retain resources")
.devices()?;
devices
.suspend_lifecycle_devices()
.map_err(|error| AxVmError::device("suspend device lifecycle", error))?;
machine.pause()
}
pub fn resume(&self) -> AxVmResult {
let mut machine = self.machine.lock();
if machine.status() != VmStatus::Paused {
return machine.resume();
}
let devices = machine
.resources()
.expect("paused VM must retain resources")
.devices()?;
devices
.resume_lifecycle_devices()
.map_err(|error| AxVmError::device("resume device lifecycle", error))?;
machine.resume()
}
pub fn stop(&self, reason: StopReason) -> AxVmResult {
info!("Stopping VM[{}]: {reason:?}", self.id());
self.machine.lock().request_stop_with(reason, |_, _| Ok(()))
}
pub(crate) fn finish_stop(&self) -> AxVmResult {
self.machine.lock().finish_stop()
}
fn wait_until_stopped(&self) -> AxVmResult {
const MAX_YIELDS: usize = 10_000;
for _ in 0..MAX_YIELDS {
match self.status() {
VmStatus::Stopped | VmStatus::Ready => return Ok(()),
VmStatus::Stopping | VmStatus::Running | VmStatus::Paused | VmStatus::Pausing => {
crate::host::task::yield_now();
}
status => {
return ax_err!(
BadState,
format!("VM[{}] cannot wait for stop from {status:?}", self.id())
);
}
}
}
ax_err!(
BadState,
format!("VM[{}] did not stop before reset timeout", self.id())
)
}
fn stop_and_join_runtime(&self, reason: StopReason) -> AxVmResult {
match self.status() {
VmStatus::Running | VmStatus::Paused => {
self.stop(reason)?;
if let Ok(()) = self.with_runtime(|runtime| {
runtime.notify_all();
Ok(())
}) {}
self.wait_until_stopped()?;
}
VmStatus::Stopping => {
if let Ok(()) = self.with_runtime(|runtime| {
runtime.notify_all();
Ok(())
}) {}
self.wait_until_stopped()?;
}
VmStatus::Stopped | VmStatus::Ready => {}
status => {
return ax_err!(
BadState,
format!("VM[{}] cannot quiesce runtime from {status:?}", self.id())
);
}
}
if let Some(runtime) = self.take_stopped_runtime() {
runtime.join_all_vcpu_tasks(self.id());
}
Ok(())
}
pub fn reset(self: &Arc<Self>) -> AxVmResult {
info!("Resetting VM[{}]", self.id());
self.stop_and_join_runtime(StopReason::Forced)?;
self.machine.lock().reset_with(|resources| {
resources
.reset_transient_resources()
.map_err(|error| AxVmError::resource_unavailable("reset resources", error))
})?;
self.prepare()?;
self.start()
}
pub fn get_devices(&self) -> AxVmResult<Arc<DeviceRuntime>> {
self.with_resources(|resources| resources.devices())
}
pub fn pulse_interrupt(&self, irq_id: usize) -> AxVmResult {
pulse_interrupt_with_snapshot(|| self.interrupt_fabric_snapshot(), irq_id)
}
pub fn device_count(&self) -> usize {
self.get_devices()
.map(|devices| devices.devices().count())
.unwrap_or(0)
}
pub fn add_fw_cfg_device(&self, config: FwCfgDeviceConfig) -> AxVmResult {
let mut pending = self.pending_fw_cfg_payload.lock();
if pending.is_some() {
return ax_err!(
AlreadyExists,
format!("VM[{}] fw_cfg device already exists", self.id())
);
}
*pending = Some(PendingFwCfgPayload {
base: config.base,
size: config.size,
kernel: config.kernel,
initrd: config.initrd,
cmdline: config.cmdline,
cpu_num: config.cpu_num,
platform: config.platform,
});
debug!(
"VM[{}] queued fw_cfg device: base={:#x}, size={:#x}, kernel={} bytes, initrd={:?}",
self.id(),
config.base.as_usize(),
config.size,
config.kernel.len(),
config.initrd.map(|data| data.len())
);
Ok(())
}
#[allow(dead_code)]
pub(crate) fn fw_cfg_payload(&self) -> Option<FwCfgPayloadConfig> {
self.pending_fw_cfg_payload
.lock()
.as_ref()
.map(|pending| FwCfgPayloadConfig {
base: pending.base,
size: pending.size,
kernel: pending.kernel,
initrd: pending.initrd,
cmdline: pending.cmdline.clone(),
cpu_num: pending.cpu_num,
platform: pending.platform.clone(),
})
}
pub(crate) fn device_access_ports(&self) -> RuntimeAccessPorts {
AxVmDeviceAccessPorts::new(self.id()).into_ports()
}
pub(crate) fn handle_mmio_write(
&self,
addr: GuestPhysAddr,
width: AccessWidth,
data: usize,
) -> AxVmResult {
let devices = self.get_devices()?;
if devices.mmio_write_needs_guest_memory(addr, width) {
let mut memory = VmDmaAccess { vm: self };
devices.handle_mmio_write_with_memory(addr, width, data, &mut memory)?;
} else {
devices.handle_mmio_write(addr, width, data)?;
}
Ok(())
}
pub(crate) fn handle_nested_page_fault(
&self,
addr: GuestPhysAddr,
access_flags: MappingFlags,
) -> bool {
self.with_resources_mut(|resources| {
let handled = resources
.address_space
.handle_page_fault(addr, access_flags);
Self::debug_nested_page_fault(self.id(), resources, addr, access_flags, handled);
Ok(handled)
})
.unwrap_or(false)
}
fn debug_nested_page_fault(
vm_id: usize,
resources: &AxVMResources,
addr: GuestPhysAddr,
access_flags: MappingFlags,
handled: bool,
) {
let root = resources.address_space.page_table_root();
match resources.address_space.page_table().query(addr) {
Ok((hpa, flags, size)) => {
if handled {
debug!(
"VM[{}] stage2 query hit: gpa={:#x} -> hpa={:#x}, access={:?}, \
pte_flags={:?}, page_size={:?}, root={:#x}",
vm_id,
addr.as_usize(),
hpa.as_usize(),
access_flags,
flags,
size,
root.as_usize()
);
} else {
warn!(
"VM[{}] stage2 query hit: gpa={:#x} -> hpa={:#x}, access={:?}, \
pte_flags={:?}, page_size={:?}, root={:#x}",
vm_id,
addr.as_usize(),
hpa.as_usize(),
access_flags,
flags,
size,
root.as_usize()
);
}
}
Err(err) => {
if handled {
debug!(
"VM[{}] stage2 query miss: gpa={:#x}, access={:?}, err={:?}, root={:#x}",
vm_id,
addr.as_usize(),
access_flags,
err,
root.as_usize()
);
} else {
warn!(
"VM[{}] stage2 query miss: gpa={:#x}, access={:?}, err={:?}, root={:#x}",
vm_id,
addr.as_usize(),
access_flags,
err,
root.as_usize()
);
}
}
}
let translate = resources.address_space.translate(addr);
if handled {
debug!(
"VM[{}] stage2 translate: gpa={:#x} -> {:?}",
vm_id,
addr.as_usize(),
translate
);
} else {
warn!(
"VM[{}] stage2 translate: gpa={:#x} -> {:?}",
vm_id,
addr.as_usize(),
translate
);
}
for (idx, region) in resources.memory_regions.iter().enumerate() {
let start = region.gpa.as_usize();
let end = start + region.size();
if (start..end).contains(&addr.as_usize()) {
if handled {
debug!(
"VM[{}] stage2 region hit[{}]: gpa=[{:#x},{:#x}) hva={:#x} hpa={:#x} \
size={:#x} identical={}",
vm_id,
idx,
start,
end,
region.hva.as_usize(),
region.host_paddr().as_usize(),
region.size(),
region.is_identical()
);
} else {
warn!(
"VM[{}] stage2 region hit[{}]: gpa=[{:#x},{:#x}) hva={:#x} hpa={:#x} \
size={:#x} identical={}",
vm_id,
idx,
start,
end,
region.hva.as_usize(),
region.host_paddr().as_usize(),
region.size(),
region.is_identical()
);
}
}
}
}
pub fn inject_interrupt_to_vcpu(
&self,
targets: CpuMask<TEMP_MAX_VCPU_NUM>,
irq: usize,
) -> AxVmResult {
for vcpu in self.vcpu_list() {
if targets.get(vcpu.id()) {
crate::runtime::vcpus::queue_interrupt(self.id(), vcpu.id(), irq)?;
}
}
Ok(())
}
pub fn get_vcpu_affinities_pcpu_ids(&self) -> Vec<(usize, Option<usize>, usize)> {
self.with_resources(|resources| Ok(resources.phys_cpu_ls.get_vcpu_affinities_pcpu_ids()))
.unwrap_or_default()
}
pub fn get_vcpu_guest_mpidrs(&self) -> Vec<(usize, u64)> {
self.vcpu_list()
.iter()
.filter_map(|vcpu| vcpu.guest_mpidr().map(|mpidr| (vcpu.id(), mpidr)))
.collect()
}
pub fn map_region(
&self,
gpa: GuestPhysAddr,
hpa: HostPhysAddr,
size: usize,
flags: MappingFlags,
) -> AxVmResult {
self.with_resources_mut(|resources| {
resources
.address_space
.map_linear(gpa, hpa, size, flags)
.map_err(|error| AxVmError::from_addrspace("map guest memory region", error))?;
Ok(())
})
}
pub fn unmap_region(&self, gpa: GuestPhysAddr, size: usize) -> AxVmResult {
self.with_resources_mut(|resources| {
resources
.address_space
.unmap(gpa, size)
.map_err(|error| AxVmError::from_addrspace("unmap guest memory region", error))?;
Ok(())
})
}
pub fn read_from_guest_of<T>(&self, gpa_ptr: GuestPhysAddr) -> AxVmResult<T> {
let size = core::mem::size_of::<T>();
if !gpa_ptr
.as_usize()
.is_multiple_of(core::mem::align_of::<T>())
{
return ax_err!(InvalidInput, "Unaligned guest physical address");
}
self.with_resources(|resources| {
let Some(buffers) = resources
.address_space
.translated_byte_buffer(gpa_ptr, size)
else {
return ax_err!(
InvalidInput,
"Failed to translate guest physical address or insufficient buffer size"
);
};
let mut data_bytes = Vec::with_capacity(size);
for chunk in buffers {
let remaining = size - data_bytes.len();
let chunk_size = remaining.min(chunk.len());
data_bytes.extend_from_slice(&chunk[..chunk_size]);
if data_bytes.len() >= size {
break;
}
}
if data_bytes.len() < size {
return ax_err!(
InvalidInput,
"Insufficient data in guest memory to read the requested object"
);
}
let data: T = unsafe {
core::ptr::read_unaligned(data_bytes.as_ptr() as *const T)
};
Ok(data)
})
}
pub fn read_from_guest(&self, gpa_ptr: GuestPhysAddr, buffer: &mut [u8]) -> AxVmResult {
self.with_resources(|resources| {
let Some(chunks) = resources
.address_space
.translated_byte_buffer(gpa_ptr, buffer.len())
else {
return ax_err!(InvalidInput, "Failed to translate guest physical address");
};
let mut copied = 0;
for chunk in chunks {
let len = (buffer.len() - copied).min(chunk.len());
buffer[copied..copied + len].copy_from_slice(&chunk[..len]);
copied += len;
if copied == buffer.len() {
return Ok(());
}
}
ax_err!(
InvalidInput,
"Insufficient guest memory to read the requested buffer"
)
})
}
pub fn write_to_guest_of<T>(&self, gpa_ptr: GuestPhysAddr, data: &T) -> AxVmResult {
let bytes = unsafe {
core::slice::from_raw_parts(data as *const T as *const u8, core::mem::size_of::<T>())
};
self.write_to_guest(gpa_ptr, bytes)
}
pub fn write_to_guest(&self, gpa_ptr: GuestPhysAddr, data: &[u8]) -> AxVmResult {
if data.is_empty() {
return Ok(());
}
self.with_resources(|resources| {
let Some(mut chunks) = resources
.address_space
.translated_byte_buffer(gpa_ptr, data.len())
else {
return ax_err!(InvalidInput, "Failed to translate guest physical address");
};
write_guest_bytes_to_chunks(chunks.as_mut_slice(), data)
})
}
pub fn alloc_ivc_channel(&self, expected_size: usize) -> AxVmResult<(GuestPhysAddr, usize)> {
let size = align_up_4k(expected_size);
let gpa = self
.get_devices()?
.alloc_ivc_channel(size)
.map_err(|error| AxVmError::memory("reserve IVC guest address range", error))?;
Ok((gpa, size))
}
pub fn release_ivc_channel(&self, gpa: GuestPhysAddr, size: usize) -> AxVmResult {
self.get_devices()?
.release_ivc_channel(gpa, size)
.map_err(|error| AxVmError::memory("release IVC guest address range", error))?;
Ok(())
}
pub fn alloc_memory_region(
&self,
layout: Layout,
gpa: Option<GuestPhysAddr>,
) -> AxVmResult<&[u8]> {
assert!(
layout.size() > 0,
"Cannot allocate zero-sized memory region"
);
let hva = unsafe { alloc::alloc::alloc_zeroed(layout) };
if hva.is_null() {
return Err(AxVmError::OutOfMemory {
operation: "allocate IVC channel",
});
}
let s = unsafe { core::slice::from_raw_parts_mut(hva, layout.size()) };
let hva = HostVirtAddr::from_mut_ptr_of(hva);
let hpa = virt_to_phys(hva);
let gpa = gpa.unwrap_or_else(|| hpa.as_usize().into());
if let Err(err) = self.with_resources_mut(|resources| {
resources
.address_space
.map_linear(
gpa,
hpa,
layout.size(),
MappingFlags::READ
| MappingFlags::WRITE
| MappingFlags::EXECUTE
| MappingFlags::USER,
)
.map_err(|error| AxVmError::from_addrspace("map allocated guest memory", error))?;
resources.memory_regions.push(VMMemoryRegion {
gpa,
hva,
layout,
needs_dealloc: true, });
Ok(())
}) {
unsafe {
alloc::alloc::dealloc(hva.as_mut_ptr(), layout);
}
return Err(err);
}
Ok(s)
}
pub fn memory_regions(&self) -> Vec<VMMemoryRegion> {
self.with_resources(|resources| Ok(resources.memory_regions.clone()))
.unwrap_or_default()
}
pub fn prepare_memory_layout(&self) -> AxVmResult<PreparedMemoryLayout> {
let memory_regions =
self.with_resources(|resources| Ok(resources.config.memory_regions().to_vec()))?;
let layout = memory::MemoryLayoutBuilder::new(self, &memory_regions).prepare()?;
let main_memory = layout.main_memory();
let boot_plan = boot::BootImagePlan::new(main_memory.gpa, main_memory.is_identical());
self.with_config(|config| boot_plan.apply_to_config(config));
Ok(layout)
}
pub fn map_reserved_memory_region(
&self,
layout: Layout,
gpa: Option<GuestPhysAddr>,
) -> AxVmResult {
assert!(
layout.size() > 0,
"Cannot allocate zero-sized memory region"
);
let gpa =
gpa.ok_or_else(|| ax_err_type!(InvalidInput, "Reserved memory GPA is required"))?;
self.with_resources_mut(|resources| {
resources
.address_space
.map_linear(
gpa,
gpa.as_usize().into(),
layout.size(),
MappingFlags::READ
| MappingFlags::WRITE
| MappingFlags::EXECUTE
| MappingFlags::USER,
)
.map_err(|error| AxVmError::from_addrspace("map reserved guest memory", error))?;
let hva = gpa.as_usize().into();
resources.memory_regions.push(VMMemoryRegion {
gpa,
hva,
layout,
needs_dealloc: false, });
Ok(())
})
}
pub fn destroy(&self) -> AxVmResult {
let vm_id = self.id();
match self.status() {
VmStatus::Running | VmStatus::Paused | VmStatus::Stopping => {
self.stop_and_join_runtime(StopReason::Forced)?;
}
VmStatus::Ready | VmStatus::Stopped | VmStatus::Failed => {
if let Some(runtime) = self.take_stopped_runtime() {
runtime.join_all_vcpu_tasks(vm_id);
}
}
VmStatus::Destroyed | VmStatus::Destroying => {}
VmStatus::Pausing => {
self.stop_and_join_runtime(StopReason::Forced)?;
}
}
self.machine.lock().destroy_with(|resources| {
if let Some(mut resources) = resources {
Self::cleanup_resource_set(vm_id, &mut resources)?;
}
Ok(())
})
}
fn cleanup_resource_set(vm_id: usize, resources: &mut AxVMResources) -> AxVmResult {
info!("Cleaning up VM[{vm_id}] resources...");
if let Some(devices) = resources.devices.take() {
devices.reset_lifecycle_devices().map_err(|error| {
AxVmError::device("reset device lifecycle during destroy", error)
})?;
debug!(
"VM[{vm_id}] devices cleanup: {} device(s)",
devices.devices().count()
);
}
let regions_to_cleanup = resources.memory_regions.clone();
for region in ®ions_to_cleanup {
debug!(
"VM[{vm_id}] unmapping memory region: GPA={:#x}, size={:#x}",
region.gpa.as_usize(),
region.size()
);
if let Err(err) = resources.address_space.unmap(region.gpa, region.size()) {
warn!(
"VM[{vm_id}] failed to unmap region at GPA={:#x}: {err:?}",
region.gpa.as_usize()
);
}
}
for region in ®ions_to_cleanup {
if region.needs_dealloc {
debug!(
"VM[{vm_id}] deallocating memory region: HVA={:#x}, size={:#x}",
region.hva.as_usize(),
region.size()
);
unsafe {
alloc::alloc::dealloc(region.hva.as_mut_ptr(), region.layout);
}
} else {
debug!(
"VM[{vm_id}] skipping reserved memory region dealloc: GPA={:#x}, HVA={:#x}, \
size={:#x}",
region.gpa.as_usize(),
region.hva.as_usize(),
region.size()
);
}
}
resources.memory_regions.clear();
resources.address_space.clear();
resources.vcpu_list = None;
resources.interrupt_fabric = None;
info!("VM[{vm_id}] resources cleanup completed");
Ok(())
}
}
impl Drop for AxVM {
fn drop(&mut self) {
info!("Dropping VM[{}]", self.id());
if let Err(err) = self.destroy() {
warn!("VM[{}] destroy during drop failed: {err:?}", self.id());
}
info!("VM[{}] dropped", self.id());
}
}
#[cfg(test)]
mod tests {
use core::{cell::RefCell, sync::atomic::AtomicBool};
use axdevice_base::{IrqError, IrqLineId, IrqResult, IrqSink};
use super::*;
#[test]
fn write_guest_bytes_to_chunks_writes_only_remaining_bytes() {
let mut first = [0u8; 2];
let mut second = [0u8; 4];
let mut chunks: [&mut [u8]; 2] = [&mut first, &mut second];
write_guest_bytes_to_chunks(&mut chunks, &[1, 2, 3]).unwrap();
assert_eq!(first, [1, 2]);
assert_eq!(second, [3, 0, 0, 0]);
}
#[test]
fn write_guest_bytes_to_chunks_rejects_insufficient_capacity() {
let mut only = [0u8; 2];
let mut chunks: [&mut [u8]; 1] = [&mut only];
let err = write_guest_bytes_to_chunks(&mut chunks, &[1, 2, 3]).unwrap_err();
assert!(matches!(err, AxVmError::InvalidInput { .. }));
assert_eq!(only, [1, 2]);
}
#[test]
fn write_guest_bytes_to_chunks_accepts_empty_writes() {
let mut chunk = [7u8; 2];
let mut chunks: [&mut [u8]; 1] = [&mut chunk];
write_guest_bytes_to_chunks(&mut chunks, &[]).unwrap();
assert_eq!(chunk, [7, 7]);
}
#[test]
fn runtime_dispatch_orders_enqueue_before_notify_and_ipi() {
let events = RefCell::new(Vec::new());
dispatch_vcpu_interrupt_with(
|| {
events.borrow_mut().push("enqueue");
Ok(3)
},
|| events.borrow_mut().push("notify"),
|cpu_id| {
assert_eq!(cpu_id, 3);
events.borrow_mut().push("ipi");
},
)
.unwrap();
assert_eq!(*events.borrow(), ["enqueue", "notify", "ipi"]);
}
#[cfg(feature = "host-test")]
#[test]
fn runtime_dispatch_releases_queue_lock_before_callbacks() {
let dispatcher = VcpuIrqDispatcher::new();
dispatcher.register_test_vcpu(0, 3);
let interrupt = PendingVcpuInterrupt {
id: crate::irq::model::VirtualInterruptId(7),
trigger: crate::InterruptTriggerMode::LevelTriggered,
};
let events = RefCell::new(Vec::new());
dispatch_vcpu_interrupt_with(
|| dispatcher.enqueue(0, interrupt),
|| {
assert_eq!(dispatcher.drain(0), alloc::vec![interrupt]);
events.borrow_mut().push("notify");
},
|_| events.borrow_mut().push("ipi"),
)
.unwrap();
assert_eq!(*events.borrow(), ["notify", "ipi"]);
}
#[test]
fn runtime_dispatch_stops_when_enqueue_fails() {
let events = RefCell::new(Vec::new());
let result = dispatch_vcpu_interrupt_with(
|| {
events.borrow_mut().push("enqueue");
Err(ax_err_type!(NotFound, "vCPU task not found"))
},
|| events.borrow_mut().push("notify"),
|_| events.borrow_mut().push("ipi"),
);
assert!(matches!(result, Err(AxVmError::ResourceUnavailable { .. })));
assert_eq!(*events.borrow(), ["enqueue"]);
}
#[test]
fn interrupt_pulse_runs_after_snapshot_lock_is_released() {
let machine_lock = Arc::new(TestMachineLock::default());
let sink = Arc::new(TestIrqSink::with_machine_lock(machine_lock.clone()));
let fabric = InterruptFabric::with_sink(VMInterruptMode::Emulated, sink.clone()).unwrap();
pulse_interrupt_with_snapshot(
|| {
let _machine = machine_lock.lock();
Ok(fabric.clone())
},
7,
)
.unwrap();
assert_eq!(sink.pulse_count.load(Ordering::Relaxed), 1);
}
#[test]
fn interrupt_snapshot_failure_does_not_call_sink() {
let sink = Arc::new(TestIrqSink::default());
let expected = ax_err_type!(BadState, "interrupt snapshot unavailable");
let result = pulse_interrupt_with_snapshot(|| Err(expected.clone()), 9);
assert_eq!(result, Err(expected));
assert_eq!(sink.pulse_count.load(Ordering::Relaxed), 0);
}
#[test]
fn interrupt_pulse_propagates_sink_error() {
let irq_error = IrqError::Backend {
line: IrqLineId(11),
operation: "test pulse",
detail: "controller rejected interrupt".into(),
};
let sink = Arc::new(TestIrqSink::with_pulse_error(irq_error.clone()));
let fabric = InterruptFabric::with_sink(VMInterruptMode::Emulated, sink).unwrap();
let result = pulse_interrupt_with_snapshot(|| Ok(fabric.clone()), 11);
assert_eq!(result, Err(AxVmError::from(irq_error)));
}
#[derive(Default)]
struct TestIrqSink {
machine_lock: Option<Arc<TestMachineLock>>,
pulse_error: Option<IrqError>,
pulse_count: AtomicUsize,
}
impl TestIrqSink {
fn with_machine_lock(machine_lock: Arc<TestMachineLock>) -> Self {
Self {
machine_lock: Some(machine_lock),
..Self::default()
}
}
fn with_pulse_error(pulse_error: IrqError) -> Self {
Self {
pulse_error: Some(pulse_error),
..Self::default()
}
}
}
impl IrqSink for TestIrqSink {
fn set_level(&self, _line: IrqLineId, _asserted: bool) -> IrqResult {
Ok(())
}
fn pulse(&self, _line: IrqLineId) -> IrqResult {
let _machine = self.machine_lock.as_ref().map(|machine_lock| {
machine_lock
.try_lock()
.expect("interrupt callback must run without the machine lock")
});
self.pulse_count.fetch_add(1, Ordering::Relaxed);
self.pulse_error.clone().map_or(Ok(()), Err)
}
}
#[derive(Default)]
struct TestMachineLock {
held: AtomicBool,
}
impl TestMachineLock {
fn lock(&self) -> TestMachineGuard<'_> {
self.try_lock().expect("test machine lock is already held")
}
fn try_lock(&self) -> Option<TestMachineGuard<'_>> {
self.held
.compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
.ok()
.map(|_| TestMachineGuard { lock: self })
}
}
struct TestMachineGuard<'a> {
lock: &'a TestMachineLock,
}
impl Drop for TestMachineGuard<'_> {
fn drop(&mut self) {
self.lock.held.store(false, Ordering::Release);
}
}
}