use alloc::{sync::Arc, vec::Vec};
use core::ops::Range;
#[cfg(target_arch = "aarch64")]
use arm_vgic::Vgic;
use ax_errno::{AxResult, ax_err};
use ax_kspin::SpinNoIrq as Mutex;
#[cfg(target_arch = "aarch64")]
use ax_memory_addr::PhysAddr;
use ax_memory_addr::is_aligned_4k;
use axdevice_base::{
AccessWidth, BaseDeviceOps, BaseMmioDeviceOps, BasePortDeviceOps, BaseSysRegDeviceOps,
DeviceAddrRange, Port, PortRange, SysRegAddr, SysRegAddrRange,
};
use axvm_types::{EmulatedDeviceConfig, EmulatedDeviceType, GuestPhysAddr, GuestPhysAddrRange};
#[cfg(target_arch = "riscv64")]
use riscv_vplic::VPlicGlobal;
#[cfg(target_arch = "x86_64")]
use x86_vlapic::{EmulatedIoApic, EmulatedPit, EmulatedSerialPort, IoApicInterrupt};
use crate::{
AxVmDeviceConfig, DeviceBuildContext, DeviceBundle, DeviceFactoryRegistry, DeviceRegistration,
PollableDeviceOps, range_alloc::RangeAllocator,
};
pub struct AxEmuDevices<R: DeviceAddrRange> {
emu_devices: Vec<Arc<dyn BaseDeviceOps<R>>>,
}
impl<R: DeviceAddrRange + 'static> AxEmuDevices<R> {
pub fn new() -> Self {
Self {
emu_devices: Vec::new(),
}
}
fn validate_dev_against<'a>(
dev: &Arc<dyn BaseDeviceOps<R>>,
existing_devices: impl IntoIterator<Item = &'a Arc<dyn BaseDeviceOps<R>>>,
) -> AxResult
where
R: 'a,
{
let new_range = dev.address_range();
let new_type = dev.emu_type();
if new_range.is_empty() {
return ax_err!(
InvalidInput,
format_args!(
"failed to register {} device type {} at range {new_range:#x}: range is empty \
or invalid, possibly due to address overflow",
R::BUS_NAME,
new_type,
)
);
}
for existing in existing_devices {
let existing_range = existing.address_range();
let existing_type = existing.emu_type();
if Arc::ptr_eq(existing, dev) {
return ax_err!(
AlreadyExists,
format_args!(
"failed to register {} device type {} at range {new_range:#x}: the same \
device is already registered as type {} at range {existing_range:#x}",
R::BUS_NAME,
new_type,
existing_type,
)
);
}
if new_range == existing_range {
return ax_err!(
AlreadyExists,
format_args!(
"failed to register {} device type {} at range {new_range:#x}: range is \
already registered by device type {} at range {existing_range:#x}",
R::BUS_NAME,
new_type,
existing_type,
)
);
}
if new_range.overlaps(&existing_range) {
return ax_err!(
AddrInUse,
format_args!(
"failed to register {} device type {} at range {new_range:#x}: overlaps \
device type {} at range {existing_range:#x}",
R::BUS_NAME,
new_type,
existing_type,
)
);
}
}
Ok(())
}
fn validate_devices(&self, devices: &[Arc<dyn BaseDeviceOps<R>>]) -> AxResult {
for (index, device) in devices.iter().enumerate() {
Self::validate_dev_against(
device,
self.emu_devices.iter().chain(devices[..index].iter()),
)?;
}
Ok(())
}
pub fn add_dev(&mut self, dev: Arc<dyn BaseDeviceOps<R>>) -> AxResult {
self.validate_devices(core::slice::from_ref(&dev))?;
self.emu_devices.push(dev);
Ok(())
}
fn commit_devices(&mut self, devices: Vec<Arc<dyn BaseDeviceOps<R>>>) {
self.emu_devices.extend(devices);
}
pub fn find_dev(&self, addr: R::Addr) -> Option<Arc<dyn BaseDeviceOps<R>>> {
self.emu_devices
.iter()
.find(|&dev| dev.address_range().contains(addr))
.cloned()
}
pub fn iter(&self) -> impl Iterator<Item = &Arc<dyn BaseDeviceOps<R>>> {
self.emu_devices.iter()
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut Arc<dyn BaseDeviceOps<R>>> {
self.emu_devices.iter_mut()
}
}
impl<R: DeviceAddrRange + 'static> Default for AxEmuDevices<R> {
fn default() -> Self {
Self::new()
}
}
type AxEmuMmioDevices = AxEmuDevices<GuestPhysAddrRange>;
type AxEmuSysRegDevices = AxEmuDevices<SysRegAddrRange>;
type AxEmuPortDevices = AxEmuDevices<PortRange>;
pub struct AxVmDevices {
emu_mmio_devices: AxEmuMmioDevices,
emu_sys_reg_devices: AxEmuSysRegDevices,
emu_port_devices: AxEmuPortDevices,
pollable_devices: Vec<Arc<dyn PollableDeviceOps>>,
#[cfg(target_arch = "x86_64")]
x86_ioapic: Option<Arc<EmulatedIoApic>>,
#[cfg(target_arch = "x86_64")]
x86_pit: Option<Arc<EmulatedPit>>,
#[cfg(target_arch = "x86_64")]
x86_serial: Option<Arc<EmulatedSerialPort>>,
ivc_channel: Option<Mutex<RangeAllocator>>,
}
#[inline]
fn log_device_io(
addr_type: &'static str,
addr: impl core::fmt::LowerHex,
addr_range: impl core::fmt::LowerHex,
read: bool,
width: AccessWidth,
) {
let rw = if read { "read" } else { "write" };
trace!("emu_device {rw}: {addr_type} {addr:#x} in range {addr_range:#x} with width {width:?}")
}
#[inline]
fn panic_device_not_found(
addr_type: &'static str,
addr: impl core::fmt::LowerHex,
read: bool,
width: AccessWidth,
) -> ! {
let rw = if read { "read" } else { "write" };
error!(
"emu_device {rw} failed: device not found for {addr_type} {addr:#x} with width {width:?}"
);
panic!("emu_device not found");
}
impl AxVmDevices {
fn empty() -> Self {
Self {
emu_mmio_devices: AxEmuMmioDevices::new(),
emu_sys_reg_devices: AxEmuSysRegDevices::new(),
emu_port_devices: AxEmuPortDevices::new(),
pollable_devices: Vec::new(),
#[cfg(target_arch = "x86_64")]
x86_ioapic: None,
#[cfg(target_arch = "x86_64")]
x86_pit: None,
#[cfg(target_arch = "x86_64")]
x86_serial: None,
ivc_channel: None,
}
}
pub fn new(config: AxVmDeviceConfig) -> AxResult<Self> {
let mut this = Self::empty();
Self::init(&mut this, &config.emu_configs)?;
Ok(this)
}
pub fn build_with_factories(
config: AxVmDeviceConfig,
factories: &DeviceFactoryRegistry,
context: &DeviceBuildContext<'_>,
) -> AxResult<Self> {
let mut this = Self::empty();
for config in &config.emu_configs {
if factories.get(config.emu_type).is_some() {
this.register_factory_device(config, factories, context)?;
} else if Self::is_legacy_fallback(config.emu_type) {
Self::init(&mut this, core::slice::from_ref(config))?;
} else {
return ax_err!(
Unsupported,
format_args!(
"no factory is registered for emulated device '{}' of type {}",
config.name, config.emu_type
)
);
}
}
Ok(this)
}
pub fn register_factory_device(
&mut self,
config: &EmulatedDeviceConfig,
factories: &DeviceFactoryRegistry,
context: &DeviceBuildContext<'_>,
) -> AxResult {
let bundle = factories.build(config, context)?;
self.register_bundle(bundle)
}
fn is_legacy_fallback(device_type: EmulatedDeviceType) -> bool {
matches!(
device_type,
EmulatedDeviceType::InterruptController
| EmulatedDeviceType::Console
| EmulatedDeviceType::IVCChannel
| EmulatedDeviceType::GPPTRedistributor
| EmulatedDeviceType::GPPTDistributor
| EmulatedDeviceType::GPPTITS
| EmulatedDeviceType::X86IoApic
| EmulatedDeviceType::X86Pit
| EmulatedDeviceType::PPPTGlobal
)
}
fn init(this: &mut Self, emu_configs: &[EmulatedDeviceConfig]) -> AxResult {
for config in emu_configs {
match config.emu_type {
EmulatedDeviceType::InterruptController => {
#[cfg(target_arch = "aarch64")]
{
this.add_mmio_dev(Arc::new(Vgic::new()))?;
}
#[cfg(not(target_arch = "aarch64"))]
{
warn!(
"emu type: {} is not supported on this platform",
config.emu_type
);
}
}
EmulatedDeviceType::GPPTRedistributor => {
#[cfg(target_arch = "aarch64")]
{
const GPPT_GICR_ARG_ERR_MSG: &str =
"expect 3 args for gppt redistributor (cpu_num, stride, pcpu_id)";
let cpu_num = config
.cfg_list
.first()
.copied()
.expect(GPPT_GICR_ARG_ERR_MSG);
let stride = config
.cfg_list
.get(1)
.copied()
.expect(GPPT_GICR_ARG_ERR_MSG);
let pcpu_id = config
.cfg_list
.get(2)
.copied()
.expect(GPPT_GICR_ARG_ERR_MSG);
for i in 0..cpu_num {
let addr = config.base_gpa + i * stride;
let size = config.length;
#[allow(clippy::arc_with_non_send_sync)]
this.add_mmio_dev(Arc::new(arm_vgic::v3::vgicr::VGicR::new(
addr.into(),
Some(size),
pcpu_id + i,
)))?;
info!(
"GPPT Redistributor initialized for vCPU {i} with base GPA \
{addr:#x} and length {size:#x}"
);
}
}
#[cfg(not(target_arch = "aarch64"))]
{
warn!(
"emu type: {} is not supported on this platform",
config.emu_type
);
}
}
EmulatedDeviceType::GPPTDistributor => {
#[cfg(target_arch = "aarch64")]
{
#[allow(clippy::arc_with_non_send_sync)]
this.add_mmio_dev(Arc::new(arm_vgic::v3::vgicd::VGicD::new(
config.base_gpa.into(),
Some(config.length),
)))?;
info!(
"GPPT Distributor initialized with base GPA {base_gpa:#x} and length \
{length:#x}",
base_gpa = config.base_gpa,
length = config.length
);
}
#[cfg(not(target_arch = "aarch64"))]
{
warn!(
"emu type: {} is not supported on this platform",
config.emu_type
);
}
}
EmulatedDeviceType::GPPTITS => {
#[cfg(target_arch = "aarch64")]
{
let host_gits_base = config
.cfg_list
.first()
.copied()
.map(PhysAddr::from_usize)
.expect("expect 1 arg for gppt its (host_gits_base)");
#[allow(clippy::arc_with_non_send_sync)]
this.add_mmio_dev(Arc::new(arm_vgic::v3::gits::Gits::new(
config.base_gpa.into(),
Some(config.length),
host_gits_base,
false,
)))?;
info!(
"GPPT ITS initialized with base GPA {base_gpa:#x} and length \
{length:#x}, host GITS base {host_gits_base:#x}",
base_gpa = config.base_gpa,
length = config.length,
host_gits_base = host_gits_base
);
}
#[cfg(not(target_arch = "aarch64"))]
{
warn!(
"emu type: {} is not supported on this platform",
config.emu_type
);
}
}
EmulatedDeviceType::PPPTGlobal => {
#[cfg(target_arch = "riscv64")]
{
let context_num = config
.cfg_list
.first()
.copied()
.expect("expect 1 arg for pppt global (context_num)");
this.add_mmio_dev(Arc::new(VPlicGlobal::new(
config.base_gpa.into(),
Some(config.length),
context_num, )))?;
info!(
"Partial PLIC Passthrough Global initialized with base GPA {:#x} and \
length {:#x}",
config.base_gpa, config.length
);
}
#[cfg(not(target_arch = "riscv64"))]
{
warn!(
"emu type: {} is not supported on this platform",
config.emu_type
);
}
}
EmulatedDeviceType::Console => {
#[cfg(target_arch = "x86_64")]
{
let serial = Arc::new(EmulatedSerialPort::new());
this.add_port_dev(serial.clone())?;
this.x86_serial = Some(serial);
info!("x86 16550 serial initialized for ports 0x3f8..=0x3ff");
}
#[cfg(not(target_arch = "x86_64"))]
{
warn!(
"emu type: {} is not supported on this platform",
config.emu_type
);
}
}
EmulatedDeviceType::X86IoApic => {
#[cfg(target_arch = "x86_64")]
{
let ioapic = Arc::new(EmulatedIoApic::new(
config.base_gpa.into(),
Some(config.length),
));
this.add_mmio_dev(ioapic.clone())?;
this.x86_ioapic = Some(ioapic);
info!(
"x86 IO APIC initialized with base GPA {:#x} and length {:#x}",
config.base_gpa, config.length
);
}
#[cfg(not(target_arch = "x86_64"))]
{
warn!(
"emu type: {} is not supported on this platform",
config.emu_type
);
}
}
EmulatedDeviceType::X86Pit => {
#[cfg(target_arch = "x86_64")]
{
let pit = Arc::new(EmulatedPit::new());
this.add_port_dev(pit.clone())?;
this.x86_pit = Some(pit);
info!("x86 PIT initialized for ports 0x40..=0x43 and 0x61");
}
#[cfg(not(target_arch = "x86_64"))]
{
warn!(
"emu type: {} is not supported on this platform",
config.emu_type
);
}
}
EmulatedDeviceType::IVCChannel => {
if this.ivc_channel.is_none() {
this.ivc_channel = Some(Mutex::new(RangeAllocator::new(Range {
start: config.base_gpa,
end: config.base_gpa + config.length,
})));
info!(
"IVCChannel initialized with base GPA {base_gpa:#x} and length \
{length:#x}",
base_gpa = config.base_gpa,
length = config.length
);
} else {
warn!("IVCChannel already initialized, ignoring additional config");
}
}
_ => {
warn!(
"Emulated device {}'s type {:?} is not supported yet",
config.name, config.emu_type
);
}
}
}
Ok(())
}
pub fn alloc_ivc_channel(&self, size: usize) -> AxResult<GuestPhysAddr> {
if size == 0 {
return ax_err!(InvalidInput, "Size must be greater than 0");
}
if !is_aligned_4k(size) {
return ax_err!(InvalidInput, "Size must be aligned to 4K");
}
if let Some(allocator) = &self.ivc_channel {
allocator
.lock()
.allocate_range(size)
.ok_or_else(|| {
warn!("Failed to allocate IVC channel range with size {size:#x}");
ax_errno::ax_err_type!(NoMemory, "IVC channel allocation failed")
})
.map(|range| {
debug!("Allocated IVC channel range: {range:x?}");
GuestPhysAddr::from_usize(range.start)
})
} else {
ax_err!(InvalidInput, "IVC channel not exists")
}
}
pub fn release_ivc_channel(&self, addr: GuestPhysAddr, size: usize) -> AxResult {
if size == 0 {
return ax_err!(InvalidInput, "Size must be greater than 0");
}
if !is_aligned_4k(size) {
return ax_err!(InvalidInput, "Size must be aligned to 4K");
}
if let Some(allocator) = &self.ivc_channel {
let range = addr.as_usize()..addr.as_usize() + size;
if allocator.lock().free_range(range.clone()) {
debug!("Released IVC channel range: {range:x?}");
Ok(())
} else {
ax_err!(InvalidInput, "Invalid IVC channel range")
}
} else {
ax_err!(InvalidInput, "IVC channel not exists")
}
}
pub fn register_bundle(&mut self, bundle: DeviceBundle) -> AxResult {
self.emu_mmio_devices.validate_devices(&bundle.mmio)?;
self.emu_port_devices.validate_devices(&bundle.port)?;
self.emu_sys_reg_devices.validate_devices(&bundle.sysreg)?;
for (index, pollable) in bundle.pollable.iter().enumerate() {
if self
.pollable_devices
.iter()
.chain(bundle.pollable[..index].iter())
.any(|existing| Arc::ptr_eq(existing, pollable))
{
return ax_err!(
AlreadyExists,
"failed to register pollable device: the same capability is already registered"
);
}
}
self.emu_mmio_devices.commit_devices(bundle.mmio);
self.emu_port_devices.commit_devices(bundle.port);
self.emu_sys_reg_devices.commit_devices(bundle.sysreg);
self.pollable_devices.extend(bundle.pollable);
Ok(())
}
pub fn add_mmio_dev(&mut self, dev: Arc<dyn BaseMmioDeviceOps>) -> AxResult {
self.register_bundle(DeviceRegistration::Mmio(dev).into())
}
pub fn add_sys_reg_dev(&mut self, dev: Arc<dyn BaseSysRegDeviceOps>) -> AxResult {
self.register_bundle(DeviceRegistration::SysReg(dev).into())
}
pub fn add_port_dev(&mut self, dev: Arc<dyn BasePortDeviceOps>) -> AxResult {
self.register_bundle(DeviceRegistration::Port(dev).into())
}
pub fn iter_mmio_dev(&self) -> impl Iterator<Item = &Arc<dyn BaseMmioDeviceOps>> {
self.emu_mmio_devices.iter()
}
pub fn iter_sys_reg_dev(&self) -> impl Iterator<Item = &Arc<dyn BaseSysRegDeviceOps>> {
self.emu_sys_reg_devices.iter()
}
pub fn iter_port_dev(&self) -> impl Iterator<Item = &Arc<dyn BasePortDeviceOps>> {
self.emu_port_devices.iter()
}
pub fn iter_pollable_dev(&self) -> impl Iterator<Item = &Arc<dyn PollableDeviceOps>> {
self.pollable_devices.iter()
}
#[cfg(target_arch = "x86_64")]
pub fn x86_ioapic_vector_for_gsi(&self, gsi: usize) -> Option<u8> {
self.x86_ioapic
.as_ref()
.and_then(|ioapic| ioapic.vector_for_gsi(gsi))
}
#[cfg(target_arch = "x86_64")]
pub fn x86_ioapic_assert_gsi(&self, gsi: usize) -> Option<IoApicInterrupt> {
self.x86_ioapic
.as_ref()
.and_then(|ioapic| ioapic.assert_gsi(gsi))
}
#[cfg(target_arch = "x86_64")]
pub fn x86_ioapic_end_of_interrupt(&self, vector: u8) -> Option<IoApicInterrupt> {
self.x86_ioapic
.as_ref()
.and_then(|ioapic| ioapic.end_of_interrupt(vector))
}
#[cfg(target_arch = "x86_64")]
pub fn x86_pit_consume_irq0_if_due(&self, now_ns: u64) -> bool {
self.x86_pit
.as_ref()
.is_some_and(|pit| pit.consume_irq0_if_due(now_ns))
}
#[cfg(target_arch = "x86_64")]
pub fn x86_serial_poll_irq(&self) -> bool {
self.x86_serial
.as_ref()
.is_some_and(|serial| serial.poll_irq())
}
pub fn iter_mut_mmio_dev(&mut self) -> impl Iterator<Item = &mut Arc<dyn BaseMmioDeviceOps>> {
self.emu_mmio_devices.iter_mut()
}
pub fn iter_mut_sys_reg_dev(
&mut self,
) -> impl Iterator<Item = &mut Arc<dyn BaseSysRegDeviceOps>> {
self.emu_sys_reg_devices.iter_mut()
}
pub fn iter_mut_port_dev(&mut self) -> impl Iterator<Item = &mut Arc<dyn BasePortDeviceOps>> {
self.emu_port_devices.iter_mut()
}
pub fn find_mmio_dev(&self, ipa: GuestPhysAddr) -> Option<Arc<dyn BaseMmioDeviceOps>> {
self.emu_mmio_devices.find_dev(ipa)
}
pub fn find_sys_reg_dev(
&self,
sys_reg_addr: SysRegAddr,
) -> Option<Arc<dyn BaseSysRegDeviceOps>> {
self.emu_sys_reg_devices.find_dev(sys_reg_addr)
}
pub fn find_port_dev(&self, port: Port) -> Option<Arc<dyn BasePortDeviceOps>> {
self.emu_port_devices.find_dev(port)
}
pub fn handle_mmio_read(&self, addr: GuestPhysAddr, width: AccessWidth) -> AxResult<usize> {
if let Some(emu_dev) = self.find_mmio_dev(addr) {
log_device_io("mmio", addr, emu_dev.address_range(), true, width);
return emu_dev.handle_read(addr, width);
}
panic_device_not_found("mmio", addr, true, width);
}
pub fn handle_mmio_write(
&self,
addr: GuestPhysAddr,
width: AccessWidth,
val: usize,
) -> AxResult {
if let Some(emu_dev) = self.find_mmio_dev(addr) {
log_device_io("mmio", addr, emu_dev.address_range(), false, width);
return emu_dev.handle_write(addr, width, val);
}
panic_device_not_found("mmio", addr, false, width);
}
pub fn handle_sys_reg_read(&self, addr: SysRegAddr, width: AccessWidth) -> AxResult<usize> {
if let Some(emu_dev) = self.find_sys_reg_dev(addr) {
log_device_io("sys_reg", addr.0, emu_dev.address_range(), true, width);
return emu_dev.handle_read(addr, width);
}
panic_device_not_found("sys_reg", addr, true, width);
}
pub fn handle_sys_reg_write(
&self,
addr: SysRegAddr,
width: AccessWidth,
val: usize,
) -> AxResult {
if let Some(emu_dev) = self.find_sys_reg_dev(addr) {
log_device_io("sys_reg", addr.0, emu_dev.address_range(), false, width);
return emu_dev.handle_write(addr, width, val);
}
panic_device_not_found("sys_reg", addr, false, width);
}
pub fn handle_port_read(&self, port: Port, width: AccessWidth) -> AxResult<usize> {
if let Some(emu_dev) = self.find_port_dev(port) {
log_device_io("port", port.0, emu_dev.address_range(), true, width);
return emu_dev.handle_read(port, width);
}
panic_device_not_found("port", port, true, width);
}
pub fn handle_port_write(&self, port: Port, width: AccessWidth, val: usize) -> AxResult {
if let Some(emu_dev) = self.find_port_dev(port) {
log_device_io("port", port.0, emu_dev.address_range(), false, width);
return emu_dev.handle_write(port, width, val);
}
panic_device_not_found("port", port, false, width);
}
}