alioth 0.12.0

// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#[cfg(target_arch = "aarch64")]
#[path = "vcpu_aarch64.rs"]
mod aarch64;
#[cfg(target_arch = "x86_64")]
#[path = "vcpu_x86_64/vcpu_x86_64.rs"]
mod x86_64;

mod vmentry;
mod vmexit;

#[cfg(target_arch = "x86_64")]
use std::arch::x86_64::CpuidResult;
#[cfg(target_arch = "x86_64")]
use std::collections::HashMap;
use std::ops::{Deref, DerefMut};
use std::os::fd::{AsRawFd, OwnedFd};
use std::ptr::null_mut;
use std::sync::Arc;

use libc::{MAP_FAILED, MAP_SHARED, PROT_READ, PROT_WRITE, mmap, munmap};
use snafu::ResultExt;

#[cfg(target_arch = "x86_64")]
use crate::arch::cpuid::CpuidIn;
#[cfg(target_arch = "x86_64")]
use crate::arch::reg::{DtReg, DtRegVal, SegReg, SegRegVal};
use crate::arch::reg::{Reg, SReg};
use crate::ffi;
use crate::hv::kvm::vm::{KvmVm, VmInner};
use crate::hv::kvm::{KvmError, kvm_error};
use crate::hv::{Error, Result, Vcpu, VmEntry, VmExit, error};
use crate::sys::kvm::{KvmExit, KvmRun, kvm_run};

#[cfg(target_arch = "aarch64")]
use self::aarch64::VcpuArch;
#[cfg(target_arch = "x86_64")]
use self::x86_64::VcpuArch;

struct KvmRunBlock {
    addr: usize,
    size: usize,
}

impl KvmRunBlock {
    unsafe fn new(fd: &OwnedFd, mmap_size: usize) -> Result<KvmRunBlock, KvmError> {
        let prot = PROT_READ | PROT_WRITE;
        let addr = ffi!(
            unsafe { mmap(null_mut(), mmap_size, prot, MAP_SHARED, fd.as_raw_fd(), 0,) },
            MAP_FAILED
        )
        .context(kvm_error::MmapVcpuFd)?;
        Ok(KvmRunBlock {
            addr: addr as usize,
            size: mmap_size,
        })
    }

    #[cfg(target_arch = "x86_64")]
    unsafe fn data_slice<T>(&self, offset: usize, count: usize) -> &[T] {
        unsafe { std::slice::from_raw_parts((self.addr + offset) as *const T, count) }
    }

    #[cfg(target_arch = "x86_64")]
    unsafe fn data_slice_mut<T>(&mut self, offset: usize, count: usize) -> &mut [T] {
        unsafe { std::slice::from_raw_parts_mut((self.addr + offset) as *mut T, count) }
    }
}

impl Deref for KvmRunBlock {
    type Target = KvmRun;

    fn deref(&self) -> &Self::Target {
        unsafe { &*(self.addr as *const Self::Target) }
    }
}

impl DerefMut for KvmRunBlock {
    fn deref_mut(&mut self) -> &mut Self::Target {
        unsafe { &mut *(self.addr as *mut Self::Target) }
    }
}

impl Drop for KvmRunBlock {
    fn drop(&mut self) {
        if let Err(e) = ffi!(unsafe { munmap(self.addr as _, self.size) }) {
            log::error!("unmap kvm_run: {e}")
        }
    }
}

pub struct KvmVcpu {
    kvm_run: KvmRunBlock,
    fd: OwnedFd,
    arch: VcpuArch,
    #[allow(dead_code)]
    vm: Arc<VmInner>,
}

impl KvmVcpu {
    pub fn new(vm: &KvmVm, index: u16, identity: u64) -> Result<Self> {
        let vcpu_fd = Self::create_vcpu(vm, index, identity)?;
        let kvm_run = unsafe { KvmRunBlock::new(&vcpu_fd, vm.vm.vcpu_mmap_size) }?;
        let arch = VcpuArch::new(identity);
        Ok(KvmVcpu {
            fd: vcpu_fd,
            kvm_run,
            vm: vm.vm.clone(),
            arch,
        })
    }
}

impl Vcpu for KvmVcpu {
    #[cfg(target_arch = "aarch64")]
    fn reset(&mut self, is_bsp: bool) -> Result<(), Error> {
        self.kvm_vcpu_init(is_bsp)
    }

    fn get_reg(&self, reg: Reg) -> Result<u64, Error> {
        self.kvm_get_reg(reg)
    }

    #[cfg(target_arch = "x86_64")]
    fn get_dt_reg(&self, reg: DtReg) -> Result<DtRegVal, Error> {
        self.kvm_get_dt_reg(reg)
    }

    #[cfg(target_arch = "x86_64")]
    fn get_seg_reg(&self, reg: SegReg) -> Result<SegRegVal, Error> {
        self.kvm_get_seg_reg(reg)
    }

    fn get_sreg(&self, reg: SReg) -> Result<u64, Error> {
        self.kvm_get_sreg(reg)
    }

    fn set_regs(&mut self, vals: &[(Reg, u64)]) -> Result<(), Error> {
        self.kvm_set_regs(vals)
    }

    #[cfg(target_arch = "x86_64")]
    fn set_sregs(
        &mut self,
        sregs: &[(SReg, u64)],
        seg_regs: &[(SegReg, SegRegVal)],
        dt_regs: &[(DtReg, DtRegVal)],
    ) -> Result<(), Error> {
        self.kvm_set_sregs(sregs, seg_regs, dt_regs)
    }

    #[cfg(target_arch = "aarch64")]
    fn set_sregs(&mut self, sregs: &[(SReg, u64)]) -> Result<(), Error> {
        self.kvm_set_sregs(sregs)
    }

    fn run(&mut self, entry: VmEntry) -> Result<VmExit, Error> {
        match entry {
            VmEntry::None => {}
            #[cfg(target_arch = "x86_64")]
            VmEntry::Io { data } => {
                let r = self.entry_io(data);
                if let Some(exit) = r {
                    return Ok(exit);
                }
            }
            VmEntry::Mmio { data } => self.entry_mmio(data),
            VmEntry::Shutdown | VmEntry::Reboot | VmEntry::Pause => self.set_immediate_exit(true),
        };
        let ret = unsafe { kvm_run(&self.fd) };
        match ret {
            Err(e) => match (e.raw_os_error(), entry) {
                (Some(libc::EAGAIN), _) => Ok(VmExit::Interrupted),
                (Some(libc::EINTR), VmEntry::Shutdown) => {
                    self.set_immediate_exit(false);
                    Ok(VmExit::Shutdown)
                }
                (Some(libc::EINTR), VmEntry::Reboot) => {
                    self.set_immediate_exit(false);
                    Ok(VmExit::Reboot)
                }
                (Some(libc::EINTR), VmEntry::Pause) => {
                    #[cfg(target_arch = "x86_64")]
                    if let Err(e) = self.kvmclock_ctrl() {
                        log::error!("Failed to control kvmclock: {e:?}");
                    }
                    self.set_immediate_exit(false);
                    Ok(VmExit::Paused)
                }
                (Some(libc::EINTR), _) => Ok(VmExit::Interrupted),
                (Some(libc::EFAULT), _) if self.kvm_run.exit_reason == KvmExit::MEMORY_FAULT => {
                    Ok(self.handle_memory_fault())
                }
                _ => Err(e).context(error::RunVcpu),
            },
            Ok(_) => match self.kvm_run.exit_reason {
                #[cfg(target_arch = "x86_64")]
                KvmExit::IO => Ok(self.handle_io()),
                KvmExit::HYPERCALL => self.handle_hypercall(),
                KvmExit::MMIO => self.handle_mmio(),
                KvmExit::SHUTDOWN => Ok(VmExit::Shutdown),
                KvmExit::SYSTEM_EVENT => self.handle_system_event(),
                reason => error::VmExit {
                    msg: format!("unkown kvm exit: {reason:#x?}"),
                }
                .fail(),
            },
        }
    }

    #[cfg(target_arch = "x86_64")]
    fn set_cpuids(&mut self, cpuids: HashMap<CpuidIn, CpuidResult>) -> Result<(), Error> {
        self.kvm_set_cpuids(&cpuids)
    }

    #[cfg(target_arch = "x86_64")]
    fn set_msrs(&mut self, msrs: &[(u32, u64)]) -> Result<()> {
        self.kvm_set_msrs(msrs)
    }

    fn dump(&self) -> Result<(), Error> {
        Ok(())
    }

    #[cfg(target_arch = "x86_64")]
    fn tdx_init_vcpu(&self, hob: u64) -> Result<()> {
        KvmVcpu::tdx_init_vcpu(self, hob)
    }

    #[cfg(target_arch = "x86_64")]
    fn tdx_init_mem_region(&self, data: &[u8], gpa: u64, measure: bool) -> Result<()> {
        KvmVcpu::tdx_init_mem_region(self, data, gpa, measure)
    }
}