ax-plat 0.14.0

//! Interrupt request (IRQ) handling.

use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering};

use ax_lazyinit::OnceLock;
pub use irq_framework::{
    AcpiGsiController, AcpiGsiRoute, AcpiIrqPolarity, AcpiIrqTrigger, AutoEnable, BoxedIrqHandler,
    CpuId, CpuMask, HwIrq, IrqAffinity, IrqContext, IrqDomainId, IrqError, IrqExecution, IrqHandle,
    IrqId, IrqOps, IrqOrigin, IrqOutcome, IrqRequest, IrqReturn, IrqScope, IrqSource, IrqStatus,
    IrqTrigger, Registry, ShareMode, TrapVector,
};

#[cfg(target_arch = "loongarch64")]
pub mod loongarch64_hv;
#[cfg(target_arch = "loongarch64")]
pub use loongarch64_hv::LoongArchHvIrqIf;
#[cfg(target_arch = "riscv64")]
pub mod riscv64_hv;
#[cfg(target_arch = "riscv64")]
pub use riscv64_hv::RiscvHvIrqIf;

/// Compatibility IRQ domain used while non-domainized platforms migrate.
pub const LEGACY_IRQ_DOMAIN: IrqDomainId = IrqDomainId(0);

/// CPU-local interrupt domain for architecture trap causes such as timers/IPIs.
pub const CPU_LOCAL_IRQ_DOMAIN: IrqDomainId = IrqDomainId(u16::MAX);

/// x86 local APIC interrupt domain.
pub const X86_LAPIC_DOMAIN: IrqDomainId = IrqDomainId(1);

/// x86 I/O APIC interrupt domain.
pub const X86_IOAPIC_DOMAIN: IrqDomainId = IrqDomainId(2);

/// AArch64 GIC interrupt domain.
pub const AARCH64_GIC_DOMAIN: IrqDomainId = IrqDomainId(3);

/// RISC-V PLIC interrupt domain.
pub const RISCV_PLIC_DOMAIN: IrqDomainId = IrqDomainId(4);

/// LoongArch EIOINTC interrupt domain.
pub const LOONGARCH_EIOINTC_DOMAIN: IrqDomainId = IrqDomainId(5);

/// LoongArch PCH-PIC interrupt domain.
pub const LOONGARCH_PCH_PIC_DOMAIN: IrqDomainId = IrqDomainId(6);

/// Creates a legacy IRQ id without truncating the raw IRQ number.
pub fn try_legacy_irq(raw: usize) -> Result<IrqId, IrqError> {
    let hwirq = u32::try_from(raw).map_err(|_| IrqError::InvalidIrq)?;
    Ok(IrqId::new(LEGACY_IRQ_DOMAIN, HwIrq(hwirq)))
}

/// Compatibility constructor for legacy numeric IRQ users.
pub fn legacy_irq(raw: usize) -> Result<IrqId, IrqError> {
    try_legacy_irq(raw)
}

/// Returns the legacy raw IRQ number when this id is in the legacy domain.
pub const fn legacy_irq_raw(irq: IrqId) -> Option<usize> {
    if irq.domain.0 == LEGACY_IRQ_DOMAIN.0 {
        Some(irq.hwirq.0 as usize)
    } else {
        None
    }
}

/// Legacy constructor kept only for upper-layer compatibility.
#[allow(non_snake_case)]
pub fn IrqNumber(raw: usize) -> Result<IrqId, IrqError> {
    legacy_irq(raw)
}

/// Raw synchronous cross-CPU call used by the IRQ registry.
pub type RunOnCpuSync = unsafe fn(usize, unsafe fn(*mut ()), *mut ()) -> Result<(), IrqError>;

static RUN_ON_CPU_SYNC: AtomicUsize = AtomicUsize::new(0);

/// Installs the runtime-provided synchronous cross-CPU call implementation.
pub fn set_run_on_cpu_sync(run_on_cpu_sync: RunOnCpuSync) {
    RUN_ON_CPU_SYNC.store(run_on_cpu_sync as usize, Ordering::Release);
}

/// Runs a raw thunk synchronously on the requested CPU.
///
/// This is the generic owner-CPU execution bridge used by device runtimes that
/// must keep register access on one non-reentrant CPU context.
///
/// # Safety
///
/// `arg` must stay valid until this function returns, and `f` must be safe to
/// execute in the target CPU's IRQ/IPI context.
pub unsafe fn run_on_cpu_sync(
    cpu: CpuId,
    f: unsafe fn(*mut ()),
    arg: *mut (),
) -> Result<(), IrqError> {
    PlatIrqOps.run_on_cpu_sync(cpu, f, arg)
}

struct PlatIrqOps;

impl IrqOps for PlatIrqOps {
    type LocalIrqState = usize;

    fn current_cpu(&self) -> CpuId {
        CpuId(crate::percpu::this_cpu_id())
    }

    fn cpu_online(&self, cpu: CpuId) -> bool {
        is_cpu_online(cpu.0)
    }

    fn in_irq_context(&self) -> bool {
        crate::irq::in_irq_context()
    }

    fn local_irq_save(&self) -> Self::LocalIrqState {
        ax_sync::irq_save_and_disable()
    }

    fn local_irq_restore(&self, state: Self::LocalIrqState) {
        // SAFETY: `state` was returned by the matching save on this CPU.
        unsafe { ax_sync::irq_restore(state) };
    }

    fn run_on_cpu_sync(
        &self,
        cpu: CpuId,
        f: unsafe fn(*mut ()),
        arg: *mut (),
    ) -> Result<(), IrqError> {
        if cpu == self.current_cpu() {
            unsafe { f(arg) };
            Ok(())
        } else {
            let run_on_cpu_sync = RUN_ON_CPU_SYNC.load(Ordering::Acquire);
            if run_on_cpu_sync == 0 {
                return Err(IrqError::Unsupported);
            }
            let run_on_cpu_sync =
                unsafe { core::mem::transmute::<usize, RunOnCpuSync>(run_on_cpu_sync) };
            unsafe { run_on_cpu_sync(cpu.0, f, arg) }
        }
    }

    fn set_enabled(&self, irq: IrqId, _cpu: Option<CpuId>, enabled: bool) -> Result<(), IrqError> {
        set_enable(irq, enabled)
    }

    fn set_affinity(&self, irq: IrqId, affinity: IrqAffinity) -> Result<(), IrqError> {
        set_affinity(irq, affinity)
    }

    fn is_enabled(&self, _irq: IrqId, _cpu: Option<CpuId>) -> Result<bool, IrqError> {
        Err(IrqError::Unsupported)
    }

    fn is_pending(&self, _irq: IrqId, _cpu: Option<CpuId>) -> Result<bool, IrqError> {
        Err(IrqError::Unsupported)
    }

    fn is_in_service(&self, _irq: IrqId, _cpu: Option<CpuId>) -> Result<bool, IrqError> {
        Err(IrqError::Unsupported)
    }

    fn relax(&self) {
        core::hint::spin_loop();
    }
}

static IRQ_REGISTRY: OnceLock<Registry<PlatIrqOps>> = OnceLock::new();
static ONLINE_CPUS: AtomicUsize = AtomicUsize::new(0);

#[ax_percpu::def_percpu]
static IRQ_CONTEXT_DEPTH: AtomicU32 = AtomicU32::new(0);

fn registry() -> &'static Registry<PlatIrqOps> {
    IRQ_REGISTRY.call_once(|| Registry::new(PlatIrqOps))
}

/// Returns whether the current CPU is dispatching an IRQ action.
pub fn in_irq_context() -> bool {
    let irq_state = ax_sync::irq_save_and_disable();
    // SAFETY: local IRQ exclusion prevents a scheduler safe point and therefore
    // fixes the selected CPU for this single owner-local read.
    let active = unsafe { ax_percpu::with_cpu_pin(in_irq_context_pinned) }
        .expect("the current CPU-local area must remain bound");
    // SAFETY: this is the matching restore for the state saved above.
    unsafe { ax_sync::irq_restore(irq_state) };
    active
}

/// Tests IRQ-action context through an existing current-CPU pin.
#[doc(hidden)]
#[inline(always)]
pub fn in_irq_context_pinned(pin: &ax_percpu::CpuPin<'_>) -> bool {
    IRQ_CONTEXT_DEPTH.with_current(pin, |depth| depth.load(Ordering::Relaxed) != 0)
}

/// Requests an IRQ action through the dynamic IRQ framework.
pub fn request_irq(irq: IrqId, request: IrqRequest) -> Result<IrqHandle, IrqError> {
    let auto_enable = request.auto_enable_mode();
    let handle = registry().request(irq, request)?;
    if auto_enable == AutoEnable::Yes
        && let Err(err) = registry().enable(handle)
    {
        let _ = registry().free(handle);
        return Err(err);
    }
    Ok(handle)
}

/// Requests a shared IRQ action.
pub fn request_shared_irq(
    irq: IrqId,
    handler: impl FnMut(IrqContext) -> IrqReturn + Send + 'static,
) -> Result<IrqHandle, IrqError> {
    request_irq(irq, IrqRequest::new(handler).share_mode(ShareMode::Shared))
}

/// Requests a per-CPU IRQ action.
pub fn request_percpu_irq(
    irq: IrqId,
    cpus: CpuMask,
    handler: impl Fn(IrqContext) -> IrqReturn + Send + Sync + 'static,
) -> Result<IrqHandle, IrqError> {
    request_irq(
        irq,
        IrqRequest::new_concurrent(handler).scope(IrqScope::PerCpu { cpus }),
    )
}

/// Frees an IRQ action.
pub fn free_irq(handle: IrqHandle) -> Result<(), IrqError> {
    registry().free(handle)
}

/// Enables an IRQ action.
pub fn enable_irq(handle: IrqHandle) -> Result<(), IrqError> {
    registry().enable(handle)
}

/// Disables an IRQ action.
pub fn disable_irq(handle: IrqHandle) -> Result<(), IrqError> {
    registry().disable(handle)
}

/// Waits until no handler for this IRQ descriptor is in flight.
pub fn synchronize_irq(handle: IrqHandle) -> Result<(), IrqError> {
    registry().synchronize(handle)
}

/// Returns the status of an IRQ action.
pub fn irq_status(handle: IrqHandle) -> Result<IrqStatus, IrqError> {
    registry().status(handle)
}

/// Marks a CPU online for pending per-CPU IRQ enables.
pub fn cpu_online(cpu: usize) -> Result<(), IrqError> {
    if cpu >= usize::BITS as usize {
        return Err(IrqError::InvalidCpu);
    }
    ONLINE_CPUS.fetch_or(1usize << cpu, Ordering::AcqRel);
    registry().cpu_online(CpuId(cpu))
}

/// Returns whether a CPU has entered the platform IRQ runtime.
pub fn is_cpu_online(cpu: usize) -> bool {
    cpu < usize::BITS as usize && (ONLINE_CPUS.load(Ordering::Acquire) & (1usize << cpu)) != 0
}

/// Prepares CPU-local runtime state before the common IRQ guard is entered.
pub fn prepare_irq_context(vector: TrapVector) {
    ax_crate_interface::call_interface!(IrqIf::prepare, vector)
}

/// Dispatches actions registered in the dynamic IRQ framework on `cpu`.
pub fn dispatch_irq_on(irq: IrqId, cpu: CpuId, origin: IrqOrigin) -> IrqOutcome {
    // SAFETY: IRQ dispatch runs with local interrupts disabled. The caller's
    // explicit CPU identity and the installed current area must describe the
    // same non-migrating owner for the complete action callback.
    unsafe {
        ax_percpu::with_cpu_pin(|pin| {
            assert_eq!(
                ax_percpu::current_cpu_index(pin).as_usize(),
                cpu.0,
                "IRQ dispatch CPU must match the current CPU-local owner"
            );
            let depth =
                IRQ_CONTEXT_DEPTH.with_current(pin, |depth| depth.fetch_add(1, Ordering::Relaxed));
            assert_ne!(depth, u32::MAX, "IRQ action nesting overflow");
            let outcome = registry().dispatch(irq, cpu, origin);
            IRQ_CONTEXT_DEPTH.with_current(pin, |depth| {
                let previous = depth.fetch_sub(1, Ordering::Relaxed);
                assert_ne!(previous, 0, "IRQ action exit without a matching entry");
            });
            outcome
        })
    }
    .expect("IRQ dispatch requires an installed current CPU-local area")
}

fn dispatch_with_controller_ack<T>(
    acknowledge_controller: impl FnOnce(),
    dispatch: impl FnOnce() -> T,
) -> T {
    acknowledge_controller();
    dispatch()
}

/// Acknowledges one physical IPI before dispatching its logical actions.
///
/// IPI actions may make a coalesced software delivery edge reusable. The
/// controller must therefore stop treating the previous IPI as pending or in
/// service before any action can publish a fresh physical notification.
pub fn dispatch_ipi_irq_on(
    irq: IrqId,
    cpu: CpuId,
    origin: IrqOrigin,
    acknowledge_controller: impl FnOnce(),
) -> IrqOutcome {
    dispatch_with_controller_ack(acknowledge_controller, || dispatch_irq_on(irq, cpu, origin))
}

/// Dispatches actions registered in the dynamic IRQ framework.
pub fn dispatch_irq(irq: IrqId, origin: IrqOrigin) -> IrqOutcome {
    dispatch_irq_on(irq, PlatIrqOps.current_cpu(), origin)
}

/// Tests one explicitly selected CPU's IRQ-action publication.
///
/// Callers that use this for the current CPU must retain their own migration
/// exclusion across CPU identity resolution and this atomic observation.
#[doc(hidden)]
pub fn in_irq_context_on(cpu: CpuId) -> bool {
    let Ok(cpu) = ax_percpu::CpuIndex::try_from(cpu.0) else {
        return false;
    };
    let Ok(area) = ax_percpu::area(cpu) else {
        return false;
    };
    // SAFETY: the generated symbol pointer addresses an AtomicU32 in every
    // initialized CPU area, so shared remote observation is synchronized by
    // the atomic itself and never aliases a non-atomic access.
    unsafe { IRQ_CONTEXT_DEPTH.remote_ptr(area).as_ref() }.load(Ordering::Relaxed) != 0
}

/// Resolves a firmware/controller interrupt source to a framework IRQ id.
pub fn resolve_irq_source(source: IrqSource) -> Result<IrqId, IrqError> {
    resolve_source(source)
}

/// Resolves an architecture-local/per-CPU hardware interrupt through the
/// platform IRQ domain.
pub fn resolve_percpu_irq(hwirq: HwIrq) -> Result<IrqId, IrqError> {
    resolve_percpu(hwirq)
}

/// Target specification for one inter-processor interrupt (IPI) delivery.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum IpiTarget {
    /// Send to the current CPU.
    Current,
    /// Send to a specific CPU.
    Cpu(CpuId),
}

/// IRQ management interface.
#[def_plat_interface]
pub trait IrqIf {
    /// Prepares CPU-local runtime state before the common IRQ handler touches
    /// per-CPU runtime data.
    fn prepare(vector: TrapVector);

    /// Initializes boot-time IRQ controller domains before runtime IRQ handlers
    /// are registered.
    fn init_boot_irqs(cpu_id: usize) -> Result<(), IrqError>;

    /// Initializes early IRQ state for a secondary CPU.
    #[cfg(feature = "smp")]
    fn init_secondary_boot_irqs(cpu_id: usize) -> Result<(), IrqError>;

    /// Enables or disables the given IRQ.
    fn set_enable(irq: IrqId, enabled: bool) -> Result<(), IrqError>;

    /// Configures the trigger mode of the given IRQ.
    fn set_trigger(irq: IrqId, trigger: IrqTrigger) -> Result<(), IrqError>;

    /// Routes a global IRQ to a fixed CPU when supported.
    fn set_affinity(irq: IrqId, affinity: IrqAffinity) -> Result<(), IrqError>;

    /// Handles the IRQ.
    ///
    /// It is called by the common interrupt handler. Platform implementations
    /// should claim/ack the controller interrupt, dispatch the real IRQ through
    /// [`dispatch_irq`], and perform the matching EOI/complete operation. IPI
    /// actions must instead be entered through [`dispatch_ipi_irq_on`] so the
    /// architecture-specific acknowledgement precedes logical IPI draining.
    ///
    /// Returns the "real" IRQ number. On some platforms, this may differ from
    /// the input `irq` number, for example on AArch64 the input `irq` is
    /// ignored and the real IRQ number is obtained from the GIC. Returns
    /// `None` if the IRQ is spurious.
    fn handle(vector: TrapVector, origin: IrqOrigin) -> Option<IrqId>;

    /// Sends an inter-processor interrupt (IPI) to one target CPU.
    ///
    /// The platform backend must order earlier Normal-memory publications on
    /// the calling CPU before the target can observe the interrupt.
    fn send_ipi(irq_num: IrqId, target: IpiTarget) -> Result<(), IrqError>;

    /// Returns the platform IRQ id used for runtime IPIs.
    fn ipi_irq() -> IrqId;

    /// Resolves a firmware/controller interrupt source to a framework IRQ id.
    fn resolve_source(source: IrqSource) -> Result<IrqId, IrqError>;

    /// Resolves an architecture-local/per-CPU hardware interrupt.
    fn resolve_percpu(hwirq: HwIrq) -> Result<IrqId, IrqError>;
}

#[cfg(test)]
mod tests {
    use core::{
        cell::RefCell,
        sync::atomic::{AtomicUsize, Ordering},
    };

    use super::*;
    use crate::impl_plat_interface;

    static ENABLE_CALLS: AtomicUsize = AtomicUsize::new(0);
    static FAIL_ENABLE: AtomicUsize = AtomicUsize::new(0);
    static FAIL_SEND_IPI: AtomicUsize = AtomicUsize::new(0);

    struct TestContextOps;

    #[ax_crate_interface::impl_interface]
    impl ax_sync::interface::ContextOps for TestContextOps {
        fn enter(_context: u8) -> ax_sync::interface::ContextState {
            ax_sync::interface::ContextState::new(0, 0)
        }

        fn exit(_context: u8, _state: ax_sync::interface::ContextState) {}

        fn irq_return_preempt_enter() -> usize {
            0
        }

        fn irq_return_preempt_exit(_state: usize) {}

        fn hardirq_enter() {}

        fn hardirq_exit() {}
    }

    struct TestIrqIf;

    #[impl_plat_interface]
    impl IrqIf for TestIrqIf {
        fn prepare(_vector: TrapVector) {}

        fn init_boot_irqs(_cpu_id: usize) -> Result<(), IrqError> {
            Ok(())
        }

        #[cfg(feature = "smp")]
        fn init_secondary_boot_irqs(_cpu_id: usize) -> Result<(), IrqError> {
            Ok(())
        }

        fn set_enable(_irq: IrqId, _enabled: bool) -> Result<(), IrqError> {
            ENABLE_CALLS.fetch_add(1, Ordering::Relaxed);
            if FAIL_ENABLE.load(Ordering::Relaxed) != 0 {
                return Err(IrqError::Controller);
            }
            Ok(())
        }

        fn set_trigger(_irq: IrqId, _trigger: IrqTrigger) -> Result<(), IrqError> {
            Ok(())
        }

        fn set_affinity(_irq: IrqId, _affinity: IrqAffinity) -> Result<(), IrqError> {
            Err(IrqError::Unsupported)
        }

        fn handle(_vector: TrapVector, _origin: IrqOrigin) -> Option<IrqId> {
            None
        }

        fn send_ipi(_irq_num: IrqId, _target: IpiTarget) -> Result<(), IrqError> {
            if FAIL_SEND_IPI.load(Ordering::Relaxed) != 0 {
                return Err(IrqError::Controller);
            }
            Ok(())
        }

        fn ipi_irq() -> IrqId {
            IrqId::new(CPU_LOCAL_IRQ_DOMAIN, HwIrq(0))
        }

        fn resolve_source(_source: IrqSource) -> Result<IrqId, IrqError> {
            Err(IrqError::Unsupported)
        }

        fn resolve_percpu(_hwirq: HwIrq) -> Result<IrqId, IrqError> {
            Err(IrqError::Unsupported)
        }
    }

    #[test]
    fn send_ipi_propagates_platform_delivery_error() {
        FAIL_SEND_IPI.store(1, Ordering::Relaxed);

        assert_eq!(
            send_ipi(
                IrqId::new(CPU_LOCAL_IRQ_DOMAIN, HwIrq(0)),
                IpiTarget::Current,
            ),
            Err(IrqError::Controller),
        );

        FAIL_SEND_IPI.store(0, Ordering::Relaxed);
    }

    #[test]
    fn ipi_controller_ack_precedes_logical_dispatch() {
        let events = RefCell::new(alloc::vec::Vec::new());

        super::dispatch_with_controller_ack(
            || events.borrow_mut().push("controller-ack"),
            || events.borrow_mut().push("logical-dispatch"),
        );

        assert_eq!(*events.borrow(), ["controller-ack", "logical-dispatch"]);
    }

    #[test]
    fn request_irq_auto_enable_no_does_not_enable_line() {
        let irq = IrqId::new(IrqDomainId(0xff), HwIrq(1));
        let request = IrqRequest::new(|_| IrqReturn::Handled).auto_enable(AutoEnable::No);

        ENABLE_CALLS.store(0, Ordering::Relaxed);
        let handle = request_irq(irq, request).unwrap();

        assert_eq!(ENABLE_CALLS.load(Ordering::Relaxed), 0);
        assert_eq!(irq_status(handle).unwrap().action_enabled, false);

        free_irq(handle).unwrap();
    }

    #[test]
    fn request_irq_rolls_back_action_when_auto_enable_fails() {
        let irq = IrqId::new(IrqDomainId(0xff), HwIrq(2));
        let request = || IrqRequest::new(|_| IrqReturn::Handled);

        ENABLE_CALLS.store(0, Ordering::Relaxed);
        FAIL_ENABLE.store(1, Ordering::Relaxed);
        let err = request_irq(irq, request()).unwrap_err();

        assert_eq!(err, IrqError::Controller);
        assert_eq!(ENABLE_CALLS.load(Ordering::Relaxed), 1);

        FAIL_ENABLE.store(0, Ordering::Relaxed);
        let handle = request_irq(irq, request()).unwrap();
        assert_eq!(irq_status(handle).unwrap().action_enabled, true);

        free_irq(handle).unwrap();
    }
}