use core::{
hint::{likely, unlikely},
sync::atomic::{AtomicBool, Ordering},
};
use bsp_define::irqchip::{IrqError, IrqType, IrqchipImpl, IrqchipIrqState, Result};
use semx_cpumask::Cpumask;
use crate::{
drivers::{readl_relaxed, writel_relaxed},
irq::{
handle_irq,
irqflags::{local_irq_restore, local_irq_save},
},
println,
processor::{isb, nr_cpus, smp_rmb, smp_wmb, this_processor_id},
smp::ipi::handle_ipi,
space::addr::Paddr,
sync::spinlock::Spinlock,
};
pub struct IrqchipGic {
dist_base: usize,
cpu_base: usize,
#[allow(unused)]
dist_size: usize,
#[allow(unused)]
cpu_size: usize,
gic_irqs: usize,
}
impl IrqchipGic {
pub const fn create(
dist_base: Paddr,
dist_size: usize,
cpu_base: Paddr,
cpu_size: usize,
) -> Self {
Self {
dist_base: dist_base.to_io_const().to_value(),
cpu_base: cpu_base.to_io_const().to_value(),
cpu_size,
dist_size,
gic_irqs: 1020,
}
}
fn gic_check_gicv2(base: usize) -> bool {
let val = readl_relaxed(base + GIC_CPU_IDENT);
val & 0xff0fff == 0x02043B
}
#[allow(clippy::unused_self)]
fn gic_check_eoimode(&self) -> bool {
false
}
fn gic_init_base(&self) {
if SUPPORTS_DEACTIVATE_KEY.load(Ordering::Relaxed) {
println!("GIC: Using split EOI/Deactivate mode");
}
self.gic_dist_init();
self.gic_cpu_init();
}
fn gic_dist_init(&self) {
writel_relaxed(self.dist_base + GIC_DIST_CTRL, GICD_DISABLE);
let mut cpumask = self.gic_get_cpumask();
cpumask |= cpumask << 8;
cpumask |= cpumask << 16;
let mut i = 32;
while i < self.gic_irqs {
writel_relaxed(self.dist_base + GIC_DIST_TARGET + i * 4 / 4, cpumask);
i += 4;
}
self.gic_dist_config();
writel_relaxed(self.dist_base + GIC_DIST_CTRL, GICD_ENABLE);
}
fn gic_get_cpumask(&self) -> u32 {
let mut mask = 0;
let mut i = 0;
while i < 32 {
mask = readl_relaxed(self.dist_base + GIC_DIST_TARGET + i);
mask |= mask >> 16;
mask |= mask >> 8;
if mask != 0 {
break;
}
i += 4;
}
assert!(
!(mask == 0 && nr_cpus() > 1),
"GIC CPU mask not found - kernel will fail to boot."
);
mask
}
fn gic_dist_config(&self) {
let mut i = 32;
let gic_irqs = self.gic_irqs;
while i < gic_irqs {
writel_relaxed(self.dist_base + GIC_DIST_CONFIG + i / 4, GICD_INT_ACTLOW_LVLTRIG);
i += 16;
}
let mut i = 32;
while i < gic_irqs {
writel_relaxed(self.dist_base + GIC_DIST_PRI + i, GICD_INT_DEF_PRI_X4);
i += 4;
}
let mut i = 32;
while i < gic_irqs {
writel_relaxed(self.dist_base + GIC_DIST_ACTIVE_CLEAR + i / 8, GICD_INT_EN_CLR_X32);
writel_relaxed(self.dist_base + GIC_DIST_ENABLE_CLEAR + i / 8, GICD_INT_EN_CLR_X32);
i += 32;
}
}
#[allow(static_mut_refs)]
fn gic_cpu_init(&self) {
let cpu = this_processor_id();
assert!(cpu < NR_GIC_CPU_IF);
let cpumask = self.gic_get_cpumask();
unsafe {
GIC_CPU_MAP[cpu] = cpumask as u8;
}
unsafe {
for (i, item) in GIC_CPU_MAP.iter_mut().enumerate().take(NR_GIC_CPU_IF) {
if i != cpu {
*item &= !(cpumask as u8);
}
}
}
IrqchipGic::gic_cpu_config(self.dist_base);
writel_relaxed(self.cpu_base + GIC_CPU_PRIMASK, GICC_INT_PRI_THRESHOLD);
self.gic_cpu_if_up();
}
fn gic_cpu_config(base: usize) {
writel_relaxed(base + GIC_DIST_ACTIVE_CLEAR, GICD_INT_EN_CLR_X32);
writel_relaxed(base + GIC_DIST_ENABLE_CLEAR, GICD_INT_EN_CLR_PPI);
writel_relaxed(base + GIC_DIST_ENABLE_SET, GICD_INT_EN_SET_SGI);
let mut i = 0;
while i < 32 {
writel_relaxed(base + GIC_DIST_PRI + i * 4 / 4, GICD_INT_DEF_PRI_X4);
i += 4;
}
}
fn gic_cpu_if_up(&self) {
let mut mode = 0;
if SUPPORTS_DEACTIVATE_KEY.load(Ordering::Relaxed) {
mode = GIC_CPU_CTRL_EOImodeNS;
}
if IrqchipGic::gic_check_gicv2(self.cpu_base) {
for i in 0..4 {
writel_relaxed(self.cpu_base + GIC_CPU_ACTIVEPRIO + i * 4, 0);
}
}
let mut bypass = readl_relaxed(self.cpu_base + GIC_CPU_CTRL);
bypass &= GICC_DIS_BYPASS_MASK;
writel_relaxed(self.cpu_base + GIC_CPU_CTRL, bypass | mode | GICC_ENABLE);
}
fn gic_poke_irq(&self, hwirq: u32, offset: usize) {
let mask = 1 << (hwirq % 32);
writel_relaxed(self.dist_base + offset + (hwirq as usize / 32) * 4, mask);
}
fn gic_peek_irq(&self, hwirq: u32, offset: usize) -> bool {
let mask = 1 << (hwirq % 32);
readl_relaxed(self.dist_base + offset + (hwirq as usize / 32) * 4) & mask != 0
}
fn gic_configure_irq(hwirq: u32, irq_type: IrqType, base: usize) {
let confmask = 0x2 << ((hwirq % 16) * 2);
let confoff = (hwirq / 16) * 4;
let mut lock = IRQ_CONTROLLER_LOCK.lock_irq_save();
let mut val = readl_relaxed(base + GIC_DIST_CONFIG + confoff as usize);
let oldval = val;
if (irq_type & IrqType::IRQ_TYPE_LEVEL_MASK).bits() != 0 {
val &= !confmask;
} else if (irq_type & IrqType::IRQ_TYPE_EDGE_BOTH).bits() != 0 {
val |= confmask;
}
if val == oldval {
return;
}
writel_relaxed(base + GIC_DIST_CONFIG + confoff as usize, val);
if readl_relaxed(base + GIC_DIST_CONFIG + confoff as usize) != val {
if hwirq >= 32 {
return;
}
println!("GIC: PPI{} is secure or misconfigured", hwirq - 16);
}
*lock = 0;
}
}
static IRQ_CONTROLLER_LOCK: Spinlock<u32> = Spinlock::new(0);
impl IrqchipImpl for IrqchipGic {
const IRQS_MAX: usize = 1020;
const NAME: &'static str = "Gicv2";
fn irq_eoi(&self, hwirq: u32) {
let cd = if SUPPORTS_DEACTIVATE_KEY.load(Ordering::Relaxed) {
self.cpu_base + GIC_CPU_DEACTIVATE
} else {
self.cpu_base + GIC_CPU_EOI
};
writel_relaxed(cd, hwirq);
}
fn irq_disable(&self, hwirq: u32) {
self.gic_poke_irq(hwirq, GIC_DIST_ENABLE_CLEAR);
}
fn irq_enable(&self, hwirq: u32) {
self.gic_poke_irq(hwirq, GIC_DIST_ENABLE_SET);
}
fn irq_set_affinity(&self, hwirq: u32, cpumask: Cpumask) -> Result<()> {
let reg = self.dist_base + GIC_DIST_TARGET + (hwirq as usize & !3);
let shift = (hwirq % 4) * 8;
let index = cpumask.weight() as usize;
if index >= NR_GIC_CPU_IF || index >= nr_cpus() {
return Err(IrqError::Invalid);
}
let mask = 0xff << shift;
let bit = u32::from(unsafe { GIC_CPU_MAP[index] << shift });
let val = readl_relaxed(reg) & !mask;
writel_relaxed(reg, val | bit);
Ok(())
}
fn irq_set_type(&self, hwirq: u32, irq_type: IrqType) -> Result<()> {
if hwirq < 16 {
return Err(IrqError::Invalid);
}
if hwirq >= 32
&& !irq_type.contains(IrqType::IRQ_TYPE_LEVEL_HIGH)
&& !irq_type.contains(IrqType::IRQ_TYPE_EDGE_RISING)
{
return Err(IrqError::Invalid);
}
IrqchipGic::gic_configure_irq(hwirq, irq_type, self.dist_base);
Ok(())
}
fn irq_get_irqchip_state(&self, hwirq: u32, which: IrqchipIrqState) -> Result<bool> {
let res = match which {
IrqchipIrqState::Pending => self.gic_peek_irq(hwirq, GIC_DIST_PENDING_SET),
IrqchipIrqState::Active => self.gic_peek_irq(hwirq, GIC_DIST_ACTIVE_SET),
IrqchipIrqState::Masked => !self.gic_peek_irq(hwirq, GIC_DIST_ENABLE_SET),
IrqchipIrqState::LineLevel => {
return Err(IrqError::Invalid);
},
};
Ok(res)
}
fn irq_set_irqchip_state(&self, hwirq: u32, which: IrqchipIrqState, state: bool) -> Result<()> {
let reg = match which {
IrqchipIrqState::Pending => {
if state {
GIC_DIST_PENDING_SET
} else {
GIC_DIST_PENDING_CLEAR
}
},
IrqchipIrqState::Active => {
if state {
GIC_DIST_ACTIVE_SET
} else {
GIC_DIST_ACTIVE_CLEAR
}
},
IrqchipIrqState::Masked => {
if state {
GIC_DIST_ENABLE_CLEAR
} else {
GIC_DIST_ENABLE_SET
}
},
IrqchipIrqState::LineLevel => {
return Err(IrqError::Invalid);
},
};
self.gic_poke_irq(hwirq, reg);
Ok(())
}
fn ipi_send_mask(&self, irq: u32, cpumask: &Cpumask) {
let dist_soft = self.dist_base + GIC_DIST_SOFTINT;
if unlikely(nr_cpus() == 1) {
writel_relaxed(dist_soft, 2 << 24 | irq);
} else {
let flags = local_irq_save();
let mut map = 0u32;
for i in cpumask {
unsafe { map |= u32::from(GIC_CPU_MAP[i]) }
}
smp_wmb();
writel_relaxed(dist_soft, (map << 16) | irq);
local_irq_restore(flags);
}
}
fn irq_handler(&self) {
loop {
let irqstat = readl_relaxed(self.cpu_base + GIC_CPU_INTACK);
let irqnr = irqstat & GICC_IAR_INT_ID_MASK;
if likely(irqnr > 15 && irqnr < 1020) {
if likely(SUPPORTS_DEACTIVATE_KEY.load(Ordering::Relaxed)) {
writel_relaxed(self.cpu_base + GIC_CPU_EOI, irqstat);
}
isb();
handle_irq(irqnr);
continue;
}
if irqnr < 16 {
writel_relaxed(self.cpu_base + GIC_CPU_EOI, irqstat);
if likely(SUPPORTS_DEACTIVATE_KEY.load(Ordering::Relaxed)) {
writel_relaxed(self.cpu_base + GIC_CPU_DEACTIVATE, irqstat);
}
smp_rmb();
handle_ipi(irqnr);
continue;
}
break;
}
}
fn irq_init(&self) {
if !self.gic_check_eoimode() {
SUPPORTS_DEACTIVATE_KEY.store(false, Ordering::Relaxed);
}
self.gic_init_base();
}
fn irq_init_cpu(&self) {
self.gic_cpu_init();
}
}
static SUPPORTS_DEACTIVATE_KEY: AtomicBool = AtomicBool::new(true);
const NR_GIC_CPU_IF: usize = 8;
static mut GIC_CPU_MAP: [u8; NR_GIC_CPU_IF] = [0xff; NR_GIC_CPU_IF];
const GICD_INT_DEF_PRI: u32 = 0xa0;
const GICD_INT_DEF_PRI_X4: u32 =
GICD_INT_DEF_PRI << 24 | GICD_INT_DEF_PRI << 16 | GICD_INT_DEF_PRI << 8 | GICD_INT_DEF_PRI;
const GIC_DIST_CTRL: usize = 0x0;
#[allow(unused)]
const GIC_DIST_CTR: usize = 0x4;
const GIC_DIST_ENABLE_SET: usize = 0x100;
const GIC_DIST_ENABLE_CLEAR: usize = 0x180;
const GIC_DIST_PENDING_SET: usize = 0x200;
const GIC_DIST_PENDING_CLEAR: usize = 0x280;
const GIC_DIST_ACTIVE_SET: usize = 0x300;
const GIC_DIST_ACTIVE_CLEAR: usize = 0x380;
const GIC_DIST_PRI: usize = 0x400;
const GIC_DIST_TARGET: usize = 0x800;
const GIC_DIST_CONFIG: usize = 0xc00;
const GIC_DIST_SOFTINT: usize = 0xf00;
const GICD_DISABLE: u32 = 0x0;
const GICD_ENABLE: u32 = 0x1;
const GICD_INT_ACTLOW_LVLTRIG: u32 = 0x0;
const GICD_INT_EN_CLR_X32: u32 = 0xffffffff;
const GICD_INT_EN_SET_SGI: u32 = 0x0000ffff;
const GICD_INT_EN_CLR_PPI: u32 = 0xffff0000;
const GIC_CPU_CTRL: usize = 0x0;
const GIC_CPU_PRIMASK: usize = 0x4;
const GIC_CPU_INTACK: usize = 0xc;
const GIC_CPU_EOI: usize = 0x10;
const GIC_CPU_ACTIVEPRIO: usize = 0xd0;
const GIC_CPU_IDENT: usize = 0xfc;
const GIC_CPU_DEACTIVATE: usize = 0x1000;
const GICC_ENABLE: u32 = 0x1;
const GICC_INT_PRI_THRESHOLD: u32 = 0xf0;
#[allow(non_upper_case_globals)]
const GIC_CPU_CTRL_EOImodeNS_SHIFT: u32 = 9;
#[allow(non_upper_case_globals)]
const GIC_CPU_CTRL_EOImodeNS: u32 = 1 << GIC_CPU_CTRL_EOImodeNS_SHIFT;
const GICC_DIS_BYPASS_MASK: u32 = 0x1e0;
const GICC_IAR_INT_ID_MASK: u32 = 0x3ff;