use std::collections::HashMap;
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::{Arc, Mutex, RwLock};
use crate::error::Result;
use super::{
CoalescingConfig, CoalescingState, Gsi, Irq, IrqConfig, IrqStats, IrqTriggerCallback, MAX_GSIS,
MAX_IRQS, TriggerMode,
};
pub struct IrqChip {
next_irq: AtomicU32,
mask: AtomicU32,
irq_configs: RwLock<HashMap<Irq, IrqConfig>>,
pending: AtomicU32,
trigger_callback: Mutex<Option<Arc<IrqTriggerCallback>>>,
coalescing_states: RwLock<HashMap<Irq, Arc<CoalescingState>>>,
stats: IrqStats,
}
impl IrqChip {
pub fn new() -> Result<Self> {
tracing::debug!("Creating IRQ chip");
Ok(Self {
next_irq: AtomicU32::new(32), mask: AtomicU32::new(0),
irq_configs: RwLock::new(HashMap::new()),
pending: AtomicU32::new(0),
trigger_callback: Mutex::new(None),
coalescing_states: RwLock::new(HashMap::new()),
stats: IrqStats::default(),
})
}
pub fn set_trigger_callback(&self, callback: Arc<IrqTriggerCallback>) {
let mut cb = self
.trigger_callback
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*cb = Some(callback);
tracing::debug!("IRQ trigger callback registered");
}
pub fn set_coalescing(&self, irq: Irq, config: CoalescingConfig) {
let state = Arc::new(CoalescingState::new(config));
let mut states = self
.coalescing_states
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
states.insert(irq, state);
}
#[must_use]
pub fn coalescing_state(&self, irq: Irq) -> Option<Arc<CoalescingState>> {
let states = self
.coalescing_states
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
states.get(&irq).cloned()
}
pub fn flush_coalesced(&self, irq: Irq) -> Result<()> {
let state = {
let states = self
.coalescing_states
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
states.get(&irq).cloned()
};
if let Some(state) = state {
let count = state.flush();
if count > 0 {
self.stats
.coalesced
.fetch_add(u64::from(count.saturating_sub(1)), Ordering::Relaxed);
self.deliver_irq(irq)?;
}
}
Ok(())
}
pub fn allocate_irq_with_config(&self, gsi: Gsi, trigger_mode: TriggerMode) -> Result<Irq> {
let irq = self.next_irq.fetch_add(1, Ordering::SeqCst);
if irq >= MAX_IRQS {
return Err(crate::error::VmmError::Irq("IRQ exhausted".to_string()));
}
let config = IrqConfig {
gsi,
trigger_mode,
asserted: false,
};
{
let mut configs = self
.irq_configs
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
configs.insert(irq, config);
}
tracing::debug!(
"Allocated IRQ {} -> GSI {}, mode={:?}",
irq,
gsi,
trigger_mode
);
Ok(irq)
}
pub fn allocate_irq(&self) -> Result<Irq> {
self.allocate_irq_inner(TriggerMode::Edge)
}
pub fn allocate_level_irq(&self) -> Result<Irq> {
self.allocate_irq_inner(TriggerMode::Level)
}
fn allocate_irq_inner(&self, trigger_mode: TriggerMode) -> Result<Irq> {
let irq = self.next_irq.fetch_add(1, Ordering::SeqCst);
if irq >= MAX_IRQS {
return Err(crate::error::VmmError::Irq("IRQ exhausted".to_string()));
}
let gsi = irq;
let config = IrqConfig {
gsi,
trigger_mode,
asserted: false,
};
{
let mut configs = self
.irq_configs
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
configs.insert(irq, config);
}
tracing::debug!("Allocated IRQ {} -> GSI {} (default edge)", irq, gsi);
Ok(irq)
}
pub fn configure_irq(&self, irq: Irq, gsi: Gsi, trigger_mode: TriggerMode) -> Result<()> {
let mut configs = self
.irq_configs
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(config) = configs.get_mut(&irq) {
config.gsi = gsi;
config.trigger_mode = trigger_mode;
tracing::debug!(
"Reconfigured IRQ {} -> GSI {}, mode={:?}",
irq,
gsi,
trigger_mode
);
Ok(())
} else {
Err(crate::error::VmmError::Irq(format!(
"IRQ {irq} not allocated"
)))
}
}
pub fn trigger_irq(&self, irq: Irq) -> Result<()> {
if self.is_masked(irq) {
tracing::trace!("IRQ {} is masked, not triggering", irq);
return Ok(());
}
let configs = self
.irq_configs
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let config = configs.get(&irq);
let trigger_mode = match config {
Some(c) => c.trigger_mode,
None => TriggerMode::Edge,
};
drop(configs);
let has_coalescing = if trigger_mode == TriggerMode::Edge {
let states = self
.coalescing_states
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
match states.get(&irq) {
Some(state) if state.config.enabled => {
let force_deliver = state.record();
if !force_deliver {
return Ok(());
}
let flushed = state.flush();
if flushed > 1 {
self.stats
.coalesced
.fetch_add(u64::from(flushed - 1), Ordering::Relaxed);
}
true
}
_ => false,
}
} else {
false
};
if !has_coalescing && trigger_mode == TriggerMode::Edge && irq < 32 {
let irq_bit = 1u32 << irq;
let old_pending = self.pending.fetch_or(irq_bit, Ordering::SeqCst);
if (old_pending & irq_bit) != 0 {
self.stats.coalesced.fetch_add(1, Ordering::Relaxed);
tracing::trace!("IRQ {} coalesced (already pending)", irq);
return Ok(());
}
}
self.deliver_irq(irq)
}
fn deliver_irq(&self, irq: Irq) -> Result<()> {
let configs = self
.irq_configs
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let config = configs.get(&irq);
let (gsi, trigger_mode) = match config {
Some(c) => (c.gsi, c.trigger_mode),
None => (irq % MAX_GSIS, TriggerMode::Edge),
};
drop(configs);
self.stats.triggered.fetch_add(1, Ordering::Relaxed);
tracing::trace!("Triggering IRQ {} -> GSI {}", irq, gsi);
let callback = self
.trigger_callback
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
if let Some(ref cb) = callback {
match trigger_mode {
TriggerMode::Edge => {
cb(gsi, true)?;
cb(gsi, false)?;
let irq_bit = 1u32 << (irq % 32);
self.pending.fetch_and(!irq_bit, Ordering::SeqCst);
}
TriggerMode::Level => {
cb(gsi, true)?;
let mut configs = self
.irq_configs
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(c) = configs.get_mut(&irq) {
c.asserted = true;
}
}
}
} else {
tracing::warn!(
"IRQ {} triggered but no callback registered (GSI {})",
irq,
gsi
);
}
Ok(())
}
pub fn deassert_irq(&self, irq: Irq) -> Result<()> {
let configs = self
.irq_configs
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let config = configs.get(&irq);
let gsi = match config {
Some(c) if c.trigger_mode == TriggerMode::Level => c.gsi,
Some(_) => {
tracing::trace!("deassert_irq called on edge-triggered IRQ {}", irq);
return Ok(());
}
None => {
return Ok(());
}
};
drop(configs);
let callback = self
.trigger_callback
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
if let Some(ref cb) = callback {
cb(gsi, false)?;
}
let mut configs = self
.irq_configs
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(c) = configs.get_mut(&irq) {
c.asserted = false;
}
tracing::trace!("Deasserted IRQ {} (GSI {})", irq, gsi);
Ok(())
}
pub fn ack_irq(&self, irq: Irq) {
if irq < 32 {
let irq_bit = 1u32 << irq;
self.pending.fetch_and(!irq_bit, Ordering::SeqCst);
tracing::trace!("Acknowledged IRQ {}", irq);
}
}
pub fn mask_irq(&self, irq: Irq) {
if irq < 32 {
let old = self.mask.fetch_or(1 << irq, Ordering::SeqCst);
tracing::trace!("Masked IRQ {}, old mask: {:#x}", irq, old);
}
}
pub fn unmask_irq(&self, irq: Irq) {
if irq < 32 {
let old = self.mask.fetch_and(!(1 << irq), Ordering::SeqCst);
tracing::trace!("Unmasked IRQ {}, old mask: {:#x}", irq, old);
}
}
#[must_use]
pub fn is_masked(&self, irq: Irq) -> bool {
if irq < 32 {
(self.mask.load(Ordering::SeqCst) & (1 << irq)) != 0
} else {
false
}
}
#[must_use]
pub fn is_pending(&self, irq: Irq) -> bool {
if irq < 32 {
(self.pending.load(Ordering::SeqCst) & (1 << irq)) != 0
} else {
false
}
}
#[must_use]
pub fn get_gsi(&self, irq: Irq) -> Option<Gsi> {
let configs = self
.irq_configs
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
configs.get(&irq).map(|c| c.gsi)
}
#[must_use]
pub fn get_trigger_mode(&self, irq: Irq) -> Option<TriggerMode> {
let configs = self
.irq_configs
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
configs.get(&irq).map(|c| c.trigger_mode)
}
pub fn stats(&self) -> (u64, u64) {
(
self.stats.triggered.load(Ordering::Relaxed),
self.stats.coalesced.load(Ordering::Relaxed),
)
}
pub fn reset_stats(&self) {
self.stats.triggered.store(0, Ordering::Relaxed);
self.stats.coalesced.store(0, Ordering::Relaxed);
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::AtomicUsize;
use std::time::Duration;
#[test]
fn test_irq_allocation() {
let chip = IrqChip::new().unwrap();
let irq1 = chip.allocate_irq().unwrap();
let irq2 = chip.allocate_irq().unwrap();
assert!(irq2 > irq1);
}
#[test]
fn test_irq_allocation_with_config() {
let chip = IrqChip::new().unwrap();
let irq = chip
.allocate_irq_with_config(5, TriggerMode::Level)
.unwrap();
assert_eq!(chip.get_gsi(irq), Some(5));
assert_eq!(chip.get_trigger_mode(irq), Some(TriggerMode::Level));
}
#[test]
fn test_irq_masking() {
let chip = IrqChip::new().unwrap();
assert!(!chip.is_masked(0));
chip.mask_irq(0);
assert!(chip.is_masked(0));
chip.unmask_irq(0);
assert!(!chip.is_masked(0));
}
#[test]
fn test_irq_device_trigger_chain() {
let chip = Arc::new(IrqChip::new().unwrap());
let events = Arc::new(Mutex::new(Vec::new()));
let events_clone = Arc::clone(&events);
let callback: IrqTriggerCallback = Box::new(move |gsi, level| {
events_clone
.lock()
.unwrap_or_else(|e| e.into_inner())
.push((gsi, level));
Ok(())
});
chip.set_trigger_callback(Arc::new(callback));
let irq: Irq = 5;
{
let mut configs = chip.irq_configs.write().unwrap_or_else(|e| e.into_inner());
configs.insert(
irq,
IrqConfig {
gsi: 5,
trigger_mode: TriggerMode::Edge,
asserted: false,
},
);
}
assert!(!chip.is_pending(irq));
chip.trigger_irq(irq).unwrap();
assert!(!chip.is_pending(irq));
let recorded = events.lock().unwrap_or_else(|e| e.into_inner());
assert_eq!(recorded.as_slice(), &[(5, true), (5, false)]);
}
#[test]
fn test_irq_trigger_with_callback() {
let chip = IrqChip::new().unwrap();
let trigger_count = Arc::new(AtomicUsize::new(0));
let count_clone = Arc::clone(&trigger_count);
let callback: IrqTriggerCallback = Box::new(move |_gsi, _level| {
count_clone.fetch_add(1, Ordering::SeqCst);
Ok(())
});
chip.set_trigger_callback(Arc::new(callback));
let irq = chip.allocate_irq_with_config(1, TriggerMode::Edge).unwrap();
chip.trigger_irq(irq).unwrap();
assert_eq!(trigger_count.load(Ordering::SeqCst), 2);
}
#[test]
fn test_irq_coalescing() {
let chip = IrqChip::new().unwrap();
let trigger_count = Arc::new(AtomicUsize::new(0));
let count_clone = Arc::clone(&trigger_count);
let callback: IrqTriggerCallback = Box::new(move |_gsi, _level| {
count_clone.fetch_add(1, Ordering::SeqCst);
Ok(())
});
chip.set_trigger_callback(Arc::new(callback));
let irq = chip.allocate_irq_with_config(1, TriggerMode::Edge).unwrap();
chip.trigger_irq(irq).unwrap();
chip.trigger_irq(irq).unwrap(); chip.trigger_irq(irq).unwrap();
let (triggered, coalesced) = chip.stats();
assert!(triggered >= 1);
tracing::debug!("triggered={}, coalesced={}", triggered, coalesced);
}
#[test]
fn test_level_triggered_irq() {
let chip = IrqChip::new().unwrap();
let levels = Arc::new(Mutex::new(Vec::new()));
let levels_clone = Arc::clone(&levels);
let callback: IrqTriggerCallback = Box::new(move |gsi, level| {
levels_clone
.lock()
.unwrap_or_else(|e| e.into_inner())
.push((gsi, level));
Ok(())
});
chip.set_trigger_callback(Arc::new(callback));
let irq = chip
.allocate_irq_with_config(3, TriggerMode::Level)
.unwrap();
chip.trigger_irq(irq).unwrap();
chip.deassert_irq(irq).unwrap();
let recorded = levels.lock().unwrap_or_else(|e| e.into_inner());
assert_eq!(recorded.len(), 2);
assert_eq!(recorded[0], (3, true)); assert_eq!(recorded[1], (3, false)); }
#[test]
fn test_masked_irq_not_triggered() {
let chip = IrqChip::new().unwrap();
let trigger_count = Arc::new(AtomicUsize::new(0));
let count_clone = Arc::clone(&trigger_count);
let callback: IrqTriggerCallback = Box::new(move |_gsi, _level| {
count_clone.fetch_add(1, Ordering::SeqCst);
Ok(())
});
chip.set_trigger_callback(Arc::new(callback));
let irq = chip.allocate_irq_with_config(0, TriggerMode::Edge).unwrap();
let legacy_irq: Irq = 5;
{
let mut configs = chip.irq_configs.write().unwrap_or_else(|e| e.into_inner());
configs.insert(
legacy_irq,
IrqConfig {
gsi: 5,
trigger_mode: TriggerMode::Edge,
asserted: false,
},
);
}
chip.mask_irq(legacy_irq);
chip.trigger_irq(legacy_irq).unwrap();
assert_eq!(trigger_count.load(Ordering::SeqCst), 0);
chip.unmask_irq(legacy_irq);
chip.trigger_irq(legacy_irq).unwrap();
assert_eq!(trigger_count.load(Ordering::SeqCst), 2);
assert!(irq >= 32);
assert!(!chip.is_masked(irq));
}
#[test]
fn test_coalescing_config_presets() {
let net = CoalescingConfig::for_net();
assert!(net.enabled);
assert_eq!(net.max_delay, Duration::from_micros(50));
let blk = CoalescingConfig::for_block();
assert_eq!(blk.max_delay, Duration::from_micros(25));
let disabled = CoalescingConfig::disabled();
assert!(!disabled.enabled);
}
#[test]
fn test_coalescing_record_first() {
let state = CoalescingState::new(CoalescingConfig::default());
assert!(!state.record()); assert!(state.timer_armed.load(Ordering::Relaxed));
assert_eq!(state.pending_count.load(Ordering::Relaxed), 1);
}
#[test]
fn test_coalescing_force_delivery() {
let config = CoalescingConfig {
max_coalesce_count: 3,
..Default::default()
};
let state = CoalescingState::new(config);
assert!(!state.record()); assert!(!state.record()); assert!(state.record()); }
#[test]
fn test_coalescing_flush() {
let state = CoalescingState::new(CoalescingConfig::default());
state.record();
state.record();
let count = state.flush();
assert_eq!(count, 2);
assert_eq!(state.pending_count.load(Ordering::Relaxed), 0);
assert!(!state.timer_armed.load(Ordering::Relaxed));
}
#[test]
fn test_coalescing_timer_expired() {
let config = CoalescingConfig {
max_delay: Duration::from_millis(1),
..Default::default()
};
let state = CoalescingState::new(config);
assert!(!state.timer_expired()); state.record();
std::thread::sleep(Duration::from_millis(2));
assert!(state.timer_expired());
}
#[test]
fn test_coalescing_disabled() {
let state = CoalescingState::new(CoalescingConfig::disabled());
assert!(!state.config.enabled);
}
#[test]
fn test_trigger_irq_with_coalescing_accumulates() {
let chip = IrqChip::new().unwrap();
let trigger_count = Arc::new(AtomicUsize::new(0));
let count_clone = Arc::clone(&trigger_count);
let callback: IrqTriggerCallback = Box::new(move |_gsi, _level| {
count_clone.fetch_add(1, Ordering::SeqCst);
Ok(())
});
chip.set_trigger_callback(Arc::new(callback));
let irq = chip.allocate_irq_with_config(1, TriggerMode::Edge).unwrap();
chip.set_coalescing(
irq,
CoalescingConfig {
max_coalesce_count: 5,
max_delay: Duration::from_secs(10),
enabled: true,
},
);
for _ in 0..4 {
chip.trigger_irq(irq).unwrap();
}
assert_eq!(trigger_count.load(Ordering::SeqCst), 0);
chip.trigger_irq(irq).unwrap();
assert_eq!(trigger_count.load(Ordering::SeqCst), 2);
}
#[test]
fn test_flush_coalesced_delivers() {
let chip = IrqChip::new().unwrap();
let trigger_count = Arc::new(AtomicUsize::new(0));
let count_clone = Arc::clone(&trigger_count);
let callback: IrqTriggerCallback = Box::new(move |_gsi, _level| {
count_clone.fetch_add(1, Ordering::SeqCst);
Ok(())
});
chip.set_trigger_callback(Arc::new(callback));
let irq = chip.allocate_irq_with_config(1, TriggerMode::Edge).unwrap();
chip.set_coalescing(irq, CoalescingConfig::default());
chip.trigger_irq(irq).unwrap();
assert_eq!(trigger_count.load(Ordering::SeqCst), 0);
chip.flush_coalesced(irq).unwrap();
assert_eq!(trigger_count.load(Ordering::SeqCst), 2); }
}