use dashmap::DashMap;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::Semaphore;
use tracing::warn;
#[derive(Debug, Clone)]
pub struct BulkheadConfig {
pub max_concurrent_per_backend: usize,
pub acquire_timeout: Duration,
}
impl Default for BulkheadConfig {
fn default() -> Self {
Self {
max_concurrent_per_backend: 25,
acquire_timeout: Duration::from_secs(10),
}
}
}
#[derive(Clone)]
pub struct Bulkhead {
pools: Arc<DashMap<String, Arc<Semaphore>>>,
config: BulkheadConfig,
}
impl Bulkhead {
pub fn new() -> Self {
Self::with_config(BulkheadConfig::default())
}
pub fn with_config(config: BulkheadConfig) -> Self {
Self {
pools: Arc::new(DashMap::new()),
config,
}
}
pub async fn try_acquire(&self, backend_id: &str) -> Option<BulkheadPermit> {
let sem = self.get_or_create_pool(backend_id);
match tokio::time::timeout(self.config.acquire_timeout, sem.clone().acquire_owned()).await {
Ok(Ok(permit)) => Some(BulkheadPermit {
_permit: permit,
backend_id: backend_id.to_string(),
}),
Ok(Err(_)) => {
warn!("Bulkhead: semaphore closed for {backend_id}");
None
}
Err(_) => {
warn!(
"Bulkhead: timeout acquiring permit for {backend_id} (all {} slots busy)",
self.config.max_concurrent_per_backend
);
None
}
}
}
fn get_or_create_pool(&self, backend_id: &str) -> Arc<Semaphore> {
self.pools
.entry(backend_id.to_string())
.or_insert_with(|| Arc::new(Semaphore::new(self.config.max_concurrent_per_backend)))
.value()
.clone()
}
pub fn available(&self, backend_id: &str) -> usize {
self.pools
.get(backend_id)
.map(|s| s.available_permits())
.unwrap_or(self.config.max_concurrent_per_backend)
}
pub fn utilization(&self) -> Vec<BulkheadStats> {
let max = self.config.max_concurrent_per_backend;
self.pools
.iter()
.map(|entry| {
let available = entry.value().available_permits();
BulkheadStats {
backend_id: entry.key().clone(),
max_concurrent: max,
in_use: max - available,
available,
}
})
.collect()
}
}
impl Default for Bulkhead {
fn default() -> Self {
Self::new()
}
}
pub struct BulkheadPermit {
_permit: tokio::sync::OwnedSemaphorePermit,
#[allow(dead_code)]
backend_id: String,
}
#[derive(Debug, Clone)]
pub struct BulkheadStats {
pub backend_id: String,
pub max_concurrent: usize,
pub in_use: usize,
pub available: usize,
}
impl BulkheadStats {
pub fn utilization(&self) -> f64 {
if self.max_concurrent == 0 {
return 0.0;
}
self.in_use as f64 / self.max_concurrent as f64
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn bulkhead_allows_requests() {
let bh = Bulkhead::new();
let permit = bh.try_acquire("backend-a").await;
assert!(permit.is_some());
}
#[tokio::test]
async fn bulkhead_isolates_backends() {
let config = BulkheadConfig {
max_concurrent_per_backend: 1,
acquire_timeout: Duration::from_millis(100),
};
let bh = Bulkhead::with_config(config);
let _p1 = bh.try_acquire("backend-a").await.unwrap();
let p2 = bh.try_acquire("backend-b").await;
assert!(p2.is_some());
let p3 = bh.try_acquire("backend-a").await;
assert!(p3.is_none());
}
#[tokio::test]
async fn bulkhead_releases_on_drop() {
let config = BulkheadConfig {
max_concurrent_per_backend: 1,
acquire_timeout: Duration::from_millis(100),
};
let bh = Bulkhead::with_config(config);
{
let _p = bh.try_acquire("backend-x").await.unwrap();
assert_eq!(bh.available("backend-x"), 0);
}
assert_eq!(bh.available("backend-x"), 1);
}
#[test]
fn utilization_stats() {
let stats = BulkheadStats {
backend_id: "test".into(),
max_concurrent: 5,
in_use: 3,
available: 2,
};
assert!((stats.utilization() - 0.6).abs() < 0.01);
}
}