use crate::cache::l2::L2Cache;
use crate::error::{FlowGuardError, StorageError};
use ahash::AHashMap as HashMap;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::RwLock;
use tracing::{debug, error, info, warn};
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub enum FallbackStrategy {
FailOpen,
FailClosed,
Degraded,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
pub enum ComponentType {
Redis,
Postgres,
L3Cache,
Config,
Ban,
Quota,
Other(String),
}
impl From<&str> for ComponentType {
fn from(s: &str) -> Self {
match s.to_lowercase().as_str() {
"redis" => ComponentType::Redis,
"postgres" => ComponentType::Postgres,
"l3_cache" => ComponentType::L3Cache,
"config" => ComponentType::Config,
"ban" => ComponentType::Ban,
"quota" => ComponentType::Quota,
other => ComponentType::Other(other.to_string()),
}
}
}
impl ComponentType {
pub fn as_str(&self) -> &str {
match self {
ComponentType::Redis => "redis",
ComponentType::Postgres => "postgres",
ComponentType::L3Cache => "l3_cache",
ComponentType::Config => "config",
ComponentType::Ban => "ban",
ComponentType::Quota => "quota",
ComponentType::Other(s) => s,
}
}
}
#[derive(Debug, Clone)]
pub struct FallbackConfig {
pub component: ComponentType,
pub strategy: FallbackStrategy,
pub enabled: bool,
pub timeout: Duration,
pub max_retries: u32,
}
impl Default for FallbackConfig {
fn default() -> Self {
Self {
component: ComponentType::Other("default".to_string()),
strategy: FallbackStrategy::Degraded,
enabled: true,
timeout: Duration::from_secs(5),
max_retries: 3,
}
}
}
impl FallbackConfig {
pub fn new(component: ComponentType, strategy: FallbackStrategy) -> Self {
Self {
component,
strategy,
..Default::default()
}
}
pub fn enabled(mut self, enabled: bool) -> Self {
self.enabled = enabled;
self
}
pub fn timeout(mut self, timeout: Duration) -> Self {
self.timeout = timeout;
self
}
pub fn max_retries(mut self, max_retries: u32) -> Self {
self.max_retries = max_retries;
self
}
}
pub struct FallbackManager {
strategies: Arc<RwLock<HashMap<ComponentType, FallbackConfig>>>,
l2_cache: Arc<L2Cache>,
failure_states: Arc<RwLock<HashMap<ComponentType, bool>>>,
}
impl FallbackManager {
pub fn new(l2_cache: Arc<L2Cache>) -> Self {
info!("创建降级策略管理器");
let mut strategies = HashMap::new();
strategies.insert(
ComponentType::Redis,
FallbackConfig::new(ComponentType::Redis, FallbackStrategy::Degraded),
);
strategies.insert(
ComponentType::Postgres,
FallbackConfig::new(ComponentType::Postgres, FallbackStrategy::Degraded),
);
strategies.insert(
ComponentType::L3Cache,
FallbackConfig::new(ComponentType::L3Cache, FallbackStrategy::Degraded),
);
strategies.insert(
ComponentType::Config,
FallbackConfig::new(ComponentType::Config, FallbackStrategy::FailClosed),
);
strategies.insert(
ComponentType::Ban,
FallbackConfig::new(ComponentType::Ban, FallbackStrategy::Degraded),
);
strategies.insert(
ComponentType::Quota,
FallbackConfig::new(ComponentType::Quota, FallbackStrategy::Degraded),
);
Self {
strategies: Arc::new(RwLock::new(strategies)),
l2_cache,
failure_states: Arc::new(RwLock::new(HashMap::new())),
}
}
pub async fn set_strategy(&self, component: ComponentType, config: FallbackConfig) {
info!(
"设置降级策略: component={:?}, strategy={:?}",
component, config.strategy
);
let mut strategies = self.strategies.write().await;
strategies.insert(component, config);
}
pub async fn get_strategy(&self, component: ComponentType) -> Option<FallbackConfig> {
let strategies = self.strategies.read().await;
strategies.get(&component).cloned()
}
pub async fn execute_with_fallback<F, Fut, FB, FBFut, T>(
&self,
component: ComponentType,
operation: F,
fallback_operation: FB,
) -> Result<T, FlowGuardError>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = Result<T, FlowGuardError>>,
FB: FnOnce() -> FBFut,
FBFut: std::future::Future<Output = Result<T, FlowGuardError>>,
{
let config = self
.get_strategy(component.clone())
.await
.unwrap_or_default();
if !config.enabled {
return operation().await;
}
let _is_failed = {
let states = self.failure_states.read().await;
*states.get(&component).unwrap_or(&false)
};
let result = operation().await;
match result {
Ok(value) => {
self.clear_failure(component).await;
Ok(value)
}
Err(e) => {
warn!("组件操作失败: component={:?}, error={}", component, e);
self.set_failure(component.clone()).await;
self.execute_fallback(component, config, fallback_operation)
.await
}
}
}
async fn execute_fallback<F, Fut, T>(
&self,
component: ComponentType,
config: FallbackConfig,
fallback_operation: F,
) -> Result<T, FlowGuardError>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = Result<T, FlowGuardError>>,
{
info!(
"执行降级策略: component={:?}, strategy={:?}",
component, config.strategy
);
match config.strategy {
FallbackStrategy::FailOpen => {
warn!("降级策略: FailOpen - 允许请求通过");
Err(FlowGuardError::LimitError(
"服务降级,但允许请求通过".to_string(),
))
}
FallbackStrategy::FailClosed => {
error!("降级策略: FailClosed - 拒绝请求");
Err(FlowGuardError::StorageError(StorageError::ConnectionError(
"服务降级,拒绝请求".to_string(),
)))
}
FallbackStrategy::Degraded => {
debug!("降级策略: Degraded - 使用备用方案");
fallback_operation().await
}
}
}
async fn set_failure(&self, component: ComponentType) {
warn!("组件故障: {:?}", component);
let mut states = self.failure_states.write().await;
states.insert(component, true);
}
async fn clear_failure(&self, component: ComponentType) {
let mut states = self.failure_states.write().await;
states.remove(&component);
info!("组件恢复: {:?}", component);
}
pub async fn record_failure(&self, component: ComponentType, _error: &str) {
warn!("组件故障记录: {:?}", component);
self.set_failure(component).await;
}
pub async fn get_failure_count(&self, component: ComponentType) -> u32 {
let states = self.failure_states.read().await;
if *states.get(&component).unwrap_or(&false) {
1
} else {
0
}
}
pub async fn is_failed(&self, component: ComponentType) -> bool {
let states = self.failure_states.read().await;
*states.get(&component).unwrap_or(&false)
}
pub async fn inject_failure(&self, component: ComponentType) {
warn!("注入故障: {:?}", component);
self.set_failure(component).await;
}
pub async fn recover_failure(&self, component: ComponentType) {
info!("恢复故障: {:?}", component);
self.clear_failure(component).await;
}
pub async fn get_all_failures(&self) -> Vec<ComponentType> {
let states = self.failure_states.read().await;
states
.iter()
.filter(|(_, &failed)| failed)
.map(|(component, _)| component.clone())
.collect()
}
pub fn l2_cache(&self) -> &Arc<L2Cache> {
&self.l2_cache
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_fallback_strategy_display() {
let strategy = FallbackStrategy::FailOpen;
assert_eq!(format!("{:?}", strategy), "FailOpen");
let strategy = FallbackStrategy::FailClosed;
assert_eq!(format!("{:?}", strategy), "FailClosed");
let strategy = FallbackStrategy::Degraded;
assert_eq!(format!("{:?}", strategy), "Degraded");
}
#[test]
fn test_component_type_from_str() {
assert_eq!(ComponentType::from("redis"), ComponentType::Redis);
assert_eq!(ComponentType::from("postgres"), ComponentType::Postgres);
assert_eq!(ComponentType::from("l3_cache"), ComponentType::L3Cache);
assert_eq!(
ComponentType::from("other"),
ComponentType::Other("other".to_string())
);
}
#[test]
fn test_component_type_as_str() {
assert_eq!(ComponentType::Redis.as_str(), "redis");
assert_eq!(ComponentType::Postgres.as_str(), "postgres");
assert_eq!(ComponentType::L3Cache.as_str(), "l3_cache");
}
#[test]
fn test_fallback_config_default() {
let config = FallbackConfig::default();
assert_eq!(config.strategy, FallbackStrategy::Degraded);
assert!(config.enabled);
assert_eq!(config.timeout, Duration::from_secs(5));
assert_eq!(config.max_retries, 3);
}
#[test]
fn test_fallback_config_builder() {
let config = FallbackConfig::new(ComponentType::Redis, FallbackStrategy::FailOpen)
.enabled(false)
.timeout(Duration::from_secs(10))
.max_retries(5);
assert_eq!(config.component, ComponentType::Redis);
assert_eq!(config.strategy, FallbackStrategy::FailOpen);
assert!(!config.enabled);
assert_eq!(config.timeout, Duration::from_secs(10));
assert_eq!(config.max_retries, 5);
}
#[tokio::test]
async fn test_fallback_manager_new() {
let l2_cache = Arc::new(L2Cache::new(10000, Duration::from_secs(60)));
let manager = FallbackManager::new(l2_cache);
let redis_strategy = manager.get_strategy(ComponentType::Redis).await;
assert!(redis_strategy.is_some());
assert_eq!(redis_strategy.unwrap().strategy, FallbackStrategy::Degraded);
let postgres_strategy = manager.get_strategy(ComponentType::Postgres).await;
assert!(postgres_strategy.is_some());
assert_eq!(
postgres_strategy.unwrap().strategy,
FallbackStrategy::Degraded
);
}
#[tokio::test]
async fn test_fallback_manager_set_strategy() {
let l2_cache = Arc::new(L2Cache::new(10000, Duration::from_secs(60)));
let manager = FallbackManager::new(l2_cache);
let config = FallbackConfig::new(ComponentType::Redis, FallbackStrategy::FailOpen);
manager.set_strategy(ComponentType::Redis, config).await;
let strategy = manager.get_strategy(ComponentType::Redis).await;
assert!(strategy.is_some());
assert_eq!(strategy.unwrap().strategy, FallbackStrategy::FailOpen);
}
#[tokio::test]
async fn test_fallback_manager_execute_success() {
let l2_cache = Arc::new(L2Cache::new(10000, Duration::from_secs(60)));
let manager = FallbackManager::new(l2_cache);
let result = manager
.execute_with_fallback(
ComponentType::Redis,
|| async { Ok::<String, FlowGuardError>("primary".to_string()) },
|| async { Ok::<String, FlowGuardError>("fallback".to_string()) },
)
.await;
assert!(result.is_ok());
assert_eq!(result.unwrap(), "primary");
assert!(!manager.is_failed(ComponentType::Redis).await);
}
#[tokio::test]
async fn test_fallback_manager_execute_fail_degraded() {
let l2_cache = Arc::new(L2Cache::new(10000, Duration::from_secs(60)));
let manager = FallbackManager::new(l2_cache);
let config = FallbackConfig::new(ComponentType::Redis, FallbackStrategy::Degraded);
manager.set_strategy(ComponentType::Redis, config).await;
let result = manager
.execute_with_fallback(
ComponentType::Redis,
|| async {
Err::<String, FlowGuardError>(FlowGuardError::StorageError(
StorageError::ConnectionError("connection failed".to_string()),
))
},
|| async { Ok::<String, FlowGuardError>("fallback".to_string()) },
)
.await;
assert!(result.is_ok());
assert_eq!(result.unwrap(), "fallback");
assert!(manager.is_failed(ComponentType::Redis).await);
}
#[tokio::test]
async fn test_fallback_manager_execute_fail_fail_open() {
let l2_cache = Arc::new(L2Cache::new(10000, Duration::from_secs(60)));
let manager = FallbackManager::new(l2_cache);
let config = FallbackConfig::new(ComponentType::Redis, FallbackStrategy::FailOpen);
manager.set_strategy(ComponentType::Redis, config).await;
let result = manager
.execute_with_fallback(
ComponentType::Redis,
|| async {
Err::<String, FlowGuardError>(FlowGuardError::StorageError(
StorageError::ConnectionError("connection failed".to_string()),
))
},
|| async { Ok::<String, FlowGuardError>("fallback".to_string()) },
)
.await;
assert!(result.is_err());
assert!(result
.unwrap_err()
.to_string()
.contains("服务降级,但允许请求通过"));
}
#[tokio::test]
async fn test_fallback_manager_execute_fail_fail_closed() {
let l2_cache = Arc::new(L2Cache::new(10000, Duration::from_secs(60)));
let manager = FallbackManager::new(l2_cache);
let config = FallbackConfig::new(ComponentType::Redis, FallbackStrategy::FailClosed);
manager.set_strategy(ComponentType::Redis, config).await;
let result = manager
.execute_with_fallback(
ComponentType::Redis,
|| async {
Err::<String, FlowGuardError>(FlowGuardError::StorageError(
StorageError::ConnectionError("connection failed".to_string()),
))
},
|| async { Ok::<String, FlowGuardError>("fallback".to_string()) },
)
.await;
assert!(result.is_err());
assert!(result
.unwrap_err()
.to_string()
.contains("服务降级,拒绝请求"));
}
#[tokio::test]
async fn test_fallback_manager_inject_failure() {
let l2_cache = Arc::new(L2Cache::new(10000, Duration::from_secs(60)));
let manager = FallbackManager::new(l2_cache);
assert!(!manager.is_failed(ComponentType::Redis).await);
manager.inject_failure(ComponentType::Redis).await;
assert!(manager.is_failed(ComponentType::Redis).await);
manager.recover_failure(ComponentType::Redis).await;
assert!(!manager.is_failed(ComponentType::Redis).await);
}
#[tokio::test]
async fn test_fallback_manager_get_all_failures() {
let l2_cache = Arc::new(L2Cache::new(10000, Duration::from_secs(60)));
let manager = FallbackManager::new(l2_cache);
manager.inject_failure(ComponentType::Redis).await;
manager.inject_failure(ComponentType::Postgres).await;
let failures = manager.get_all_failures().await;
assert_eq!(failures.len(), 2);
assert!(failures.contains(&ComponentType::Redis));
assert!(failures.contains(&ComponentType::Postgres));
}
#[tokio::test]
async fn test_fallback_manager_recovery() {
let l2_cache = Arc::new(L2Cache::new(10000, Duration::from_secs(60)));
let manager = FallbackManager::new(l2_cache);
let result = manager
.execute_with_fallback(
ComponentType::Redis,
|| async {
Err::<String, FlowGuardError>(FlowGuardError::StorageError(
StorageError::ConnectionError("connection failed".to_string()),
))
},
|| async { Ok::<String, FlowGuardError>("fallback".to_string()) },
)
.await;
assert!(result.is_ok());
assert!(manager.is_failed(ComponentType::Redis).await);
let result = manager
.execute_with_fallback(
ComponentType::Redis,
|| async { Ok::<String, FlowGuardError>("recovered".to_string()) },
|| async { Ok::<String, FlowGuardError>("fallback".to_string()) },
)
.await;
assert!(result.is_ok());
assert_eq!(result.unwrap(), "recovered");
assert!(!manager.is_failed(ComponentType::Redis).await);
}
#[tokio::test]
async fn test_fallback_manager_l2_cache() {
let l2_cache = Arc::new(L2Cache::new(10000, Duration::from_secs(60)));
let manager = FallbackManager::new(l2_cache);
let cache = manager.l2_cache();
assert_eq!(cache.len().await, 0);
}
}