use async_trait::async_trait;
use camel_api::{BackoffConfig, BackoffState};
use camel_component_api::{CamelError, Exchange};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use tokio::sync::Mutex;
use crate::config::{RedisCommand, is_transient_redis_error};
#[async_trait]
pub trait RedisCommandExecutor: Send + Sync {
async fn execute_command(
&mut self,
cmd: &RedisCommand,
exchange: &mut Exchange,
) -> Result<(), CamelError>;
async fn reconnect(&mut self) -> Result<(), CamelError>;
}
pub struct FakeExecutor {
results: Arc<Mutex<Vec<Result<(), FakeError>>>>,
pub call_count: Arc<AtomicUsize>,
pub reconnect_count: Arc<AtomicUsize>,
}
#[derive(Debug, Clone)]
pub struct FakeError {
pub message: String,
pub is_transient: bool,
}
impl std::fmt::Display for FakeError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.message)
}
}
impl FakeExecutor {
pub fn new(results: Vec<Result<(), FakeError>>) -> Self {
Self {
results: Arc::new(Mutex::new(results)),
call_count: Arc::new(AtomicUsize::new(0)),
reconnect_count: Arc::new(AtomicUsize::new(0)),
}
}
pub fn call_count(&self) -> Arc<AtomicUsize> {
Arc::clone(&self.call_count)
}
pub fn reconnect_count(&self) -> Arc<AtomicUsize> {
Arc::clone(&self.reconnect_count)
}
}
#[async_trait]
impl RedisCommandExecutor for FakeExecutor {
async fn execute_command(
&mut self,
_cmd: &RedisCommand,
_exchange: &mut Exchange,
) -> Result<(), CamelError> {
self.call_count.fetch_add(1, Ordering::SeqCst);
let result = {
let mut results = self.results.lock().await;
results.pop().unwrap_or(Ok(()))
};
match result {
Ok(()) => Ok(()),
Err(fake_err) => {
if fake_err.is_transient {
Err(CamelError::ProcessorError(format!(
"Connection error: {}",
fake_err.message
)))
} else {
Err(CamelError::ProcessorError(fake_err.message))
}
}
}
}
async fn reconnect(&mut self) -> Result<(), CamelError> {
self.reconnect_count.fetch_add(1, Ordering::SeqCst);
Ok(())
}
}
pub async fn execute_with_retry<E: RedisCommandExecutor>(
executor: &mut E,
cmd: &RedisCommand,
exchange: &mut Exchange,
is_idempotent: bool,
max_retries: u32,
base_ms: u64,
max_backoff: std::time::Duration,
) -> Result<(), CamelError> {
use tracing::{debug, warn};
let result = executor.execute_command(cmd, exchange).await;
if let Err(ref e) = result
&& is_transient_redis_error(e)
&& is_idempotent
{
warn!(
command = ?cmd,
error = %e,
"Transient error on idempotent command, retrying with backoff"
);
let mut last_err = e.clone();
let mut backoff = BackoffState::new(BackoffConfig {
initial_delay: std::time::Duration::from_millis(base_ms),
multiplier: 2.0,
max_delay: max_backoff,
});
for attempt in 0..max_retries {
executor.reconnect().await?;
let delay = backoff.next_delay();
debug!(
command = ?cmd,
attempt = attempt + 1,
delay_ms = delay.as_millis(),
"Waiting before retry"
);
tokio::time::sleep(delay).await;
match executor.execute_command(cmd, exchange).await {
Ok(()) => {
backoff.reset();
return Ok(());
}
Err(retry_err) => {
if is_transient_redis_error(&retry_err) {
warn!(
command = ?cmd,
attempt = attempt + 1,
error = %retry_err,
"Retry failed with transient error"
);
last_err = retry_err;
continue;
} else {
return Err(retry_err);
}
}
}
}
return Err(CamelError::ProcessorError(format!(
"Command {:?} failed after {} retries: {}",
cmd, max_retries, last_err
)));
}
result
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Duration;
fn transient_err(msg: &str) -> Result<(), FakeError> {
Err(FakeError {
message: msg.to_string(),
is_transient: true,
})
}
fn non_transient_err(msg: &str) -> Result<(), FakeError> {
Err(FakeError {
message: msg.to_string(),
is_transient: false,
})
}
#[tokio::test]
async fn test_retry_succeeds_after_transient_failures() {
let executor = FakeExecutor::new(vec![
Ok(()),
transient_err("connection reset"),
transient_err("connection reset"),
transient_err("connection reset"),
]);
let call_count = executor.call_count();
let reconnect_count = executor.reconnect_count();
let mut exchange = Exchange::default();
let cmd = RedisCommand::Get;
let result = execute_with_retry(
&mut { executor },
&cmd,
&mut exchange,
true, 10,
1, Duration::from_millis(10),
)
.await;
assert!(result.is_ok(), "should succeed after retries: {:?}", result);
assert_eq!(call_count.load(Ordering::SeqCst), 4);
assert_eq!(reconnect_count.load(Ordering::SeqCst), 3);
}
#[tokio::test]
async fn test_retry_exhausted_after_max_retries() {
let mut results = vec![transient_err("connection refused"); 10];
results.push(transient_err("final failure")); let executor = FakeExecutor::new(results);
let call_count = executor.call_count();
let mut exchange = Exchange::default();
let cmd = RedisCommand::Get;
let result = execute_with_retry(
&mut { executor },
&cmd,
&mut exchange,
true,
10,
1,
Duration::from_millis(10),
)
.await;
assert!(result.is_err(), "should fail after exhausting retries");
assert_eq!(call_count.load(Ordering::SeqCst), 11);
}
#[tokio::test]
async fn test_non_idempotent_command_not_retried() {
let executor = FakeExecutor::new(vec![transient_err("connection reset")]);
let call_count = executor.call_count();
let mut exchange = Exchange::default();
let cmd = RedisCommand::Incr;
let result = execute_with_retry(
&mut { executor },
&cmd,
&mut exchange,
false, 10,
1,
Duration::from_millis(10),
)
.await;
assert!(result.is_err(), "non-idempotent should not be retried");
assert_eq!(call_count.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_non_transient_error_not_retried() {
let executor = FakeExecutor::new(vec![non_transient_err("WRONGTYPE")]);
let call_count = executor.call_count();
let mut exchange = Exchange::default();
let cmd = RedisCommand::Get;
let result = execute_with_retry(
&mut { executor },
&cmd,
&mut exchange,
true, 10,
1,
Duration::from_millis(10),
)
.await;
assert!(result.is_err(), "non-transient error should not be retried");
assert_eq!(call_count.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn test_idempotent_command_retried_on_transient() {
let executor = FakeExecutor::new(vec![
Ok(()),
transient_err("EOF"),
transient_err("timed out"),
]);
let call_count = executor.call_count();
let mut exchange = Exchange::default();
let cmd = RedisCommand::Set;
let result = execute_with_retry(
&mut { executor },
&cmd,
&mut exchange,
true,
10,
1,
Duration::from_millis(10),
)
.await;
assert!(result.is_ok());
assert_eq!(call_count.load(Ordering::SeqCst), 3);
}
#[tokio::test]
async fn test_immediate_success_no_retries() {
let executor = FakeExecutor::new(vec![Ok(())]);
let call_count = executor.call_count();
let reconnect_count = executor.reconnect_count();
let mut exchange = Exchange::default();
let cmd = RedisCommand::Get;
let result = execute_with_retry(
&mut { executor },
&cmd,
&mut exchange,
true,
10,
1,
Duration::from_millis(10),
)
.await;
assert!(result.is_ok());
assert_eq!(call_count.load(Ordering::SeqCst), 1);
assert_eq!(reconnect_count.load(Ordering::SeqCst), 0);
}
}