use serde::{Deserialize, Serialize};
use std::collections::HashMap as FxHashMap;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::RwLock;
use tokio::task::JoinHandle;
#[cfg(feature = "http_client")]
use reqwest;
#[cfg(feature = "pyo3")]
use pyo3::PyResult;
use crate::core::{RiResult, RiError};
use crate::observability::{RiTracer, RiSpanKind, RiSpanStatus};
#[cfg(feature = "http_client")]
use crate::observability::RiContextCarrier;
#[cfg_attr(feature = "pyo3", pyo3::prelude::pyclass)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RiTrafficRoute {
pub name: String,
pub source_service: String,
pub destination_service: String,
pub match_criteria: RiMatchCriteria,
pub route_action: RiRouteAction,
pub retry_policy: Option<RiRetryPolicy>,
pub timeout: Option<Duration>,
pub fault_injection: Option<RiFaultInjection>,
}
#[cfg(feature = "pyo3")]
#[pyo3::prelude::pymethods]
impl RiTrafficRoute {
#[new]
fn py_new(name: String, source_service: String, destination_service: String) -> Self {
Self {
name,
source_service,
destination_service,
match_criteria: RiMatchCriteria {
path_prefix: None,
headers: FxHashMap::default(),
method: None,
query_parameters: FxHashMap::default(),
},
route_action: RiRouteAction::Route(vec![]),
retry_policy: None,
timeout: None,
fault_injection: None,
}
}
fn get_name(&self) -> String {
self.name.clone()
}
fn get_source_service(&self) -> String {
self.source_service.clone()
}
fn get_destination_service(&self) -> String {
self.destination_service.clone()
}
}
#[cfg_attr(feature = "pyo3", pyo3::prelude::pyclass)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RiMatchCriteria {
pub path_prefix: Option<String>,
pub headers: FxHashMap<String, String>,
pub method: Option<String>,
pub query_parameters: FxHashMap<String, String>,
}
#[cfg(feature = "pyo3")]
#[pyo3::prelude::pymethods]
impl RiMatchCriteria {
#[new]
fn py_new() -> Self {
Self {
path_prefix: None,
headers: FxHashMap::default(),
method: None,
query_parameters: FxHashMap::default(),
}
}
fn get_path_prefix(&self) -> Option<String> {
self.path_prefix.clone()
}
fn get_method(&self) -> Option<String> {
self.method.clone()
}
}
#[cfg_attr(feature = "pyo3", pyo3::prelude::pyclass)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum RiRouteAction {
Route(Vec<RiWeightedDestination>),
Redirect(String),
DirectResponse(u16, String),
}
#[cfg_attr(feature = "pyo3", pyo3::prelude::pyclass)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RiWeightedDestination {
pub service: String,
pub weight: u32,
pub subset: Option<String>,
}
#[cfg(feature = "pyo3")]
#[pyo3::prelude::pymethods]
impl RiWeightedDestination {
#[new]
fn py_new(service: String, weight: u32) -> Self {
Self {
service,
weight,
subset: None,
}
}
fn get_service(&self) -> String {
self.service.clone()
}
fn get_weight(&self) -> u32 {
self.weight
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RiRetryPolicy {
pub attempts: u32,
pub per_try_timeout: Duration,
pub retry_on: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RiFaultInjection {
pub delay: Option<RiDelayFault>,
pub abort: Option<RiAbortFault>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RiDelayFault {
pub percentage: f64,
pub fixed_delay: Duration,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RiAbortFault {
pub percentage: f64,
pub http_status: u16,
}
#[derive(Debug, Clone)]
pub struct RiTrafficSplit {
pub service: String,
pub subsets: FxHashMap<String, RiSubset>,
pub default_subset: String,
}
#[derive(Debug, Clone)]
pub struct RiSubset {
pub name: String,
pub labels: FxHashMap<String, String>,
pub weight: u32,
}
#[cfg_attr(feature = "pyo3", pyo3::prelude::pyclass)]
pub struct RiTrafficManager {
enabled: bool,
routes: Arc<RwLock<FxHashMap<String, Vec<RiTrafficRoute>>>>,
traffic_splits: Arc<RwLock<FxHashMap<String, RiTrafficSplit>>>,
circuit_breakers: Arc<RwLock<FxHashMap<String, RiCircuitBreakerConfig>>>,
rate_limits: Arc<RwLock<FxHashMap<String, RiRateLimitConfig>>>,
background_tasks: Arc<RwLock<Vec<JoinHandle<()>>>>,
#[cfg(feature = "http_client")]
http_client: reqwest::Client,
tracer: Option<Arc<RiTracer>>,
}
#[cfg_attr(feature = "pyo3", pyo3::prelude::pyclass)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RiCircuitBreakerConfig {
pub consecutive_errors: u32,
pub interval: Duration,
pub base_ejection_time: Duration,
pub max_ejection_percent: f64,
}
#[cfg(feature = "pyo3")]
#[pyo3::prelude::pymethods]
impl RiCircuitBreakerConfig {
#[new]
fn py_new(consecutive_errors: u32, max_ejection_percent: f64) -> Self {
Self {
consecutive_errors,
interval: Duration::from_secs(10),
base_ejection_time: Duration::from_secs(30),
max_ejection_percent,
}
}
fn get_consecutive_errors(&self) -> u32 {
self.consecutive_errors
}
fn get_max_ejection_percent(&self) -> f64 {
self.max_ejection_percent
}
}
#[cfg_attr(feature = "pyo3", pyo3::prelude::pyclass)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RiRateLimitConfig {
pub requests_per_second: u32,
pub burst_size: u32,
pub window: Duration,
}
impl RiTrafficManager {
pub fn new(enabled: bool) -> Self {
Self {
enabled,
routes: Arc::new(RwLock::new(FxHashMap::default())),
traffic_splits: Arc::new(RwLock::new(FxHashMap::default())),
circuit_breakers: Arc::new(RwLock::new(FxHashMap::default())),
rate_limits: Arc::new(RwLock::new(FxHashMap::default())),
background_tasks: Arc::new(RwLock::new(Vec::new())),
#[cfg(feature = "http_client")]
http_client: reqwest::Client::builder()
.timeout(Duration::from_secs(30))
.connect_timeout(Duration::from_secs(10))
.build()
.unwrap_or_else(|_| reqwest::Client::new()),
tracer: None,
}
}
pub fn with_tracer(mut self, tracer: Arc<RiTracer>) -> Self {
self.tracer = Some(tracer);
self
}
pub fn set_tracer(&mut self, tracer: Arc<RiTracer>) {
self.tracer = Some(tracer);
}
pub async fn add_traffic_route(&self, route: RiTrafficRoute) -> RiResult<()> {
if !self.enabled {
return Err(RiError::ServiceMesh("Traffic management is disabled".to_string()));
}
let mut routes = self.routes.write().await;
routes.entry(route.source_service.clone())
.or_insert_with(Vec::new)
.push(route);
Ok(())
}
pub async fn remove_traffic_route(&self, source_service: &str, route_name: &str) -> RiResult<()> {
if !self.enabled {
return Err(RiError::ServiceMesh("Traffic management is disabled".to_string()));
}
let mut routes = self.routes.write().await;
if let Some(service_routes) = routes.get_mut(source_service) {
service_routes.retain(|r| r.name != route_name);
if service_routes.is_empty() {
routes.remove(source_service);
}
}
Ok(())
}
pub async fn get_traffic_routes(&self, source_service: &str) -> RiResult<Vec<RiTrafficRoute>> {
if !self.enabled {
return Err(RiError::ServiceMesh("Traffic management is disabled".to_string()));
}
let routes = self.routes.read().await;
let service_routes = routes.get(source_service)
.cloned()
.unwrap_or_default();
Ok(service_routes)
}
pub async fn create_traffic_split(&self, split: RiTrafficSplit) -> RiResult<()> {
if !self.enabled {
return Err(RiError::ServiceMesh("Traffic management is disabled".to_string()));
}
let mut traffic_splits = self.traffic_splits.write().await;
traffic_splits.insert(split.service.clone(), split);
Ok(())
}
pub async fn get_traffic_split(&self, service: &str) -> RiResult<Option<RiTrafficSplit>> {
if !self.enabled {
return Err(RiError::ServiceMesh("Traffic management is disabled".to_string()));
}
let traffic_splits = self.traffic_splits.read().await;
Ok(traffic_splits.get(service).cloned())
}
pub async fn route_request(&self, endpoint: &str, request_data: Vec<u8>) -> RiResult<Vec<u8>> {
let span_id = if let Some(tracer) = &self.tracer {
let span_id = tracer.start_span_from_context(
format!("route_request:{}", endpoint),
RiSpanKind::Client,
);
if let Some(ref sid) = span_id {
let _ = tracer.span_mut(sid, |span| {
span.set_attribute("endpoint".to_string(), endpoint.to_string());
span.set_attribute("request_size".to_string(), request_data.len().to_string());
});
}
span_id
} else {
None
};
let result = self.route_request_internal(endpoint, request_data).await;
if let (Some(tracer), Some(sid)) = (&self.tracer, span_id) {
let status = match &result {
Ok(_) => RiSpanStatus::Ok,
Err(e) => RiSpanStatus::Error(e.to_string()),
};
let _ = tracer.end_span(&sid, status);
}
result
}
async fn route_request_internal(&self, endpoint: &str, request_data: Vec<u8>) -> RiResult<Vec<u8>> {
if !self.enabled {
return Ok(request_data);
}
if let Some(fault_injection) = self.should_inject_fault() {
self.inject_fault(&fault_injection).await?;
}
if self.should_rate_limit(endpoint).await? {
return Err(RiError::ServiceMesh("Rate limit exceeded".to_string()));
}
let transformed_request = self.apply_traffic_policies(request_data).await;
if let Some(matching_route) = self.find_matching_route(endpoint).await {
return self.apply_route(&matching_route, endpoint, transformed_request).await;
}
self.make_http_request(endpoint, transformed_request).await
}
async fn find_matching_route(&self, endpoint: &str) -> Option<RiTrafficRoute> {
let routes = self.routes.read().await;
for (_source_service, service_routes) in &*routes {
for route in service_routes {
if self.is_route_match(route, endpoint) {
return Some(route.clone());
}
}
}
None
}
fn is_route_match(&self, _route: &RiTrafficRoute, _endpoint: &str) -> bool {
#[cfg(feature = "http_client")]
if let Ok(url) = _endpoint.parse::<reqwest::Url>() {
let host = url.host_str().unwrap_or("");
if _route.destination_service.contains(host) {
return true;
}
}
false
}
async fn apply_route(&self, route: &RiTrafficRoute, original_endpoint: &str, request_data: Vec<u8>) -> RiResult<Vec<u8>> {
match &route.route_action {
RiRouteAction::Route(destinations) => {
let selected_index = self.select_destination_index(destinations).await;
let mut selected_destination = destinations[selected_index].clone();
if let Some(split_destination) = self.apply_traffic_split(&selected_destination.service).await {
selected_destination.service = split_destination;
}
let new_endpoint = self.replace_endpoint(original_endpoint, &selected_destination).await;
if let Some(retry_policy) = &route.retry_policy {
self.retry_request(new_endpoint.as_str(), request_data, retry_policy).await
} else {
self.make_http_request(new_endpoint.as_str(), request_data).await
}
},
RiRouteAction::Redirect(redirect_uri) => {
Err(RiError::ServiceMesh(format!("Redirect to: {}", redirect_uri)))
},
RiRouteAction::DirectResponse(_status, body) => {
Ok(body.clone().into())
}
}
}
async fn select_destination_index(&self, destinations: &[RiWeightedDestination]) -> usize {
if destinations.len() == 1 {
return 0;
}
let total_weight: u32 = destinations.iter().map(|d| d.weight).sum();
use rand::Rng;
let mut rng = rand::thread_rng();
let mut current_weight = 0;
let random_weight = rng.gen_range(0..total_weight);
for (index, destination) in destinations.iter().enumerate() {
current_weight += destination.weight;
if random_weight < current_weight {
return index;
}
}
0
}
async fn replace_endpoint(&self, original_endpoint: &str, _destination: &RiWeightedDestination) -> String {
original_endpoint.to_string()
}
async fn retry_request(&self, endpoint: &str, request_data: Vec<u8>, retry_policy: &RiRetryPolicy) -> RiResult<Vec<u8>> {
let max_attempts = retry_policy.attempts;
for attempt in 1..=max_attempts {
let result = self.make_http_request(endpoint, request_data.clone()).await;
match result {
Ok(response) => return Ok(response),
Err(e) => {
if attempt < max_attempts && self.should_retry(&e, retry_policy) {
let delay = Duration::from_millis(100 * 2u64.pow(attempt - 1));
tokio::time::sleep(delay).await;
continue;
}
return Err(e);
}
}
}
Err(RiError::ServiceMesh("All retry attempts failed".to_string()))
}
fn should_retry(&self, _error: &RiError, retry_policy: &RiRetryPolicy) -> bool {
retry_policy.retry_on.iter().any(|s| s == "5xx" || s == "all")
}
#[cfg(feature = "http_client")]
async fn make_http_request(&self, endpoint: &str, request_data: Vec<u8>) -> RiResult<Vec<u8>> {
let url = endpoint.parse::<reqwest::Url>()
.map_err(|e| RiError::ServiceMesh(format!("Invalid endpoint URL: {e}")))?;
let mut request_builder = self.http_client
.post(url)
.header("Content-Type", "application/octet-stream");
if let Some(_tracer) = &self.tracer {
let mut headers = FxHashMap::default();
RiContextCarrier::inject_current_into_headers(&mut headers);
for (key, value) in headers {
request_builder = request_builder.header(key, value);
}
}
let response = request_builder
.body(request_data)
.send()
.await
.map_err(|e| RiError::ServiceMesh(format!("HTTP request failed: {e}")))?;
if !response.status().is_success() {
return Err(RiError::ServiceMesh(format!(
"HTTP request failed with status: {}",
response.status()
)));
}
let response_data = response
.bytes()
.await
.map_err(|e| RiError::ServiceMesh(format!("Failed to read response body: {e}")))?
.to_vec();
Ok(response_data)
}
#[cfg(not(feature = "http_client"))]
async fn make_http_request(&self, _endpoint: &str, _request_data: Vec<u8>) -> RiResult<Vec<u8>> {
Err(RiError::ServiceMesh(format!("HTTP client is not enabled. Enable the 'http_client' feature to use HTTP requests.")))
}
async fn apply_traffic_policies(&self, request_data: Vec<u8>) -> Vec<u8> {
request_data
}
async fn apply_traffic_split(&self, service: &str) -> Option<String> {
let traffic_splits = self.traffic_splits.read().await;
if let Some(traffic_split) = traffic_splits.get(service) {
let total_weight: u32 = traffic_split.subsets.values()
.map(|subset| subset.weight)
.sum();
if total_weight == 0 {
Some(traffic_split.default_subset.clone())
} else {
use rand::Rng;
let mut rng = rand::thread_rng();
let random_weight = rng.gen_range(0..total_weight);
let mut current_weight = 0;
for subset in traffic_split.subsets.values() {
current_weight += subset.weight;
if random_weight < current_weight {
return Some(subset.name.clone());
}
}
Some(traffic_split.default_subset.clone())
}
} else {
None
}
}
fn should_inject_fault(&self) -> Option<RiFaultInjection> {
use rand::Rng;
let mut rng = rand::thread_rng();
if rng.gen_bool(0.01) {
Some(RiFaultInjection {
delay: Some(RiDelayFault {
percentage: 0.5,
fixed_delay: Duration::from_millis(100),
}),
abort: None,
})
} else {
None
}
}
async fn inject_fault(&self, fault: &RiFaultInjection) -> RiResult<()> {
if let Some(delay) = &fault.delay {
use rand::Rng;
let mut rng = rand::thread_rng();
if rng.gen_bool(delay.percentage) {
tokio::time::sleep(delay.fixed_delay).await;
}
}
if let Some(abort) = &fault.abort {
use rand::Rng;
let mut rng = rand::thread_rng();
if rng.gen_bool(abort.percentage) {
return Err(RiError::ServiceMesh(format!("Fault injection: HTTP {}", abort.http_status)));
}
}
Ok(())
}
async fn should_rate_limit(&self, endpoint: &str) -> RiResult<bool> {
let rate_limits = self.rate_limits.read().await;
if let Some(config) = rate_limits.get(endpoint) {
use std::sync::atomic::{AtomicU64, Ordering};
use std::collections::HashMap as FxHashMap;
use std::sync::Arc;
static RATE_LIMITERS: std::sync::Mutex<Option<FxHashMap<String, Arc<RateLimiter>>>> =
std::sync::Mutex::new(None);
struct RateLimiter {
capacity: u32,
rate: f64, tokens: AtomicU64, last_update: AtomicU64, }
impl RateLimiter {
fn new(config: &RiRateLimitConfig) -> Self {
let rate = config.requests_per_second as f64;
Self {
capacity: config.burst_size,
rate,
tokens: AtomicU64::new(config.burst_size as u64),
last_update: AtomicU64::new(
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or(std::time::Duration::from_secs(0))
.as_millis() as u64
),
}
}
fn try_acquire(&self) -> bool {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or(std::time::Duration::from_secs(0))
.as_millis() as u64;
let last = self.last_update.load(Ordering::Acquire);
let elapsed = now - last;
let tokens_to_add = (elapsed as f64 / 1000.0) * self.rate;
let tokens_to_add = tokens_to_add as u64;
let current = self.tokens.load(Ordering::Acquire);
let new_tokens = std::cmp::min(current.saturating_add(tokens_to_add), self.capacity as u64);
if new_tokens > 0 {
if self.tokens.compare_exchange(current, new_tokens - 1,
Ordering::AcqRel, Ordering::Acquire).is_ok() {
self.last_update.store(now, Ordering::Release);
return true;
}
}
false
}
}
let mut limiters = RATE_LIMITERS.lock()
.map_err(|e| RiError::ServiceMesh(format!("Failed to acquire rate limiter lock: {}", e)))?;
if limiters.is_none() {
*limiters = Some(FxHashMap::default());
}
let limiters = limiters.as_mut()
.ok_or_else(|| RiError::InvalidState("Rate limiters not initialized".to_string()))?;
let limiter = limiters.entry(endpoint.to_string())
.or_insert_with(|| Arc::new(RateLimiter::new(config)));
Ok(!limiter.try_acquire())
} else {
Ok(false) }
}
pub async fn set_circuit_breaker_config(&self, service: &str, config: RiCircuitBreakerConfig) -> RiResult<()> {
if !self.enabled {
return Err(RiError::ServiceMesh("Traffic management is disabled".to_string()));
}
let mut circuit_breakers = self.circuit_breakers.write().await;
circuit_breakers.insert(service.to_string(), config);
Ok(())
}
pub async fn set_rate_limit_config(&self, service: &str, config: RiRateLimitConfig) -> RiResult<()> {
if !self.enabled {
return Err(RiError::ServiceMesh("Traffic management is disabled".to_string()));
}
let mut rate_limits = self.rate_limits.write().await;
rate_limits.insert(service.to_string(), config);
Ok(())
}
pub async fn get_circuit_breaker_config(&self, service: &str) -> RiResult<Option<RiCircuitBreakerConfig>> {
let circuit_breakers = self.circuit_breakers.read().await;
Ok(circuit_breakers.get(service).cloned())
}
pub async fn get_rate_limit_config(&self, service: &str) -> RiResult<Option<RiRateLimitConfig>> {
let rate_limits = self.rate_limits.read().await;
Ok(rate_limits.get(service).cloned())
}
pub async fn start_background_tasks(&self) -> RiResult<()> {
if !self.enabled {
return Ok(());
}
Ok(())
}
pub async fn stop_background_tasks(&self) -> RiResult<()> {
let mut tasks = self.background_tasks.write().await;
for task in tasks.drain(..) {
task.abort();
}
Ok(())
}
pub async fn health_check(&self) -> RiResult<bool> {
Ok(self.enabled)
}
}
#[cfg(feature = "pyo3")]
#[pyo3::prelude::pymethods]
impl RiTrafficManager {
#[new]
fn py_new(enabled: bool) -> PyResult<Self> {
Ok(Self::new(enabled))
}
#[pyo3(name = "add_traffic_route")]
fn add_traffic_route_impl(&self, route: RiTrafficRoute) -> PyResult<()> {
let rt = tokio::runtime::Runtime::new().map_err(|e| {
pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to create runtime: {}", e))
})?;
rt.block_on(async {
self.add_traffic_route(route)
.await
.map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to add traffic route: {e}")))
})
}
#[pyo3(name = "get_traffic_routes")]
fn get_traffic_routes_impl(&self, service_name: String) -> PyResult<Vec<RiTrafficRoute>> {
let rt = tokio::runtime::Runtime::new().map_err(|e| {
pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to create runtime: {}", e))
})?;
rt.block_on(async {
self.get_traffic_routes(&service_name)
.await
.map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to get traffic routes: {e}")))
})
}
#[pyo3(name = "remove_traffic_route")]
fn remove_traffic_route_impl(&self, source_service: String, route_name: String) -> PyResult<()> {
let rt = tokio::runtime::Runtime::new().map_err(|e| {
pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to create runtime: {}", e))
})?;
rt.block_on(async {
self.remove_traffic_route(&source_service, &route_name)
.await
.map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to remove traffic route: {e}")))
})
}
#[pyo3(name = "set_circuit_breaker_config")]
fn set_circuit_breaker_config_impl(&self, service: String, config: RiCircuitBreakerConfig) -> PyResult<()> {
let rt = tokio::runtime::Runtime::new().map_err(|e| {
pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to create runtime: {}", e))
})?;
rt.block_on(async {
self.set_circuit_breaker_config(&service, config)
.await
.map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to set circuit breaker config: {e}")))
})
}
#[pyo3(name = "set_rate_limit_config")]
fn set_rate_limit_config_impl(&self, service: String, config: RiRateLimitConfig) -> PyResult<()> {
let rt = tokio::runtime::Runtime::new().map_err(|e| {
pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to create runtime: {}", e))
})?;
rt.block_on(async {
self.set_rate_limit_config(&service, config)
.await
.map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to set rate limit config: {e}")))
})
}
}