use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use std::collections::HashMap as FxHashMap;
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use tokio::sync::RwLock;
#[cfg(feature = "pyo3")]
use pyo3::PyResult;
use crate::core::{RiModule, RiResult, RiError};
pub use crate::gateway::{RiCircuitBreaker, RiCircuitBreakerConfig, RiLoadBalancer, RiLoadBalancerStrategy};
use crate::gateway::load_balancer::RiBackendServer;
use crate::observability::{RiTracer, RiSpanKind, RiSpanStatus};
pub mod service_discovery;
pub mod health_check;
pub mod traffic_management;
pub use service_discovery::{RiServiceDiscovery, RiServiceInstance, RiServiceStatus};
pub use health_check::{RiHealthChecker, RiHealthCheckResult, RiHealthSummary, RiHealthStatus, RiHealthCheckType};
pub use traffic_management::{RiTrafficRoute, RiMatchCriteria, RiRouteAction, RiWeightedDestination, RiTrafficManager};
#[cfg_attr(feature = "pyo3", pyo3::prelude::pyclass)]
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RiServiceMeshConfig {
pub enable_service_discovery: bool,
pub enable_health_check: bool,
pub enable_traffic_management: bool,
pub health_check_interval: Duration,
pub circuit_breaker_config: RiCircuitBreakerConfig,
pub load_balancer_strategy: RiLoadBalancerStrategy,
pub max_retry_attempts: u32,
pub retry_timeout: Duration,
}
impl Default for RiServiceMeshConfig {
fn default() -> Self {
Self {
enable_service_discovery: true,
enable_health_check: true,
enable_traffic_management: true,
health_check_interval: Duration::from_secs(30),
circuit_breaker_config: RiCircuitBreakerConfig::default(),
load_balancer_strategy: RiLoadBalancerStrategy::RoundRobin,
max_retry_attempts: 3,
retry_timeout: Duration::from_secs(5),
}
}
}
#[cfg_attr(feature = "pyo3", pyo3::prelude::pyclass)]
#[derive(Debug, Clone)]
pub struct RiServiceEndpoint {
pub service_name: String,
pub endpoint: String,
pub weight: u32,
pub metadata: FxHashMap<String, String>,
pub health_status: RiServiceHealthStatus,
pub last_health_check: SystemTime,
}
#[cfg_attr(feature = "pyo3", pyo3::prelude::pyclass)]
#[derive(Debug, Clone, PartialEq)]
pub enum RiServiceHealthStatus {
Healthy,
Unhealthy,
Unknown,
}
#[cfg(feature = "pyo3")]
#[pyo3::prelude::pymethods]
impl RiServiceEndpoint {
#[new]
fn py_new(
service_name: String,
endpoint: String,
weight: u32,
) -> Self {
Self {
service_name,
endpoint,
weight,
metadata: FxHashMap::default(),
health_status: RiServiceHealthStatus::Unknown,
last_health_check: SystemTime::now(),
}
}
#[getter]
fn service_name(&self) -> &str {
&self.service_name
}
#[getter]
fn endpoint(&self) -> &str {
&self.endpoint
}
#[getter]
fn weight(&self) -> u32 {
self.weight
}
#[getter]
fn health_status(&self) -> RiServiceHealthStatus {
self.health_status.clone()
}
}
#[derive(Debug, Clone)]
struct ServiceDiscoveryCacheEntry {
endpoints: Vec<RiServiceEndpoint>,
expiration: SystemTime,
}
#[derive(Debug, Clone)]
#[cfg_attr(feature = "pyo3", pyo3::prelude::pyclass)]
pub struct RiServiceMeshStats {
pub total_services: usize,
pub total_endpoints: usize,
pub healthy_endpoints: usize,
pub unhealthy_endpoints: usize,
}
#[cfg(feature = "pyo3")]
#[pyo3::prelude::pymethods]
impl RiServiceMeshStats {
#[new]
fn py_new() -> Self {
Self {
total_services: 0,
total_endpoints: 0,
healthy_endpoints: 0,
unhealthy_endpoints: 0,
}
}
#[getter]
fn total_services(&self) -> usize {
self.total_services
}
#[getter]
fn total_endpoints(&self) -> usize {
self.total_endpoints
}
#[getter]
fn healthy_endpoints(&self) -> usize {
self.healthy_endpoints
}
#[getter]
fn unhealthy_endpoints(&self) -> usize {
self.unhealthy_endpoints
}
}
#[cfg_attr(feature = "pyo3", pyo3::prelude::pyclass)]
pub struct RiServiceMesh {
config: RiServiceMeshConfig,
service_discovery: Arc<RiServiceDiscovery>,
health_checker: Arc<RiHealthChecker>,
traffic_manager: Arc<RiTrafficManager>,
circuit_breaker: Arc<RiCircuitBreaker>,
load_balancer: Arc<RiLoadBalancer>,
services: Arc<RwLock<FxHashMap<String, Vec<RiServiceEndpoint>>>>,
discovery_cache: Arc<RwLock<FxHashMap<String, ServiceDiscoveryCacheEntry>>>,
cache_expiration: Duration,
tracer: Option<Arc<RiTracer>>,
}
impl RiServiceMesh {
pub fn new(config: RiServiceMeshConfig) -> RiResult<Self> {
let service_discovery = Arc::new(RiServiceDiscovery::new(config.enable_service_discovery));
let health_checker = Arc::new(RiHealthChecker::new(config.health_check_interval));
let traffic_manager = Arc::new(RiTrafficManager::new(config.enable_traffic_management));
let circuit_breaker = Arc::new(RiCircuitBreaker::new(config.circuit_breaker_config.clone()));
let load_balancer = Arc::new(RiLoadBalancer::new(config.load_balancer_strategy.clone()));
Ok(Self {
config,
service_discovery,
health_checker,
traffic_manager,
circuit_breaker,
load_balancer,
services: Arc::new(RwLock::new(FxHashMap::default())),
discovery_cache: Arc::new(RwLock::new(FxHashMap::default())),
cache_expiration: Duration::from_secs(30),
tracer: None,
})
}
pub fn with_tracer(mut self, tracer: Arc<RiTracer>) -> Self {
self.tracer = Some(tracer.clone());
let mut traffic_manager = RiTrafficManager::new(self.config.enable_traffic_management);
traffic_manager.set_tracer(tracer);
self.traffic_manager = Arc::new(traffic_manager);
self
}
pub fn set_tracer(&mut self, tracer: Arc<RiTracer>) {
self.tracer = Some(tracer.clone());
let mut traffic_manager = RiTrafficManager::new(self.config.enable_traffic_management);
traffic_manager.set_tracer(tracer);
self.traffic_manager = Arc::new(traffic_manager);
}
pub async fn register_service(&self, service_name: &str, endpoint: &str, weight: u32, metadata: Option<FxHashMap<String, String>>) -> RiResult<()> {
if service_name.is_empty() {
return Err(RiError::ServiceMesh("Service name cannot be empty".to_string()));
}
if endpoint.is_empty() {
return Err(RiError::ServiceMesh("Endpoint cannot be empty".to_string()));
}
if weight == 0 {
return Err(RiError::ServiceMesh("Weight must be greater than zero".to_string()));
}
let service_endpoint = RiServiceEndpoint {
service_name: service_name.to_string(),
endpoint: endpoint.to_string(),
weight,
metadata: metadata.unwrap_or_default(),
health_status: RiServiceHealthStatus::Unknown,
last_health_check: SystemTime::now(),
};
let mut services = self.services.write().await;
services.entry(service_name.to_string())
.or_insert_with(Vec::new)
.push(service_endpoint);
if self.config.enable_health_check {
self.health_checker.start_health_check(service_name, endpoint).await?;
}
Ok(())
}
pub async fn register_versioned_service(&self, service_name: &str, version: &str, endpoint: &str, weight: u32, metadata: Option<FxHashMap<String, String>>) -> RiResult<()> {
let mut enriched_metadata = metadata.unwrap_or_default();
enriched_metadata.insert("version".to_string(), version.to_string());
self.register_service(service_name, endpoint, weight, Some(enriched_metadata)).await
}
pub async fn unregister_service(&self, service_name: &str, endpoint: &str) -> RiResult<()> {
let mut services = self.services.write().await;
if let Some(endpoints) = services.get_mut(service_name) {
endpoints.retain(|ep| ep.endpoint != endpoint);
if endpoints.is_empty() {
services.remove(service_name);
}
if self.config.enable_health_check {
self.health_checker.stop_health_check(service_name, endpoint).await?;
}
}
Ok(())
}
pub async fn get_all_endpoints(&self, service_name: &str) -> RiResult<Vec<RiServiceEndpoint>> {
let services = self.services.read().await;
services.get(service_name)
.cloned()
.ok_or_else(|| RiError::ServiceMesh(format!("Service '{service_name}' not found")))
}
pub async fn get_stats(&self) -> RiServiceMeshStats {
let services = self.services.read().await;
let healthy_count = services.values()
.flat_map(|endpoints| endpoints.iter())
.filter(|ep| ep.health_status == RiServiceHealthStatus::Healthy)
.count();
RiServiceMeshStats {
total_services: services.len(),
total_endpoints: services.values().map(|v| v.len()).sum(),
healthy_endpoints: healthy_count,
unhealthy_endpoints: services.values()
.flat_map(|endpoints| endpoints.iter())
.filter(|ep| ep.health_status == RiServiceHealthStatus::Unhealthy)
.count(),
}
}
pub async fn discover_service(&self, service_name: &str) -> RiResult<Vec<RiServiceEndpoint>> {
if !self.config.enable_service_discovery {
return Err(RiError::ServiceMesh("Service discovery is disabled".to_string()));
}
{
let cache = self.discovery_cache.read().await;
if let Some(entry) = cache.get(service_name) {
if entry.expiration > SystemTime::now() {
return Ok(entry.endpoints.clone());
}
}
}
let services = self.services.read().await;
let endpoints = services.get(service_name)
.ok_or_else(|| RiError::ServiceMesh(format!("Service '{service_name}' not found")))?
.clone();
let healthy_endpoints: Vec<RiServiceEndpoint> = endpoints
.into_iter()
.filter(|ep| ep.health_status == RiServiceHealthStatus::Healthy)
.collect();
if healthy_endpoints.is_empty() {
return Err(RiError::ServiceMesh(format!("No healthy endpoints for service '{service_name}'")));
}
let expiration = SystemTime::now() + self.cache_expiration;
let cache_entry = ServiceDiscoveryCacheEntry {
endpoints: healthy_endpoints.clone(),
expiration,
};
let mut cache = self.discovery_cache.write().await;
cache.insert(service_name.to_string(), cache_entry);
Ok(healthy_endpoints)
}
pub async fn call_service(&self, service_name: &str, request_data: Vec<u8>) -> RiResult<Vec<u8>> {
let span_id = if let Some(tracer) = &self.tracer {
let span_id = tracer.start_span_from_context(
format!("call_service:{}", service_name),
RiSpanKind::Client,
);
if let Some(ref sid) = span_id {
let _ = tracer.span_mut(sid, |span| {
span.set_attribute("service_name".to_string(), service_name.to_string());
span.set_attribute("request_size".to_string(), request_data.len().to_string());
});
}
span_id
} else {
None
};
let result = self.call_service_internal(service_name, request_data).await;
if let (Some(tracer), Some(sid)) = (&self.tracer, span_id) {
let status = match &result {
Ok(_) => RiSpanStatus::Ok,
Err(e) => RiSpanStatus::Error(e.to_string()),
};
let _ = tracer.end_span(&sid, status);
}
result
}
async fn call_service_internal(&self, service_name: &str, request_data: Vec<u8>) -> RiResult<Vec<u8>> {
let endpoints = self.discover_service(service_name).await?;
let mut existing_servers = self.load_balancer.get_healthy_servers().await;
existing_servers.retain(|s| !s.id.starts_with(&format!("{service_name}-")));
for ep in &endpoints {
if ep.health_status == RiServiceHealthStatus::Healthy {
let server = RiBackendServer {
id: format!("{}-{}", service_name, ep.endpoint),
url: ep.endpoint.clone(),
weight: ep.weight,
max_connections: 100,
health_check_path: "/health".to_string(),
is_healthy: true,
};
let _ = self.load_balancer.add_server(server).await;
}
}
let selected_server = match self.load_balancer.select_server(None).await {
Ok(server) => server,
Err(_) => return Err(RiError::ServiceMesh("No available backend server".to_string())),
};
if !self.circuit_breaker.allow_request() {
return Err(RiError::ServiceMesh("Circuit breaker is open".to_string()));
}
let result = self.execute_service_call(&selected_server.url, request_data.clone()).await;
match &result {
Ok(_) => {
self.circuit_breaker.record_success();
}
Err(_) => {
self.circuit_breaker.record_failure();
}
}
result
}
async fn execute_service_call(&self, endpoint: &str, request_data: Vec<u8>) -> RiResult<Vec<u8>> {
let mut last_error = None;
for attempt in 0..self.config.max_retry_attempts {
match self.traffic_manager.route_request(endpoint, request_data.clone()).await {
Ok(response) => return Ok(response),
Err(_e) => {
let sanitized_error = RiError::ServiceMesh(format!("Retry attempt {} failed", attempt + 1));
last_error = Some(sanitized_error);
if attempt < self.config.max_retry_attempts - 1 {
tokio::time::sleep(Duration::from_millis(100 * (attempt + 1) as u64)).await;
}
}
}
}
Err(last_error.unwrap_or_else(|| RiError::ServiceMesh("All retry attempts failed".to_string())))
}
pub async fn update_service_health(&self, service_name: &str, endpoint: &str, is_healthy: bool) -> RiResult<()> {
let mut services = self.services.write().await;
if let Some(endpoints) = services.get_mut(service_name) {
if let Some(service_ep) = endpoints.iter_mut().find(|ep| ep.endpoint == endpoint) {
service_ep.health_status = if is_healthy {
RiServiceHealthStatus::Healthy
} else {
RiServiceHealthStatus::Unhealthy
};
service_ep.last_health_check = SystemTime::now();
}
}
Ok(())
}
pub fn get_circuit_breaker(&self) -> &RiCircuitBreaker {
&self.circuit_breaker
}
pub fn get_load_balancer(&self) -> &RiLoadBalancer {
&self.load_balancer
}
pub fn get_health_checker(&self) -> &RiHealthChecker {
&self.health_checker
}
pub fn get_traffic_manager(&self) -> &RiTrafficManager {
&self.traffic_manager
}
pub fn get_service_discovery(&self) -> &RiServiceDiscovery {
&self.service_discovery
}
}
#[cfg(feature = "pyo3")]
#[pyo3::prelude::pymethods]
impl RiServiceMesh {
#[new]
fn py_new(config: RiServiceMeshConfig) -> PyResult<Self> {
match Self::new(config) {
Ok(mesh) => Ok(mesh),
Err(e) => Err(pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to create service mesh: {e}"))),
}
}
#[pyo3(name = "register_service")]
fn register_service_impl(&self, service_name: String, endpoint: String, weight: u32) -> PyResult<()> {
let rt = tokio::runtime::Runtime::new().map_err(|e| {
pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to create runtime: {}", e))
})?;
rt.block_on(async {
self.register_service(&service_name, &endpoint, weight, None)
.await
.map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to register service: {e}")))
})
}
#[pyo3(name = "discover_service")]
fn discover_service_impl(&self, service_name: String) -> PyResult<Vec<RiServiceEndpoint>> {
let rt = tokio::runtime::Runtime::new().map_err(|e| {
pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to create runtime: {}", e))
})?;
rt.block_on(async {
self.discover_service(&service_name)
.await
.map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to discover service: {e}")))
})
}
#[pyo3(name = "update_service_health")]
fn update_service_health_impl(&self, service_name: String, endpoint: String, is_healthy: bool) -> PyResult<()> {
let rt = tokio::runtime::Runtime::new().map_err(|e| {
pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to create runtime: {}", e))
})?;
rt.block_on(async {
self.update_service_health(&service_name, &endpoint, is_healthy)
.await
.map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to update health: {e}")))
})
}
fn get_config(&self) -> RiServiceMeshConfig {
self.config.clone()
}
}
#[async_trait]
impl RiModule for RiServiceMesh {
fn name(&self) -> &str {
"Ri.ServiceMesh"
}
fn is_critical(&self) -> bool {
true
}
async fn start(&mut self, _ctx: &mut crate::core::RiServiceContext) -> RiResult<()> {
if self.config.enable_health_check {
self.health_checker.start_background_tasks().await?;
}
if self.config.enable_service_discovery {
self.service_discovery.start_background_tasks().await?;
}
if self.config.enable_traffic_management {
self.traffic_manager.start_background_tasks().await?;
}
Ok(())
}
async fn shutdown(&mut self, _ctx: &mut crate::core::RiServiceContext) -> RiResult<()> {
if self.config.enable_health_check {
self.health_checker.stop_background_tasks().await?;
}
if self.config.enable_service_discovery {
self.service_discovery.stop_background_tasks().await?;
}
if self.config.enable_traffic_management {
self.traffic_manager.stop_background_tasks().await?;
}
Ok(())
}
}