mod private
{
use serde::{ Deserialize, Serialize };
use core::time::Duration;
use std::time::SystemTime;
use std::sync::{ Arc, Mutex };
use std::collections::HashMap;
use futures::Future;
#[ derive( Debug, Clone, PartialEq, Serialize, Deserialize ) ]
pub struct FailoverConfig
{
pub primary_endpoint : String,
pub backup_endpoints : Vec< String >,
pub timeout : Duration,
pub max_retries : u32,
pub strategy : FailoverStrategy,
}
impl Default for FailoverConfig
{
#[ inline ]
fn default() -> Self
{
Self {
primary_endpoint : "https://generativelanguage.googleapis.com".to_string(),
backup_endpoints : Vec::new(),
timeout : Duration::from_secs( 5 ),
max_retries : 3,
strategy : FailoverStrategy::Priority,
}
}
}
#[ derive( Debug, Clone ) ]
pub struct FailoverConfigBuilder
{
config : FailoverConfig,
}
impl Default for FailoverConfigBuilder
{
#[ inline ]
fn default() -> Self
{
Self {
config : FailoverConfig::default(),
}
}
}
impl FailoverConfigBuilder
{
#[ inline ]
#[ must_use ]
pub fn new() -> Self
{
Self {
config : FailoverConfig::default(),
}
}
#[ inline ]
#[ must_use ]
pub fn primary_endpoint( mut self, endpoint : String ) -> Self
{
self.config.primary_endpoint = endpoint;
self
}
#[ inline ]
#[ must_use ]
pub fn backup_endpoint( mut self, endpoint : String ) -> Self
{
self.config.backup_endpoints.push( endpoint );
self
}
#[ inline ]
#[ must_use ]
pub fn timeout( mut self, timeout : Duration ) -> Self
{
self.config.timeout = timeout;
self
}
#[ inline ]
#[ must_use ]
pub fn max_retries( mut self, retries : u32 ) -> Self
{
self.config.max_retries = retries;
self
}
#[ inline ]
#[ must_use ]
pub fn strategy( mut self, strategy : FailoverStrategy ) -> Self
{
self.config.strategy = strategy;
self
}
#[ inline ]
pub fn build( self ) -> Result< FailoverConfig, crate::error::Error >
{
Self::validate_config( &self.config )?;
Ok( self.config )
}
#[ inline ]
fn validate_config( config : &FailoverConfig ) -> Result< (), crate::error::Error >
{
if config.primary_endpoint.is_empty()
{
return Err( crate::error::Error::ConfigurationError(
"Primary endpoint cannot be empty".to_string()
) );
}
if config.backup_endpoints.is_empty()
{
return Err( crate::error::Error::ConfigurationError(
"At least one backup endpoint is required for failover".to_string()
) );
}
if config.timeout.is_zero()
{
return Err( crate::error::Error::ConfigurationError(
"Timeout cannot be zero".to_string()
) );
}
if config.max_retries == 0
{
return Err( crate::error::Error::ConfigurationError(
"Max retries must be at least 1".to_string()
) );
}
Ok( () )
}
}
impl FailoverConfig
{
#[ inline ]
#[ must_use ]
pub fn builder() -> FailoverConfigBuilder
{
FailoverConfigBuilder::new()
}
}
#[ derive( Debug, Clone, PartialEq, Eq, Serialize, Deserialize ) ]
pub enum FailoverStrategy
{
Priority,
RoundRobin,
}
#[ derive( Debug, Clone, PartialEq, Eq, Serialize, Deserialize ) ]
pub enum HealthStatus
{
Healthy,
Unhealthy,
Unknown,
}
#[ derive( Debug, Clone ) ]
pub struct EndpointHealth
{
pub endpoint : String,
pub status : HealthStatus,
pub last_check : SystemTime,
pub response_time : Option< Duration >,
pub consecutive_failures : u32,
}
#[ derive( Debug, Clone ) ]
pub struct HealthCheckResult
{
pub primary_healthy : bool,
pub backup_available : bool,
pub endpoint_health : Vec< EndpointHealth >,
pub recommended_endpoint : Option< String >,
}
#[ derive( Debug, Clone ) ]
pub struct FailoverMetrics
{
pub total_endpoints : usize,
pub failover_count : u64,
pub endpoint_health : Vec< EndpointHealth >,
pub active_endpoint : String,
}
#[ allow( missing_debug_implementations ) ] pub struct FailoverManager
{
client : crate::client::Client,
config : FailoverConfig,
metrics : Arc< Mutex< FailoverMetrics > >,
round_robin_index : Arc< Mutex< usize > >,
endpoint_health : Arc< Mutex< HashMap< String, EndpointHealth > > >,
}
impl FailoverManager
{
#[ inline ]
#[ must_use ]
pub fn new( client : crate::client::Client, config : FailoverConfig ) -> Self
{
let total_endpoints = 1 + config.backup_endpoints.len();
let metrics = FailoverMetrics {
total_endpoints,
failover_count : 0,
endpoint_health : Vec::new(),
active_endpoint : config.primary_endpoint.clone(),
};
Self {
client,
config,
metrics : Arc::new( Mutex::new( metrics ) ),
round_robin_index : Arc::new( Mutex::new( 0 ) ),
endpoint_health : Arc::new( Mutex::new( HashMap::new() ) ),
}
}
#[ inline ]
#[ must_use ]
pub fn current_config( &self ) -> &FailoverConfig
{
&self.config
}
#[ inline ]
pub async fn check_endpoint_health( &self ) -> Result< HealthCheckResult, crate::error::Error >
{
let mut endpoint_health = Vec::new();
let mut backup_available = false;
let primary_health = self.check_single_endpoint( &self.config.primary_endpoint ).await;
let primary_healthy = primary_health.status == HealthStatus::Healthy;
endpoint_health.push( primary_health );
for backup in &self.config.backup_endpoints
{
let backup_health = self.check_single_endpoint( backup ).await;
if backup_health.status == HealthStatus::Healthy
{
backup_available = true;
}
endpoint_health.push( backup_health );
}
{
let mut stored_health = self.endpoint_health.lock().unwrap();
for health in &endpoint_health
{
stored_health.insert( health.endpoint.clone(), health.clone() );
}
}
let recommended_endpoint = if !primary_healthy && backup_available
{
self.get_next_healthy_endpoint().ok()
} else {
None
};
Ok( HealthCheckResult {
primary_healthy,
backup_available,
endpoint_health,
recommended_endpoint,
} )
}
async fn check_single_endpoint( &self, endpoint : &str ) -> EndpointHealth
{
let start_time = SystemTime::now();
let client = reqwest::Client::builder()
.timeout( self.config.timeout )
.build()
.unwrap_or_default();
let result = client.head( endpoint ).send().await;
let ( status, response_time, consecutive_failures ) = match result
{
Ok( response ) => {
if response.status().is_success() || response.status().is_redirection()
{
( HealthStatus::Healthy, start_time.elapsed().ok(), 0 )
} else {
( HealthStatus::Unhealthy, None, 1 )
}
},
Err( _ ) => ( HealthStatus::Unhealthy, None, 1 ),
};
EndpointHealth {
endpoint : endpoint.to_string(),
status,
last_check : start_time,
response_time,
consecutive_failures,
}
}
#[ inline ]
pub fn get_next_endpoint( &self ) -> Result< String, crate::error::Error >
{
match self.config.strategy
{
FailoverStrategy::Priority => {
self.config.backup_endpoints.first()
.cloned()
.map_or_else(
|| Err( crate::error::Error::ConfigurationError(
"No backup endpoints configured".to_string()
) ),
Ok
)
},
FailoverStrategy::RoundRobin => {
let mut index = self.round_robin_index.lock().unwrap();
let backup_count = self.config.backup_endpoints.len();
if backup_count == 0
{
return Err( crate::error::Error::ConfigurationError(
"No backup endpoints configured".to_string()
) );
}
let selected = &self.config.backup_endpoints[ *index % backup_count ];
*index = ( *index + 1 ) % backup_count;
Ok( selected.clone() )
},
}
}
#[ inline ]
fn get_next_healthy_endpoint( &self ) -> Result< String, crate::error::Error >
{
self.config.backup_endpoints.first()
.cloned()
.map_or_else(
|| Err( crate::error::Error::ConfigurationError(
"No backup endpoints available".to_string()
) ),
Ok
)
}
#[ inline ]
pub fn switch_to_backup( &self ) -> Result< crate::client::Client, crate::error::Error >
{
let backup_endpoint = self.get_next_endpoint()?;
let backup_client = crate::client::Client::builder()
.api_key( std::env::var( "GEMINI_API_KEY" ).unwrap_or_else( |_| "test-key".to_string() ) )
.base_url( backup_endpoint.clone() )
.build()?;
{
let mut metrics = self.metrics.lock().unwrap();
metrics.failover_count += 1;
metrics.active_endpoint = backup_endpoint;
}
Ok( backup_client )
}
#[ inline ]
pub async fn execute_with_failover< F, Fut, T >(
&self,
operation : F
) -> Result< T, crate::error::Error >
where
F: Fn( crate::client::Client ) -> Fut,
Fut : Future< Output = Result< T, crate::error::Error > >,
{
if let Ok( result ) = operation( self.client.clone() ).await
{
Ok( result )
} else {
let backup_client = self.switch_to_backup()?;
operation( backup_client ).await
}
}
#[ inline ]
#[ must_use ]
pub fn get_metrics( &self ) -> FailoverMetrics
{
let mut metrics = self.metrics.lock().unwrap();
let stored_health = self.endpoint_health.lock().unwrap();
metrics.endpoint_health = stored_health.values().cloned().collect();
metrics.clone()
}
}
#[ derive( Debug ) ]
pub struct FailoverBuilder
{
client : crate::client::Client,
}
impl FailoverBuilder
{
#[ inline ]
#[ must_use ]
pub fn new( client : crate::client::Client ) -> Self
{
Self { client }
}
#[ inline ]
#[ must_use ]
pub fn configure( self, config : FailoverConfig ) -> FailoverManager
{
FailoverManager::new( self.client, config )
}
}
}
::mod_interface::mod_interface!
{
exposed use private::FailoverConfig;
exposed use private::FailoverConfigBuilder;
exposed use private::FailoverStrategy;
exposed use private::HealthStatus;
exposed use private::EndpointHealth;
exposed use private::HealthCheckResult;
exposed use private::FailoverMetrics;
exposed use private::FailoverManager;
exposed use private::FailoverBuilder;
}