use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::time::Duration;
use tokio::sync::{RwLock, broadcast};
use tokio::time::Instant;
use crate::client::CortexClient;
use crate::config::CortexConfig;
use crate::error::CortexResult;
use crate::health::HealthMonitor;
use crate::protocol::warnings::CortexWarning;
mod endpoints;
mod operation_layer;
mod reconnect_layer;
mod token_layer;
const TOKEN_REFRESH_INTERVAL: Duration = Duration::from_secs(55 * 60);
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ConnectionEvent {
Connected,
Disconnected { reason: String },
Reconnecting { attempt: u32 },
Reconnected,
ReconnectFailed { attempts: u32, last_error: String },
}
struct ClientState {
client: Arc<CortexClient>,
cortex_token: String,
token_obtained_at: Instant,
}
pub struct ResilientClient {
config: CortexConfig,
state: RwLock<ClientState>,
event_tx: broadcast::Sender<ConnectionEvent>,
warning_tx: broadcast::Sender<CortexWarning>,
reconnecting: Arc<AtomicBool>,
health_monitor: std::sync::Mutex<Option<HealthMonitor>>,
}
impl ResilientClient {
pub async fn connect(config: CortexConfig) -> CortexResult<Self> {
let client = CortexClient::connect(&config).await?;
let warning_rx = client.warning_receiver();
let cortex_token = client
.authenticate(&config.client_id, &config.client_secret)
.await?;
let (event_tx, _) = broadcast::channel(64);
let _ = event_tx.send(ConnectionEvent::Connected);
let (warning_tx, _) = broadcast::channel(64);
let client = Arc::new(client);
Self::spawn_warning_relay(&warning_tx, warning_rx);
let state = ClientState {
client,
cortex_token,
token_obtained_at: Instant::now(),
};
let resilient = Self {
config,
state: RwLock::new(state),
event_tx,
warning_tx,
reconnecting: Arc::new(AtomicBool::new(false)),
health_monitor: std::sync::Mutex::new(None),
};
if resilient.config.health.enabled {
resilient.start_health_monitor().await;
}
Ok(resilient)
}
pub fn event_receiver(&self) -> broadcast::Receiver<ConnectionEvent> {
self.event_tx.subscribe()
}
pub fn warning_receiver(&self) -> broadcast::Receiver<CortexWarning> {
self.warning_tx.subscribe()
}
pub(super) fn spawn_warning_relay(
warning_tx: &broadcast::Sender<CortexWarning>,
mut rx: broadcast::Receiver<CortexWarning>,
) {
let tx = warning_tx.clone();
tokio::spawn(async move {
loop {
match rx.recv().await {
Ok(warning) => {
let _ = tx.send(warning);
}
Err(broadcast::error::RecvError::Lagged(skipped)) => {
tracing::warn!(skipped, "Warning relay lagged; warnings dropped");
}
Err(broadcast::error::RecvError::Closed) => break,
}
}
});
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_connection_event_variants() {
let connected = ConnectionEvent::Connected;
let disconnected = ConnectionEvent::Disconnected {
reason: "test".into(),
};
let reconnecting = ConnectionEvent::Reconnecting { attempt: 1 };
let reconnected = ConnectionEvent::Reconnected;
let failed = ConnectionEvent::ReconnectFailed {
attempts: 3,
last_error: "timeout".into(),
};
assert_eq!(connected, ConnectionEvent::Connected);
assert_ne!(connected, reconnected);
assert_eq!(
disconnected,
ConnectionEvent::Disconnected {
reason: "test".into()
}
);
assert_eq!(reconnecting, ConnectionEvent::Reconnecting { attempt: 1 });
assert_ne!(reconnecting, ConnectionEvent::Reconnecting { attempt: 2 });
assert_eq!(
failed,
ConnectionEvent::ReconnectFailed {
attempts: 3,
last_error: "timeout".into()
}
);
}
#[test]
fn test_token_refresh_interval() {
assert_eq!(TOKEN_REFRESH_INTERVAL, Duration::from_secs(55 * 60));
}
}