use super::{
Arc, Duration, HEALTH_PROBE_FAILURE_THRESHOLD, HEALTH_PROBE_TIMEOUT, ManagerError, Ordering,
SessionHandle, SessionManager, SessionState, TERMINAL_SESSION_RETENTION, Weak,
};
use crate::proto::{trading, worker};
impl SessionManager {
pub(crate) fn should_renew_cod(
state: Option<SessionState>,
cod: Option<&Arc<dyn CodProvider>>,
) -> bool {
state == Some(SessionState::Active) && cod.is_some_and(|c| c.supports_cod())
}
async fn gateway_health_check(&self) -> Result<(), crate::ports::PortError> {
let res = tokio::time::timeout(
HEALTH_PROBE_TIMEOUT,
self.gateway.sync_state(&self.default_exchange),
)
.await
.map_err(|_| {
crate::ports::PortError::Transport("gateway health check timeout".to_string())
})?;
res?;
Ok(())
}
pub(crate) fn note_health_probe(&self, probe: &Result<(), crate::ports::PortError>) -> u32 {
if probe.is_ok() {
self.health_failures.store(0, Ordering::Release);
return 0;
}
loop {
if self.health_failures.load(Ordering::Acquire) == u32::MAX {
return u32::MAX;
}
if let Ok(prev) =
self.health_failures
.try_update(Ordering::AcqRel, Ordering::Acquire, |v| v.checked_add(1))
{
return prev.saturating_add(1);
}
}
}
pub async fn check_lease_expiry(&self, session_id: &str) -> Result<bool, ManagerError> {
let handle = self.get(session_id).await?;
let policy = handle.policy().await;
let expired = handle.state() == Some(SessionState::Active) &&
u128::from(handle.last_seen_elapsed_ms().await) > policy.lease_timeout.as_millis();
if expired {
self.trip_kill_switch(&handle, policy.lease_timeout).await;
}
Ok(expired)
}
pub(crate) async fn ensure_watchdog(&self) {
if self.watchdog_started.load(Ordering::Acquire) {
return;
}
let Some(weak) = self.self_weak.lock().await.clone() else {
tracing::warn!(
"SessionManager::ensure_watchdog without install_self: leases will never expire; \
call install_self() after SessionManager::new()"
);
return;
};
if self.watchdog_started.swap(true, Ordering::AcqRel) {
return;
}
tokio::spawn(watchdog_loop(weak));
}
async fn trip_kill_switch(&self, handle: &SessionHandle, timeout: Duration) {
match handle.state() {
Some(SessionState::Active) => {}
state => {
if state.is_none() {
tracing::error!(session = %handle.id, "corrupt session state; refusing kill-switch");
}
return;
}
}
let Some(prev) = handle.transition(SessionState::Active, SessionState::KillSwitchTripped)
else {
return;
};
handle.mark_terminal().await;
tracing::error!(
session = %handle.id,
lease_timeout_ms = u64::try_from(timeout.as_millis()).unwrap_or(u64::MAX),
"session lease expired; tripping kill-switch"
);
self.publish_state_change(handle, prev, SessionState::KillSwitchTripped, "lease expired")
.await;
let policy = handle.policy().await;
match policy.scope {
worker::kill_switch_policy::Scope::SessionOrders => {
let failures = self.cancel_session_orders(handle).await;
if !failures.is_empty() {
tracing::error!(
session = %handle.id,
failures = ?failures,
"kill-switch scope SESSION_ORDERS left orders resting on the venue"
);
}
}
worker::kill_switch_policy::Scope::AllOrders => {
let req = trading::CancelAllOrdersRequest {
exchange_id: buffa::MessageField::some(self.default_exchange.clone()),
..Default::default()
};
if let Err(err) = self.gateway.cancel_all_orders(req).await {
tracing::error!(
session = %handle.id,
error = %err,
"kill-switch cancel-all failed"
);
}
}
_ => {
tracing::warn!(session = %handle.id, "kill-switch scope NONE; no cancellations");
}
}
}
}
#[async_trait::async_trait]
pub trait CodProvider: Send + Sync {
async fn keepalive(&self, session_id: &str) -> Result<(), crate::ports::PortError>;
fn supports_cod(&self) -> bool {
true
}
}
pub struct NullCodProvider;
#[async_trait::async_trait]
impl CodProvider for NullCodProvider {
async fn keepalive(&self, _session_id: &str) -> Result<(), crate::ports::PortError> {
Ok(())
}
fn supports_cod(&self) -> bool {
false
}
}
#[allow(clippy::cognitive_complexity, clippy::collapsible_if)]
async fn watchdog_loop(weak: Weak<SessionManager>) {
let mut tick = tokio::time::interval(Duration::from_secs(1));
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
tick.tick().await;
let Some(manager) = weak.upgrade() else {
return;
};
manager.retire_terminal_sessions(TERMINAL_SESSION_RETENTION).await;
let handles: Vec<Arc<SessionHandle>> = {
let sessions = manager.sessions.lock().await;
sessions.values().map(Arc::clone).collect()
};
let semaphore = Arc::new(tokio::sync::Semaphore::new(32));
let mut futures = Vec::with_capacity(handles.len());
for handle in handles {
let manager = Arc::clone(&manager);
let sem = Arc::clone(&semaphore);
futures.push(tokio::spawn(async move {
let _permit = sem.acquire().await;
if SessionManager::should_renew_cod(handle.state(), manager.cod_provider.as_ref())
&& let Some(cod) = manager.cod_provider.as_ref()
&& let Err(e) = cod.keepalive(&handle.id).await
{
tracing::debug!(session = %handle.id, error = %e, "COD keepalive failed — will rely on L1/L2");
}
match manager.check_lease_expiry(&handle.id).await {
Ok(true) => {
tracing::warn!(
session = %handle.id,
"L1 lease expired — kill-switch executed (L2/L3 as fallback)"
);
}
Ok(false) => {}
Err(error) => {
tracing::warn!(
session = %handle.id,
error = %error,
"L1 lease check failed; leaving the session alone"
);
}
}
}));
}
for f in futures {
let _ = f.await;
}
let probe = manager.gateway_health_check().await;
let misses = manager.note_health_probe(&probe);
let tripped = misses >= HEALTH_PROBE_FAILURE_THRESHOLD;
if let Err(error) = &probe {
if tripped {
tracing::error!(
error = %error,
consecutive_misses = misses,
"L2 daemon watchdog tripping all active sessions"
);
} else {
tracing::warn!(
error = %error,
consecutive_misses = misses,
"gateway health probe failed; below the L2 trip threshold"
);
}
}
if tripped {
let active: Vec<Arc<SessionHandle>> = {
let sessions = manager.sessions.lock().await;
sessions
.values()
.filter(|h| h.state() == Some(SessionState::Active))
.cloned()
.collect()
};
let semaphore = Arc::new(tokio::sync::Semaphore::new(32));
let mut jobs = Vec::with_capacity(active.len());
for handle in active {
let manager = Arc::clone(&manager);
let sem = Arc::clone(&semaphore);
jobs.push(tokio::spawn(async move {
let _permit = sem.acquire().await;
let lease_timeout = handle.policy().await.lease_timeout;
manager.trip_kill_switch(&handle, lease_timeout).await;
}));
}
for job in jobs {
let _ = job.await;
}
}
}
}
pub fn spawn_strategy_lease_guard(
manager: Arc<SessionManager>,
session_id: String,
heartbeat_interval: Duration,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let mut tick = tokio::time::interval(heartbeat_interval);
loop {
tick.tick().await;
match manager.check_lease_expiry(&session_id).await {
Ok(true) => {
tracing::warn!(
session = %session_id,
"strategy-side lease timeout detected; kill-switch executed"
);
break;
}
Ok(false) => {}
Err(_) => break,
}
}
})
}