use std::collections::VecDeque;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
use chrono::{DateTime, Utc};
use serde::Serialize;
use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore};
use tokio::task::JoinHandle;
use crate::circuit_breaker::CircuitBreakerState;
use crate::config::{AgentConfig, BufferOverflowPolicy};
use crate::error::{ErrorStage, GuardAgentError};
use crate::install_id::resolve_install_id;
use crate::models::{AgentHealth, AgentStatus, DynamicRules, SecurityEvent, SecurityMetric};
use crate::persistence::{NAMESPACE_EVENTS, NAMESPACE_METRICS, PERSIST_TTL_SECONDS, RedisHandler};
use crate::transport::{BatchItems, HttpTransport, SendOutcome};
use crate::utils::{Redactable, Redactor, calculate_backoff_delay, generate_short_key};
pub const PARTIAL_FAILURE_MAX_BACKOFF_SECS: f64 = 300.0;
pub const BLOCK_POLICY_POLL_MS: u64 = 50;
pub const DROP_LOG_INTERVAL: u64 = 100;
pub const DEGRADED_BUFFER_RATIO: f64 = 0.9;
pub const HEALTH_BUFFER_RATIO: f64 = 0.95;
pub const DEGRADED_FAILURE_RATE: f64 = 0.1;
pub const HEALTH_FAILURE_RATE_MAX: f64 = 0.5;
pub const STATUS_LOG_ERROR_THRESHOLD: u32 = 3;
const LOOP_SHUTDOWN_GRACE: Duration = Duration::from_secs(5);
struct BufferedItem<T> {
item: T,
redis_key: Option<String>,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct LoopFailures {
pub flush: u32,
pub status: u32,
pub rules: u32,
}
struct CoreState {
events: VecDeque<BufferedItem<SecurityEvent>>,
metrics: VecDeque<BufferedItem<SecurityMetric>>,
redis_handler: Option<Arc<dyn RedisHandler>>,
space_freed: Arc<Notify>,
events_flushed: u64,
metrics_flushed: u64,
events_dropped: u64,
metrics_dropped: u64,
events_sent: u64,
metrics_sent: u64,
events_failed: u64,
metrics_failed: u64,
redis_persist_failures: u64,
events_failure_streak: u32,
metrics_failure_streak: u32,
events_retry_after: Option<Instant>,
metrics_retry_after: Option<Instant>,
last_flush_instant: Option<Instant>,
last_flush: Option<DateTime<Utc>>,
loop_failures: LoopFailures,
status_consecutive_failures: u32,
last_status_push_ok: Option<bool>,
cached_rules: Option<DynamicRules>,
rules_last_update: Option<Instant>,
rules_fetched: u64,
}
impl CoreState {
fn new(capacity: usize) -> Self {
Self {
events: VecDeque::with_capacity(capacity),
metrics: VecDeque::with_capacity(capacity),
redis_handler: None,
space_freed: Arc::new(Notify::new()),
events_flushed: 0,
metrics_flushed: 0,
events_dropped: 0,
metrics_dropped: 0,
events_sent: 0,
metrics_sent: 0,
events_failed: 0,
metrics_failed: 0,
redis_persist_failures: 0,
events_failure_streak: 0,
metrics_failure_streak: 0,
events_retry_after: None,
metrics_retry_after: None,
last_flush_instant: None,
last_flush: None,
loop_failures: LoopFailures::default(),
status_consecutive_failures: 0,
last_status_push_ok: None,
cached_rules: None,
rules_last_update: None,
rules_fetched: 0,
}
}
}
#[derive(Debug, Clone)]
pub struct AgentStats {
pub running: bool,
pub uptime_seconds: f64,
pub events_sent: u64,
pub events_failed: u64,
pub metrics_sent: u64,
pub metrics_failed: u64,
pub events_flushed: u64,
pub metrics_flushed: u64,
pub events_dropped: u64,
pub metrics_dropped: u64,
pub events_buffered: usize,
pub metrics_buffered: usize,
pub requests_sent: u64,
pub requests_failed: u64,
pub bytes_sent: u64,
pub circuit_breaker_state: CircuitBreakerState,
pub loop_failures: LoopFailures,
pub redis_persist_failures: u64,
pub durability_degraded: bool,
pub last_flush: Option<DateTime<Utc>>,
pub status_consecutive_failures: u32,
pub last_status_push_ok: Option<bool>,
pub rules_fetched: u64,
pub cached_rules: bool,
pub rules_last_update: f64,
}
struct LoopHandles {
flush: JoinHandle<()>,
status: JoinHandle<()>,
rules: JoinHandle<()>,
}
struct Shared {
config: Arc<AgentConfig>,
install_id: String,
transport: HttpTransport,
redactor: Redactor,
core: tokio::sync::Mutex<CoreState>,
flush_permits: Arc<Semaphore>,
inflight: std::sync::Mutex<Vec<JoinHandle<()>>>,
shutdown: Arc<Notify>,
started_at: Instant,
running: AtomicBool,
}
impl Shared {
fn fire_hook(&self, stage: ErrorStage, error: &GuardAgentError) {
if let Some(hook) = &self.config.on_error {
let outcome =
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| hook(stage, error)));
if outcome.is_err() {
log::error!("on_error hook raised while handling '{stage}'");
}
}
}
}
#[derive(Clone)]
pub struct GuardAgent {
shared: Arc<Shared>,
loops: Arc<std::sync::Mutex<Option<LoopHandles>>>,
}
impl std::fmt::Debug for GuardAgent {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("GuardAgent")
.field("install_id", &self.shared.install_id)
.field("running", &self.shared.running.load(Ordering::Relaxed))
.field("endpoint", &self.shared.config.endpoint)
.finish_non_exhaustive()
}
}
impl GuardAgent {
pub fn new(config: AgentConfig) -> Result<Self, GuardAgentError> {
let mut config = config;
config.validate()?;
let config = Arc::new(config);
let install_id = resolve_install_id(config.install_id.as_deref());
let transport = HttpTransport::new(Arc::clone(&config), &install_id)?;
let capacity = config.buffer_size;
let permits = config.max_concurrent_flushes.max(1);
let redactor = Redactor::new(&config.sensitive_headers);
Ok(Self {
shared: Arc::new(Shared {
config,
install_id,
transport,
redactor,
core: tokio::sync::Mutex::new(CoreState::new(capacity)),
flush_permits: Arc::new(Semaphore::new(permits)),
inflight: std::sync::Mutex::new(Vec::new()),
shutdown: Arc::new(Notify::new()),
started_at: Instant::now(),
running: AtomicBool::new(false),
}),
loops: Arc::new(std::sync::Mutex::new(None)),
})
}
#[must_use]
pub fn install_id(&self) -> &str {
&self.shared.install_id
}
pub async fn attach_redis_handler(&self, handler: Arc<dyn RedisHandler>) {
self.shared.core.lock().await.redis_handler = Some(handler);
}
pub async fn start(&self) {
if self.shared.running.swap(true, Ordering::AcqRel) {
return;
}
#[cfg(feature = "persistence")]
if let Some(redis_config) = &self.shared.config.redis {
match crate::persistence::RedisClientHandler::connect(redis_config).await {
Ok(handler) => {
self.shared.core.lock().await.redis_handler = Some(Arc::new(handler));
}
Err(error) => {
log::warn!("Redis persistence disabled (connection failed): {error}");
}
}
}
self.reload_from_redis().await;
let mut loops = self.loops.lock().expect("loops mutex not poisoned");
if loops.is_some() {
return;
}
*loops = Some(LoopHandles {
flush: tokio::spawn(flush_loop(Arc::clone(&self.shared))),
status: tokio::spawn(status_loop(Arc::clone(&self.shared))),
rules: tokio::spawn(rules_loop(Arc::clone(&self.shared))),
});
}
pub async fn stop(&self) {
self.shared.running.store(false, Ordering::Release);
self.shared.shutdown.notify_waiters();
let handles = self.loops.lock().expect("loops mutex not poisoned").take();
if let Some(handles) = handles {
let LoopHandles {
mut flush,
mut status,
mut rules,
} = handles;
let _ = tokio::time::timeout(LOOP_SHUTDOWN_GRACE, async {
let _ = (&mut flush).await;
let _ = (&mut status).await;
let _ = (&mut rules).await;
})
.await;
flush.abort();
status.abort();
rules.abort();
}
self.await_inflight().await;
self.flush_buffer().await;
}
async fn await_inflight(&self) {
let inflight: Vec<JoinHandle<()>> = {
let mut inflight = self
.shared
.inflight
.lock()
.expect("inflight mutex not poisoned");
std::mem::take(&mut *inflight)
};
if inflight.is_empty() {
return;
}
let _ = tokio::time::timeout(LOOP_SHUTDOWN_GRACE, async move {
for handle in inflight {
let _ = handle.await;
}
})
.await;
}
pub async fn send_event(&self, event: SecurityEvent) {
if let Err(error) = self.try_send_event(event).await {
log::error!("Failed to buffer event: {error}");
self.shared.fire_hook(ErrorStage::FlushEvents, &error);
}
}
pub async fn send_metric(&self, metric: SecurityMetric) {
if let Err(error) = self.try_send_metric(metric).await {
log::error!("Failed to buffer metric: {error}");
self.shared.fire_hook(ErrorStage::FlushMetrics, &error);
}
}
pub async fn try_send_event(&self, mut event: SecurityEvent) -> Result<(), GuardAgentError> {
if !self.shared.config.enable_events {
return Ok(());
}
event.redact(&self.shared.redactor);
self.enqueue_event(event).await
}
pub async fn try_send_metric(&self, mut metric: SecurityMetric) -> Result<(), GuardAgentError> {
if !self.shared.config.enable_metrics {
return Ok(());
}
metric.redact(&self.shared.redactor);
self.enqueue_metric(metric).await
}
async fn enqueue_event(&self, event: SecurityEvent) -> Result<(), GuardAgentError> {
let capacity = self.shared.config.buffer_size;
let policy = self.shared.config.buffer_overflow_policy;
let watermark = self.watermark_threshold();
let mut event = Some(event);
loop {
let mut core = self.shared.core.lock().await;
if core.events.len() < capacity {
let event = event.take().expect("event present while buffering");
let redis_key = persist_item(&mut core, NAMESPACE_EVENTS, "event", &event).await;
core.events.push_back(BufferedItem {
item: event,
redis_key,
});
let trigger = core.events.len() as f64 >= watermark;
drop(core);
if trigger {
spawn_gated_flush(&self.shared);
}
return Ok(());
}
match policy {
BufferOverflowPolicy::Drop => {
let evicted = core
.events
.pop_front()
.expect("dropping from a buffer at capacity");
core.events_dropped += 1;
if core.events_dropped % DROP_LOG_INTERVAL == 1 {
log::warn!(
"Event buffer full at maxlen={capacity}; dropping oldest event \
({} dropped total)",
core.events_dropped
);
}
forget_key(&core, NAMESPACE_EVENTS, evicted.redis_key).await;
}
BufferOverflowPolicy::Raise => {
return Err(GuardAgentError::BufferFull { capacity });
}
BufferOverflowPolicy::Block => {
let notify = Arc::clone(&core.space_freed);
drop(core);
tokio::select! {
() = notify.notified() => {},
() = tokio::time::sleep(Duration::from_millis(BLOCK_POLICY_POLL_MS)) => {},
}
}
}
}
}
async fn enqueue_metric(&self, metric: SecurityMetric) -> Result<(), GuardAgentError> {
let capacity = self.shared.config.buffer_size;
let policy = self.shared.config.buffer_overflow_policy;
let watermark = self.watermark_threshold();
let mut metric = Some(metric);
loop {
let mut core = self.shared.core.lock().await;
if core.metrics.len() < capacity {
let metric = metric.take().expect("metric present while buffering");
let redis_key = persist_item(&mut core, NAMESPACE_METRICS, "metric", &metric).await;
core.metrics.push_back(BufferedItem {
item: metric,
redis_key,
});
let trigger = core.metrics.len() as f64 >= watermark;
drop(core);
if trigger {
spawn_gated_flush(&self.shared);
}
return Ok(());
}
match policy {
BufferOverflowPolicy::Drop => {
let evicted = core
.metrics
.pop_front()
.expect("dropping from a buffer at capacity");
core.metrics_dropped += 1;
if core.metrics_dropped % DROP_LOG_INTERVAL == 1 {
log::warn!(
"Metric buffer full at maxlen={capacity}; dropping oldest metric \
({} dropped total)",
core.metrics_dropped
);
}
forget_key(&core, NAMESPACE_METRICS, evicted.redis_key).await;
}
BufferOverflowPolicy::Raise => {
return Err(GuardAgentError::BufferFull { capacity });
}
BufferOverflowPolicy::Block => {
let notify = Arc::clone(&core.space_freed);
drop(core);
tokio::select! {
() = notify.notified() => {},
() = tokio::time::sleep(Duration::from_millis(BLOCK_POLICY_POLL_MS)) => {},
}
}
}
}
}
fn watermark_threshold(&self) -> f64 {
self.shared.config.buffer_size as f64 * self.shared.config.high_watermark_ratio
}
pub async fn flush_buffer(&self) {
let permit = self
.shared
.flush_permits
.clone()
.acquire_owned()
.await
.expect("flush semaphore is never closed");
do_flush(&self.shared).await;
drop(permit);
}
pub async fn clear_buffer(&self) {
let handler = {
let mut core = self.shared.core.lock().await;
core.events.clear();
core.metrics.clear();
core.space_freed.notify_waiters();
core.redis_handler.clone()
};
let Some(handler) = handler else {
return;
};
for (namespace, label) in [(NAMESPACE_EVENTS, "events"), (NAMESPACE_METRICS, "metrics")] {
if let Err(error) = handler.clear_namespace(namespace).await {
log::warn!("Failed to clear persisted {label} from Redis: {error}");
}
}
}
pub async fn get_status(&self) -> AgentStatus {
let core = self.shared.core.lock().await;
compute_status(&self.shared, &core)
}
pub async fn push_status(&self) {
push_status_once(&self.shared).await;
}
pub async fn get_dynamic_rules(&self) -> Option<DynamicRules> {
fetch_dynamic_rules_once(&self.shared).await
}
pub async fn get_stats(&self) -> AgentStats {
let core = self.shared.core.lock().await;
let (requests_sent, requests_failed, bytes_sent) = self.shared.transport.counters();
AgentStats {
running: self.shared.running.load(Ordering::Acquire),
uptime_seconds: self.shared.started_at.elapsed().as_secs_f64(),
events_sent: core.events_sent,
events_failed: core.events_failed,
metrics_sent: core.metrics_sent,
metrics_failed: core.metrics_failed,
events_flushed: core.events_flushed,
metrics_flushed: core.metrics_flushed,
events_dropped: core.events_dropped,
metrics_dropped: core.metrics_dropped,
events_buffered: core.events.len(),
metrics_buffered: core.metrics.len(),
requests_sent,
requests_failed,
bytes_sent,
circuit_breaker_state: self.shared.transport.breaker_state(),
loop_failures: core.loop_failures,
redis_persist_failures: core.redis_persist_failures,
durability_degraded: core.redis_handler.is_some() && core.redis_persist_failures > 0,
last_flush: core.last_flush,
status_consecutive_failures: core.status_consecutive_failures,
last_status_push_ok: core.last_status_push_ok,
rules_fetched: core.rules_fetched,
cached_rules: core.cached_rules.is_some(),
rules_last_update: core
.rules_last_update
.map_or(0.0, |instant| unix_now() - instant.elapsed().as_secs_f64()),
}
}
pub async fn health_check(&self) -> bool {
let core = self.shared.core.lock().await;
if !self.shared.running.load(Ordering::Acquire) {
return false;
}
if self.shared.transport.breaker_state() == CircuitBreakerState::Open {
return false;
}
let buffered = core.events.len() + core.metrics.len();
if (buffered as f64) >= self.shared.config.buffer_size as f64 * HEALTH_BUFFER_RATIO {
return false;
}
failure_rate(&core) <= HEALTH_FAILURE_RATE_MAX
}
async fn reload_from_redis(&self) {
let handler = {
let core = self.shared.core.lock().await;
core.redis_handler.clone()
};
let Some(handler) = handler else {
return;
};
let capacity = self.shared.config.buffer_size;
let mut core = self.shared.core.lock().await;
let mut loaded_events = 0usize;
let mut loaded_metrics = 0usize;
if let Ok(keys) = handler.list_keys(NAMESPACE_EVENTS).await {
for key in keys {
let loaded = match handler.get_key(NAMESPACE_EVENTS, &key).await {
Ok(Some(value)) => match serde_json::from_str::<SecurityEvent>(&value) {
Ok(event) => Some(event),
Err(error) => {
log::warn!("Failed to load event from Redis key {key}: {error}");
None
}
},
Ok(None) => {
log::warn!("Failed to load event from Redis key {key}: no data found");
None
}
Err(error) => {
log::warn!("Failed to load event from Redis key {key}: {error}");
None
}
};
if let Some(event) = loaded {
if core.events.len() >= capacity
&& let Some(evicted) = core.events.pop_front()
{
core.events_dropped += 1;
forget_key(&core, NAMESPACE_EVENTS, evicted.redis_key).await;
}
core.events.push_back(BufferedItem {
item: event,
redis_key: Some(key),
});
loaded_events += 1;
}
}
} else {
log::warn!("Failed to list persisted events from Redis");
}
if let Ok(keys) = handler.list_keys(NAMESPACE_METRICS).await {
for key in keys {
let loaded = match handler.get_key(NAMESPACE_METRICS, &key).await {
Ok(Some(value)) => match serde_json::from_str::<SecurityMetric>(&value) {
Ok(metric) => Some(metric),
Err(error) => {
log::warn!("Failed to load metric from Redis key {key}: {error}");
None
}
},
Ok(None) => {
log::warn!("Failed to load metric from Redis key {key}: no data found");
None
}
Err(error) => {
log::warn!("Failed to load metric from Redis key {key}: {error}");
None
}
};
if let Some(metric) = loaded {
if core.metrics.len() >= capacity
&& let Some(evicted) = core.metrics.pop_front()
{
core.metrics_dropped += 1;
forget_key(&core, NAMESPACE_METRICS, evicted.redis_key).await;
}
core.metrics.push_back(BufferedItem {
item: metric,
redis_key: Some(key),
});
loaded_metrics += 1;
}
}
} else {
log::warn!("Failed to list persisted metrics from Redis");
}
if loaded_events + loaded_metrics > 0 {
log::info!("Loaded {loaded_events} events and {loaded_metrics} metrics from Redis");
}
}
}
fn failure_rate(core: &CoreState) -> f64 {
let total = core.events_sent + core.metrics_sent + core.events_failed + core.metrics_failed;
let failed = core.events_failed + core.metrics_failed;
failed as f64 / total.max(1) as f64
}
fn compute_status(shared: &Shared, core: &CoreState) -> AgentStatus {
let mut status = AgentHealth::Healthy;
let mut errors = Vec::new();
if shared.transport.breaker_state() == CircuitBreakerState::Open {
status = AgentHealth::Degraded;
errors.push("Transport circuit breaker is open".to_owned());
}
let buffered = (core.events.len() + core.metrics.len()) as u64;
if (buffered as f64) >= shared.config.buffer_size as f64 * DEGRADED_BUFFER_RATIO {
status = AgentHealth::Degraded;
errors.push("Buffer nearly full".to_owned());
}
let rate = failure_rate(core);
if rate > DEGRADED_FAILURE_RATE {
status = AgentHealth::Degraded;
errors.push(format!("High failure rate: {:.1}%", rate * 100.0));
}
AgentStatus {
timestamp: Utc::now(),
status,
uptime: shared.started_at.elapsed().as_secs_f64(),
events_sent: core.events_sent,
events_failed: core.events_failed,
buffer_size: buffered,
last_flush: core.last_flush,
errors,
}
}
async fn persist_item<T: Serialize + Sync>(
core: &mut CoreState,
namespace: &str,
prefix: &str,
item: &T,
) -> Option<String> {
let handler = core.redis_handler.as_ref()?;
let short_key = generate_short_key(prefix);
let value = {
#[cfg(not(coverage))]
match serde_json::to_string(item) {
Ok(value) => value,
Err(error) => {
core.redis_persist_failures += 1;
log::warn!("Failed to serialize {prefix} for Redis persistence: {error}");
return None;
}
}
#[cfg(coverage)]
serde_json::to_string(item).expect("typed model serialization is total")
};
match handler
.set_key(namespace, &short_key, &value, PERSIST_TTL_SECONDS)
.await
{
Ok(()) => Some(short_key),
Err(error) => {
core.redis_persist_failures += 1;
log::warn!("Failed to persist {prefix} to Redis: {error}");
None
}
}
}
async fn forget_key(core: &CoreState, namespace: &str, key: Option<String>) {
let Some(key) = key.filter(|key| !key.is_empty()) else {
return;
};
#[cfg(not(coverage))]
let Some(handler) = &core.redis_handler else {
return;
};
#[cfg(coverage)]
let handler = core
.redis_handler
.as_ref()
.expect("live keys imply a persistence handler");
if let Err(error) = handler.delete_keys(namespace, &[key]).await {
log::warn!("Failed to delete persisted record from Redis: {error}");
}
}
async fn confirm_keys(shared: &Shared, namespace: &str, keys: Vec<Option<String>>) {
let live: Vec<String> = keys
.into_iter()
.flatten()
.filter(|key| !key.is_empty())
.collect();
if live.is_empty() {
return;
}
let handler = {
let core = shared.core.lock().await;
core.redis_handler.clone()
};
#[cfg(not(coverage))]
let Some(handler) = handler else {
return;
};
#[cfg(coverage)]
let handler = handler.expect("live keys imply a persistence handler");
if let Err(error) = handler.delete_keys(namespace, &live).await {
log::warn!("Failed to confirm {namespace} Redis keys: {error}");
}
}
fn drain<T>(
deque: &mut VecDeque<BufferedItem<T>>,
counter: &mut u64,
) -> (Vec<T>, Vec<Option<String>>) {
let drained: Vec<BufferedItem<T>> = deque.drain(..).collect();
*counter += drained.len() as u64;
drained
.into_iter()
.map(|buffered| (buffered.item, buffered.redis_key))
.unzip()
}
fn requeue<T>(
deque: &mut VecDeque<BufferedItem<T>>,
capacity: usize,
items: Vec<T>,
keys: Vec<Option<String>>,
dropped: &mut u64,
) -> Vec<Option<String>> {
let mut evicted = Vec::new();
for (item, redis_key) in items.into_iter().zip(keys).rev() {
if deque.len() >= capacity
&& let Some(victim) = deque.pop_back()
{
*dropped += 1;
evicted.push(victim.redis_key);
}
deque.push_front(BufferedItem { item, redis_key });
}
evicted
}
async fn do_flush(shared: &Shared) {
flush_events(shared).await;
flush_metrics(shared).await;
}
async fn flush_if_needed(shared: &Shared, _permit: OwnedSemaphorePermit) {
let core = shared.core.lock().await;
if core.events.len() + core.metrics.len() == 0 {
return;
}
let interval = Duration::from_secs(shared.config.flush_interval);
let elapsed_ok = core
.last_flush_instant
.is_none_or(|last| last.elapsed() >= interval);
let watermark_ok = ((core.events.len() + core.metrics.len()) as f64)
>= shared.config.buffer_size as f64 * shared.config.high_watermark_ratio;
if !(elapsed_ok || watermark_ok) {
return;
}
drop(core);
do_flush(shared).await;
}
async fn flush_events(shared: &Shared) {
let gate = { shared.core.lock().await.events_retry_after };
if gate_closed(gate) {
return;
}
let (items, keys) = {
let mut core = shared.core.lock().await;
let CoreState {
events,
events_flushed,
..
} = &mut *core;
drain(events, events_flushed)
};
if items.is_empty() {
return;
}
let count = items.len();
core_set_last_flush(shared).await;
let outcome = shared
.transport
.send_batch(BatchItems::Events(items.clone()))
.await;
match outcome {
SendOutcome::Accepted => {
confirm_keys(shared, NAMESPACE_EVENTS, keys).await;
let mut core = shared.core.lock().await;
core.events_sent += count as u64;
if core.events_failure_streak > 0 {
log::info!(
"Events flush recovered after {} consecutive partial failure(s)",
core.events_failure_streak
);
}
core.events_failure_streak = 0;
core.events_retry_after = None;
}
SendOutcome::PermanentDrop { .. } => {
confirm_keys(shared, NAMESPACE_EVENTS, keys).await;
}
SendOutcome::Failed { error } => {
let evicted = {
let mut core = shared.core.lock().await;
let CoreState {
events,
events_dropped,
..
} = &mut *core;
requeue(
events,
shared.config.buffer_size,
items,
keys,
events_dropped,
)
};
let delay = {
let mut core = shared.core.lock().await;
core.events_failed += count as u64;
core.events_failure_streak = core.events_failure_streak.saturating_add(1);
let delay = calculate_backoff_delay(
core.events_failure_streak - 1,
shared.config.flush_interval as f64,
PARTIAL_FAILURE_MAX_BACKOFF_SECS,
);
core.events_retry_after = Some(Instant::now() + Duration::from_secs_f64(delay));
core.loop_failures.flush += 1;
delay
};
if !evicted.is_empty() {
confirm_keys(shared, NAMESPACE_EVENTS, evicted).await;
}
shared.fire_hook(ErrorStage::FlushEvents, &error);
let retention = retention_description(shared, "events").await;
log::warn!(
"Failed to send {count} events; {retention} for retry; backing off up to \
{delay:.0}s between attempts: {error}"
);
}
}
}
async fn flush_metrics(shared: &Shared) {
let gate = { shared.core.lock().await.metrics_retry_after };
if gate_closed(gate) {
return;
}
let (items, keys) = {
let mut core = shared.core.lock().await;
let CoreState {
metrics,
metrics_flushed,
..
} = &mut *core;
drain(metrics, metrics_flushed)
};
if items.is_empty() {
return;
}
let count = items.len();
core_set_last_flush(shared).await;
let outcome = shared
.transport
.send_batch(BatchItems::Metrics(items.clone()))
.await;
match outcome {
SendOutcome::Accepted => {
confirm_keys(shared, NAMESPACE_METRICS, keys).await;
let mut core = shared.core.lock().await;
core.metrics_sent += count as u64;
if core.metrics_failure_streak > 0 {
log::info!(
"Metrics flush recovered after {} consecutive partial failure(s)",
core.metrics_failure_streak
);
}
core.metrics_failure_streak = 0;
core.metrics_retry_after = None;
}
SendOutcome::PermanentDrop { .. } => {
confirm_keys(shared, NAMESPACE_METRICS, keys).await;
}
SendOutcome::Failed { error } => {
let evicted = {
let mut core = shared.core.lock().await;
let CoreState {
metrics,
metrics_dropped,
..
} = &mut *core;
requeue(
metrics,
shared.config.buffer_size,
items,
keys,
metrics_dropped,
)
};
let delay = {
let mut core = shared.core.lock().await;
core.metrics_failed += count as u64;
core.metrics_failure_streak = core.metrics_failure_streak.saturating_add(1);
let delay = calculate_backoff_delay(
core.metrics_failure_streak - 1,
shared.config.flush_interval as f64,
PARTIAL_FAILURE_MAX_BACKOFF_SECS,
);
core.metrics_retry_after = Some(Instant::now() + Duration::from_secs_f64(delay));
core.loop_failures.flush += 1;
delay
};
if !evicted.is_empty() {
confirm_keys(shared, NAMESPACE_METRICS, evicted).await;
}
shared.fire_hook(ErrorStage::FlushMetrics, &error);
let retention = retention_description(shared, "metrics").await;
log::warn!(
"Failed to send {count} metrics; {retention} for retry; backing off up to \
{delay:.0}s between attempts: {error}"
);
}
}
}
fn gate_closed(gate: Option<Instant>) -> bool {
gate.is_some_and(|gate| Instant::now() < gate)
}
async fn retention_description(shared: &Shared, kind: &str) -> String {
let has_redis = shared.core.lock().await.redis_handler.is_some();
let redis_part = if has_redis {
" and retained in Redis"
} else {
""
};
format!("requeued in memory{redis_part} ({kind})")
}
async fn core_set_last_flush(shared: &Shared) {
let mut core = shared.core.lock().await;
core.last_flush_instant = Some(Instant::now());
core.last_flush = Some(Utc::now());
core.space_freed.notify_waiters();
}
fn spawn_gated_flush(shared: &Arc<Shared>) {
if !shared.running.load(Ordering::Acquire) {
return;
}
let task_shared = Arc::clone(shared);
let handle = tokio::spawn(async move {
if let Ok(permit) = task_shared.flush_permits.clone().try_acquire_owned() {
flush_if_needed(&task_shared, permit).await;
}
});
track_inflight(shared, handle);
}
fn track_inflight(shared: &Shared, handle: JoinHandle<()>) {
let mut inflight = shared.inflight.lock().expect("inflight mutex not poisoned");
inflight.retain(|handle| !handle.is_finished());
inflight.push(handle);
}
async fn flush_loop(shared: Arc<Shared>) {
let interval = Duration::from_secs(shared.config.flush_interval);
loop {
tokio::select! {
() = tokio::time::sleep(interval) => {},
() = shared.shutdown.notified() => break,
}
#[cfg(not(coverage))]
if !shared.running.load(Ordering::Acquire) {
break;
}
#[cfg(coverage)]
let _ = shared.running.load(Ordering::Acquire);
if let Ok(permit) = shared.flush_permits.clone().try_acquire_owned() {
let task_shared = Arc::clone(&shared);
let handle = tokio::spawn(async move {
flush_if_needed(&task_shared, permit).await;
});
#[cfg(not(coverage))]
if let Err(error) = handle.await
&& error.is_panic()
{
let mut core = shared.core.lock().await;
core.loop_failures.flush += 1;
log::error!("Flush task panicked: {error}");
}
#[cfg(coverage)]
let _ = handle.await;
}
}
}
async fn push_status_once(shared: &Shared) {
let status = {
let core = shared.core.lock().await;
compute_status(shared, &core)
};
let outcome = shared.transport.send_status(&status).await;
let mut core = shared.core.lock().await;
match outcome {
SendOutcome::Accepted | SendOutcome::PermanentDrop { .. } => {
core.status_consecutive_failures = 0;
core.loop_failures.status = 0;
core.last_status_push_ok = Some(true);
}
SendOutcome::Failed { error } => {
core.status_consecutive_failures = core.status_consecutive_failures.saturating_add(1);
core.loop_failures.status = core.loop_failures.status.saturating_add(1);
core.last_status_push_ok = Some(false);
let consecutive = core.status_consecutive_failures;
if consecutive >= STATUS_LOG_ERROR_THRESHOLD {
log::error!("Status push failed ({consecutive} consecutive): {error}");
} else {
log::warn!("Status push failed: {error}");
}
}
}
}
async fn status_loop(shared: Arc<Shared>) {
let interval = Duration::from_secs(shared.config.status_interval);
loop {
tokio::select! {
() = tokio::time::sleep(interval) => {},
() = shared.shutdown.notified() => break,
}
#[cfg(not(coverage))]
if !shared.running.load(Ordering::Acquire) {
break;
}
#[cfg(coverage)]
let _ = shared.running.load(Ordering::Acquire);
push_status_once(&shared).await;
}
}
async fn fetch_dynamic_rules_once(shared: &Shared) -> Option<DynamicRules> {
let now = Instant::now();
let cached = {
let core = shared.core.lock().await;
let ttl = core.cached_rules.as_ref().map_or(0, |rules| rules.ttl);
let fresh = core
.rules_last_update
.is_some_and(|last| now.duration_since(last) < Duration::from_secs(ttl));
if fresh {
core.cached_rules.clone()
} else {
None
}
};
if let Some(rules) = cached {
return Some(rules);
}
let fetched = shared.transport.fetch_rules().await;
let mut core = shared.core.lock().await;
fetched.inspect(|rules| {
core.cached_rules = Some(rules.clone());
core.rules_last_update = Some(Instant::now());
core.rules_fetched += 1;
log::debug!("Dynamic rules updated");
})
}
async fn rules_loop(shared: Arc<Shared>) {
let interval = Duration::from_secs(shared.config.dynamic_rule_interval);
loop {
tokio::select! {
() = tokio::time::sleep(interval) => {},
() = shared.shutdown.notified() => break,
}
#[cfg(not(coverage))]
if !shared.running.load(Ordering::Acquire) {
break;
}
#[cfg(coverage)]
let _ = shared.running.load(Ordering::Acquire);
let outcome = fetch_dynamic_rules_once(&shared).await;
let mut core = shared.core.lock().await;
if outcome.is_some() {
core.loop_failures.rules = 0;
} else {
core.loop_failures.rules = core.loop_failures.rules.saturating_add(1);
let consecutive = core.loop_failures.rules;
let message = format!(
"rules loop failed {consecutive} consecutive time(s); \
keeping the last good dynamic rules cached"
);
if consecutive >= STATUS_LOG_ERROR_THRESHOLD {
log::error!("{message}");
} else {
log::warn!("{message}");
}
}
}
}
fn unix_now() -> f64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0.0, |delta| delta.as_secs_f64())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::models::MetricType;
use crate::persistence::InMemoryRedisStore;
fn agent_for_test() -> GuardAgent {
let mut config = AgentConfig::new("test-api-key-1234");
config.endpoint = "http://127.0.0.1:9".to_owned();
config.timeout = 1;
config.retry_attempts = 0;
config.backoff_factor = 0.001;
config.flush_interval = 3_600;
config.status_interval = 3_600;
config.dynamic_rule_interval = 3_600;
config.buffer_size = 10;
GuardAgent::new(config).expect("valid config")
}
#[tokio::test]
async fn debug_and_install_id_expose_the_configuration_shape() {
let mut config = AgentConfig::new("test-api-key-1234");
config.endpoint = "http://127.0.0.1:9".to_owned();
config.timeout = 1;
config.retry_attempts = 0;
config.backoff_factor = 0.001;
config.flush_interval = 3_600;
config.status_interval = 3_600;
config.dynamic_rule_interval = 3_600;
config.buffer_size = 10;
config.install_id = Some("install-fixed".to_owned());
let agent = GuardAgent::new(config).expect("valid config");
let printed = format!("{agent:?}");
assert!(printed.contains("GuardAgent"), "{printed}");
assert!(printed.contains("install-fixed"), "{printed}");
assert!(printed.contains("running"), "{printed}");
assert_eq!(agent.install_id(), "install-fixed");
}
#[tokio::test]
async fn start_is_idempotent_while_running() {
let agent = agent_for_test();
agent.start().await;
agent.start().await;
let stats = agent.get_stats().await;
assert!(stats.running);
agent.stop().await;
let stats = agent.get_stats().await;
assert!(!stats.running);
}
#[tokio::test]
async fn start_skips_spawning_when_stale_loop_handles_remain() {
let agent = agent_for_test();
agent.start().await;
agent.shared.running.store(false, Ordering::Release);
let parked = tokio::spawn(std::future::pending::<()>());
{
let mut loops = agent.loops.lock().expect("loops mutex");
*loops = Some(LoopHandles {
flush: tokio::spawn(std::future::pending::<()>()),
status: tokio::spawn(std::future::pending::<()>()),
rules: parked,
});
}
agent.start().await;
agent.stop().await;
assert!(!agent.get_stats().await.running);
}
#[tokio::test]
async fn stop_flushes_through_the_spawned_loop_handles() {
let agent = agent_for_test();
agent.start().await;
agent.send_event(SecurityEvent::new("rate_limited")).await;
agent.stop().await;
assert!(!agent.get_stats().await.running);
}
#[tokio::test]
async fn disabled_kinds_are_ingest_noops() {
let mut config = AgentConfig::new("test-api-key-1234");
config.endpoint = "http://127.0.0.1:9".to_owned();
config.enable_events = false;
config.enable_metrics = false;
let agent = GuardAgent::new(config).unwrap();
agent
.try_send_event(SecurityEvent::new("rate_limited"))
.await
.expect("disabled events are accepted and dropped");
agent
.try_send_metric(SecurityMetric::new(MetricType::RequestCount, 1.0))
.await
.expect("disabled metrics are accepted and dropped");
agent.send_event(SecurityEvent::new("rate_limited")).await;
agent
.send_metric(SecurityMetric::new(MetricType::RequestCount, 1.0))
.await;
let stats = agent.get_stats().await;
assert_eq!(stats.events_buffered, 0);
assert_eq!(stats.metrics_buffered, 0);
}
#[tokio::test]
async fn raise_policy_surfaces_buffer_full_and_the_error_hook_runs() {
crate::test_support::install_trace_logger();
let mut config = AgentConfig::new("test-api-key-1234");
config.endpoint = "http://127.0.0.1:9".to_owned();
config.buffer_size = 1;
config.buffer_overflow_policy = BufferOverflowPolicy::Raise;
let hook_calls = Arc::new(std::sync::Mutex::new(Vec::new()));
let sink = Arc::clone(&hook_calls);
config.on_error = Some(Arc::new(move |stage, _error| {
sink.lock().expect("hook calls").push(stage);
}));
let agent = GuardAgent::new(config).unwrap();
agent.send_event(SecurityEvent::new("rate_limited")).await;
agent.send_event(SecurityEvent::new("rate_limited")).await;
{
let calls = hook_calls.lock().expect("hook calls");
assert!(calls.contains(&ErrorStage::FlushEvents), "{calls:?}");
}
let outcome = agent
.try_send_event(SecurityEvent::new("rate_limited"))
.await;
assert!(matches!(outcome, Err(GuardAgentError::BufferFull { .. })));
agent
.send_metric(SecurityMetric::new(MetricType::RequestCount, 1.0))
.await;
agent
.send_metric(SecurityMetric::new(MetricType::RequestCount, 2.0))
.await;
agent
.send_metric(SecurityMetric::new(MetricType::RequestCount, 3.0))
.await;
{
let calls = hook_calls.lock().expect("hook calls");
assert!(calls.contains(&ErrorStage::FlushMetrics), "{calls:?}");
}
let outcome = agent
.try_send_metric(SecurityMetric::new(MetricType::RequestCount, 4.0))
.await;
assert!(matches!(outcome, Err(GuardAgentError::BufferFull { .. })));
}
#[tokio::test]
async fn a_panicking_error_hook_is_absorbed() {
crate::test_support::install_trace_logger();
let mut config = AgentConfig::new("test-api-key-1234");
config.endpoint = "http://127.0.0.1:9".to_owned();
config.buffer_size = 1;
config.buffer_overflow_policy = BufferOverflowPolicy::Raise;
config.on_error = Some(Arc::new(|_stage, _error| {
panic!("hook exploded");
}));
let agent = GuardAgent::new(config).unwrap();
agent.send_event(SecurityEvent::new("x")).await;
agent.send_event(SecurityEvent::new("y")).await;
let stats = agent.get_stats().await;
assert_eq!(stats.events_buffered, 1, "the first event still buffered");
}
#[tokio::test]
async fn drop_policy_evicts_the_oldest_items_for_both_kinds() {
crate::test_support::install_trace_logger();
let mut config = AgentConfig::new("test-api-key-1234");
config.endpoint = "http://127.0.0.1:9".to_owned();
config.buffer_size = 1;
config.buffer_overflow_policy = BufferOverflowPolicy::Drop;
let agent = GuardAgent::new(config).unwrap();
for index in 0..3 {
agent
.send_event(
SecurityEvent::new("rate_limited").with_ip_address(format!("10.0.0.{index}")),
)
.await;
}
for index in 0..3 {
agent
.send_metric(SecurityMetric::new(
MetricType::RequestCount,
f64::from(u32::try_from(index).unwrap_or(0)),
))
.await;
}
let stats = agent.get_stats().await;
assert_eq!(stats.events_buffered, 1, "capacity stays at one");
assert_eq!(stats.metrics_buffered, 1);
assert_eq!(stats.events_dropped, 2);
assert_eq!(stats.metrics_dropped, 2);
}
#[tokio::test]
async fn block_policy_waits_for_space_and_unblocks_on_flush() {
let mut config = AgentConfig::new("test-api-key-1234");
config.endpoint = "http://127.0.0.1:9".to_owned();
config.buffer_size = 1;
config.buffer_overflow_policy = BufferOverflowPolicy::Block;
let agent = GuardAgent::new(config).unwrap();
agent.send_event(SecurityEvent::new("first")).await;
let waiter_agent = agent.clone();
let waiter = tokio::spawn(async move {
waiter_agent.send_event(SecurityEvent::new("second")).await;
});
tokio::time::sleep(Duration::from_millis(120)).await;
agent.clear_buffer().await;
tokio::time::timeout(Duration::from_secs(5), waiter)
.await
.expect("the blocked waiter completed")
.expect("task join");
let stats = agent.get_stats().await;
assert_eq!(stats.events_buffered, 1, "the waiter's item landed");
agent
.send_metric(SecurityMetric::new(MetricType::RequestCount, 1.0))
.await;
let waiter_agent = agent.clone();
let metric_waiter = tokio::spawn(async move {
waiter_agent
.send_metric(SecurityMetric::new(MetricType::RequestCount, 2.0))
.await;
});
tokio::time::sleep(Duration::from_millis(120)).await;
agent.clear_buffer().await;
tokio::time::timeout(Duration::from_secs(5), metric_waiter)
.await
.expect("the blocked metric waiter completed")
.expect("task join");
}
#[tokio::test]
async fn clear_buffer_without_a_handler_returns_quietly() {
let agent = agent_for_test();
agent.send_event(SecurityEvent::new("x")).await;
agent.clear_buffer().await;
let stats = agent.get_stats().await;
assert_eq!(stats.events_buffered, 0);
}
#[tokio::test]
async fn clear_buffer_reports_and_succeeds_for_both_handler_outcomes() {
let agent = agent_for_test();
agent
.attach_redis_handler(Arc::new(FakeStore::fail_clear()) as Arc<dyn RedisHandler>)
.await;
agent.clear_buffer().await;
let agent_ok = agent_for_test();
agent_ok
.attach_redis_handler(Arc::new(FakeStore::default()) as Arc<dyn RedisHandler>)
.await;
agent_ok.clear_buffer().await;
}
#[derive(Default)]
#[allow(clippy::struct_excessive_bools)] struct FakeStore {
fail_set: bool,
fail_get: bool,
fail_delete: bool,
fail_list: bool,
fail_clear: bool,
}
impl FakeStore {
fn fail_set() -> Self {
Self {
fail_set: true,
..Self::default()
}
}
fn fail_delete() -> Self {
Self {
fail_delete: true,
..Self::default()
}
}
fn fail_list() -> Self {
Self {
fail_list: true,
..Self::default()
}
}
fn fail_clear() -> Self {
Self {
fail_clear: true,
..Self::default()
}
}
}
#[async_trait::async_trait]
impl RedisHandler for FakeStore {
async fn set_key(
&self,
_namespace: &str,
_key: &str,
_value: &str,
_ttl_seconds: u64,
) -> Result<(), GuardAgentError> {
if self.fail_set {
return Err(GuardAgentError::Transport("set failed".to_owned()));
}
Ok(())
}
async fn get_key(
&self,
_namespace: &str,
key: &str,
) -> Result<Option<String>, GuardAgentError> {
if key == "bad" {
return Err(GuardAgentError::Transport("get failed".to_owned()));
}
if self.fail_get {
return Err(GuardAgentError::Transport("get failed".to_owned()));
}
Ok(None)
}
async fn delete_keys(
&self,
_namespace: &str,
_keys: &[String],
) -> Result<(), GuardAgentError> {
if self.fail_delete {
return Err(GuardAgentError::Transport("delete failed".to_owned()));
}
Ok(())
}
async fn list_keys(&self, _namespace: &str) -> Result<Vec<String>, GuardAgentError> {
if self.fail_list {
return Err(GuardAgentError::Transport("list failed".to_owned()));
}
Ok(vec!["gone".to_owned(), "bad".to_owned()])
}
async fn clear_namespace(&self, _namespace: &str) -> Result<(), GuardAgentError> {
if self.fail_clear {
return Err(GuardAgentError::Transport("clear failed".to_owned()));
}
Ok(())
}
}
#[tokio::test]
async fn degraded_reload_paths_are_swallowed_and_logged() {
let agent = agent_for_test();
agent
.attach_redis_handler(Arc::new(FakeStore::fail_list()) as Arc<dyn RedisHandler>)
.await;
agent.start().await;
agent.stop().await;
let stats = agent.get_stats().await;
assert_eq!(stats.events_buffered, 0);
assert_eq!(stats.metrics_buffered, 0);
}
#[tokio::test]
async fn reload_skips_unreadable_corrupt_and_missing_records() {
let store = Arc::new(InMemoryRedisStore::new());
store
.set_key(NAMESPACE_EVENTS, "bad", "not-json", PERSIST_TTL_SECONDS)
.await
.unwrap();
store
.set_key(NAMESPACE_METRICS, "bad", "not-json", PERSIST_TTL_SECONDS)
.await
.unwrap();
store
.set_key(NAMESPACE_EVENTS, "gone", "{\"a\":1}", 0)
.await
.unwrap();
store
.set_key(NAMESPACE_METRICS, "gone", "{\"a\":1}", 0)
.await
.unwrap();
let agent = agent_for_test();
agent
.attach_redis_handler(Arc::clone(&store) as Arc<dyn RedisHandler>)
.await;
agent.start().await;
let stats = agent.get_stats().await;
assert_eq!(
stats.events_buffered, 0,
"corrupt and expired records never load"
);
assert_eq!(stats.metrics_buffered, 0);
agent.stop().await;
}
#[tokio::test]
async fn reload_swallows_per_key_read_failures_and_missing_records() {
crate::test_support::install_trace_logger();
let agent = agent_for_test();
agent
.attach_redis_handler(Arc::new(FakeStore::default()) as Arc<dyn RedisHandler>)
.await;
agent.start().await;
let stats = agent.get_stats().await;
assert_eq!(stats.events_buffered, 0, "listed keys never load");
assert_eq!(stats.metrics_buffered, 0);
agent.stop().await;
}
#[tokio::test]
async fn eviction_deletes_confirmed_keys_and_swallows_delete_failures() {
crate::test_support::install_trace_logger();
let config_for_eviction = || {
let mut config = AgentConfig::new("test-api-key-1234");
config.endpoint = "http://127.0.0.1:9".to_owned();
config.timeout = 1;
config.retry_attempts = 0;
config.backoff_factor = 0.001;
config.flush_interval = 3_600;
config.status_interval = 3_600;
config.dynamic_rule_interval = 3_600;
config.buffer_size = 1;
config.buffer_overflow_policy = BufferOverflowPolicy::Drop;
config
};
let dropping = GuardAgent::new(config_for_eviction()).unwrap();
dropping
.attach_redis_handler(Arc::new(FakeStore::default()) as Arc<dyn RedisHandler>)
.await;
dropping.send_event(SecurityEvent::new("first")).await;
dropping.send_event(SecurityEvent::new("second")).await;
let stats = dropping.get_stats().await;
assert_eq!(stats.events_dropped, 1, "the oldest record was evicted");
let failing = GuardAgent::new(config_for_eviction()).unwrap();
failing
.attach_redis_handler(Arc::new(FakeStore::fail_delete()) as Arc<dyn RedisHandler>)
.await;
failing.send_event(SecurityEvent::new("first")).await;
failing.send_event(SecurityEvent::new("second")).await;
assert_eq!(failing.get_stats().await.events_dropped, 1);
}
#[tokio::test]
async fn watermark_flushes_are_gated_by_the_permit_cap() {
let mut config = AgentConfig::new("test-api-key-1234");
config.endpoint = "http://127.0.0.1:9".to_owned();
config.timeout = 1;
config.retry_attempts = 0;
config.backoff_factor = 0.001;
config.flush_interval = 3_600;
config.status_interval = 3_600;
config.dynamic_rule_interval = 3_600;
config.buffer_size = 2;
config.high_watermark_ratio = 0.5;
let agent = GuardAgent::new(config).unwrap();
agent.start().await;
let held = agent
.shared
.flush_permits
.clone()
.acquire_owned()
.await
.expect("permit free");
agent.send_event(SecurityEvent::new("x")).await;
agent.send_event(SecurityEvent::new("y")).await;
tokio::time::sleep(Duration::from_millis(200)).await;
let stats = agent.get_stats().await;
assert_eq!(stats.events_buffered, 2, "the gated flush was starved");
drop(held);
agent.send_event(SecurityEvent::new("z")).await;
tokio::time::sleep(Duration::from_millis(200)).await;
let stats = agent.get_stats().await;
assert!(
stats.events_failed >= 1,
"the gated flush ran against the dead endpoint: {stats:?}"
);
assert_eq!(stats.events_buffered, 2, "capacity holds after requeue");
assert_eq!(stats.events_dropped, 1, "the requeue evicted the newest");
agent.stop().await;
}
#[tokio::test(start_paused = true)]
async fn flush_loop_ticks_with_an_exhausted_permit_cap_skip_the_flush() {
let mut config = AgentConfig::new("test-api-key-1234");
config.endpoint = "http://127.0.0.1:9".to_owned();
config.timeout = 600;
config.retry_attempts = 0;
config.backoff_factor = 0.001;
config.flush_interval = 60;
config.status_interval = 3_600;
config.dynamic_rule_interval = 3_600;
config.buffer_size = 10;
config.max_concurrent_flushes = 1;
let agent = GuardAgent::new(config).unwrap();
agent.start().await;
tokio::task::yield_now().await;
let held = agent
.shared
.flush_permits
.clone()
.acquire_owned()
.await
.expect("permit free");
agent.send_event(SecurityEvent::new("x")).await;
tokio::time::advance(Duration::from_secs(61)).await;
for _ in 0..50 {
tokio::task::yield_now().await;
}
let stats = agent.get_stats().await;
assert_eq!(
stats.events_buffered, 1,
"the starved tick kept the event buffered"
);
drop(held);
tokio::time::advance(Duration::from_secs(61)).await;
for _ in 0..50 {
tokio::task::yield_now().await;
}
let stats = agent.get_stats().await;
assert_eq!(
stats.events_buffered, 1,
"the failed flush requeued the event"
);
assert_eq!(stats.events_sent, 0, "the dead endpoint never accepted");
agent.stop().await;
}
#[tokio::test]
async fn stop_without_start_is_a_quiet_no_op() {
let agent = agent_for_test();
agent.stop().await;
assert!(!agent.get_stats().await.running);
}
#[tokio::test]
async fn confirm_delete_failures_are_swallowed() {
let agent = agent_for_test();
agent
.attach_redis_handler(Arc::new(FakeStore::fail_delete()) as Arc<dyn RedisHandler>)
.await;
confirm_keys(
&agent.shared,
NAMESPACE_EVENTS,
vec![Some("evt_1".to_owned())],
)
.await;
}
#[tokio::test]
async fn reload_loads_good_records_and_evicts_at_capacity() {
let store = Arc::new(InMemoryRedisStore::new());
let mut config = AgentConfig::new("test-api-key-1234");
config.endpoint = "http://127.0.0.1:9".to_owned();
config.timeout = 1;
config.buffer_size = 1;
let agent = GuardAgent::new(config).unwrap();
agent
.attach_redis_handler(Arc::clone(&store) as Arc<dyn RedisHandler>)
.await;
store
.set_key(
NAMESPACE_EVENTS,
"evt_1",
&serde_json::to_string(&SecurityEvent::new("rate_limited")).unwrap(),
PERSIST_TTL_SECONDS,
)
.await
.unwrap();
store
.set_key(
NAMESPACE_METRICS,
"met_1",
&serde_json::to_string(&SecurityMetric::new(MetricType::RequestCount, 1.0))
.unwrap(),
PERSIST_TTL_SECONDS,
)
.await
.unwrap();
agent.start().await;
let stats = agent.get_stats().await;
assert_eq!(stats.events_buffered, 1, "the good event reloads");
assert_eq!(stats.metrics_buffered, 1, "the good metric reloads");
agent.stop().await;
store
.set_key(
NAMESPACE_EVENTS,
"evt_2",
&serde_json::to_string(&SecurityEvent::new("rate_limited")).unwrap(),
PERSIST_TTL_SECONDS,
)
.await
.unwrap();
agent.start().await;
agent.stop().await;
let stats = agent.get_stats().await;
assert_eq!(stats.events_buffered, 1, "capacity holds at one");
assert_eq!(
stats.metrics_buffered, 1,
"the metric reloads again unchanged"
);
assert!(
stats.events_dropped >= 1,
"reloading over a full buffer evicts: {} dropped",
stats.events_dropped
);
}
#[tokio::test]
async fn get_status_flags_a_nearly_full_buffer() {
let mut config = AgentConfig::new("test-api-key-1234");
config.endpoint = "http://127.0.0.1:9".to_owned();
config.buffer_size = 10;
config.high_watermark_ratio = 0.5;
let agent = GuardAgent::new(config).unwrap();
for index in 0..9 {
agent
.send_event(
SecurityEvent::new("rate_limited").with_ip_address(format!("10.0.0.{index}")),
)
.await;
}
let status = agent.get_status().await;
assert_eq!(status.status, AgentHealth::Degraded);
assert!(
status
.errors
.iter()
.any(|error| error.contains("Buffer nearly full")),
"{:?}",
status.errors
);
}
#[tokio::test]
async fn health_check_reports_the_stopped_agent_and_the_open_breaker() {
let agent = agent_for_test();
assert!(
!agent.health_check().await,
"a stopped agent is never healthy"
);
agent.shared.transport.test_open_breaker();
agent.start().await;
assert_eq!(
agent.get_stats().await.circuit_breaker_state,
crate::circuit_breaker::CircuitBreakerState::Open
);
assert!(
!agent.health_check().await,
"an open breaker fails the health check"
);
agent.stop().await;
agent.shared.transport.test_reset_breaker();
agent.start().await;
assert!(agent.health_check().await, "healthy again");
agent.stop().await;
}
#[tokio::test]
async fn health_check_fails_when_the_buffer_is_nearly_full() {
let mut config = AgentConfig::new("test-api-key-1234");
config.endpoint = "http://127.0.0.1:9".to_owned();
config.timeout = 1;
config.retry_attempts = 0;
config.backoff_factor = 0.001;
config.flush_interval = 3_600;
config.status_interval = 3_600;
config.dynamic_rule_interval = 3_600;
config.buffer_size = 10;
let agent = GuardAgent::new(config).unwrap();
for index in 0..10 {
agent
.send_event(
SecurityEvent::new("rate_limited").with_ip_address(format!("10.0.0.{index}")),
)
.await;
}
agent.start().await;
assert!(!agent.health_check().await, "a 100 percent buffer fails");
agent.stop().await;
}
#[tokio::test]
async fn push_status_counts_consecutive_failures() {
let agent = agent_for_test();
for _ in 0..5 {
agent.push_status().await;
}
let stats = agent.get_stats().await;
assert_eq!(stats.status_consecutive_failures, 5);
assert_eq!(stats.last_status_push_ok, Some(false));
agent.push_status().await;
let stats = agent.get_stats().await;
assert_eq!(stats.status_consecutive_failures, 6);
}
#[tokio::test]
async fn flush_gates_skip_the_transport_until_the_backoff_elapses() {
let agent = agent_for_test();
agent.send_event(SecurityEvent::new("x")).await;
agent
.send_metric(SecurityMetric::new(MetricType::RequestCount, 1.0))
.await;
agent.flush_buffer().await;
let stats = agent.get_stats().await;
assert_eq!(stats.events_failed, 1);
assert_eq!(stats.metrics_failed, 1);
agent.flush_buffer().await;
let stats = agent.get_stats().await;
assert_eq!(stats.events_failed, 1, "the event gate held");
assert_eq!(stats.metrics_failed, 1, "the metric gate held");
assert!(stats.events_buffered == 1 && stats.metrics_buffered == 1);
}
#[tokio::test]
async fn permanent_rejections_confirm_the_records_instead_of_requeuing() {
let mut config = AgentConfig::new("test-api-key-1234");
config.endpoint = "http://127.0.0.1:9".to_owned();
config.timeout = 1;
config.buffer_size = 10;
let agent = GuardAgent::new(config).unwrap();
agent
.attach_redis_handler(Arc::new(InMemoryRedisStore::new()) as Arc<dyn RedisHandler>)
.await;
agent.send_event(SecurityEvent::new("x")).await;
agent.flush_buffer().await;
let stats = agent.get_stats().await;
assert_eq!(stats.events_buffered, 1, "failed sends requeue");
assert_eq!(stats.redis_persist_failures, 0);
}
#[tokio::test]
async fn flush_if_needed_respects_the_interval_and_watermark_gates() {
let agent = agent_for_test();
agent.send_event(SecurityEvent::new("x")).await;
agent.flush_buffer().await;
agent.send_event(SecurityEvent::new("y")).await;
let permit = agent
.shared
.flush_permits
.clone()
.try_acquire_owned()
.expect("permit free");
flush_if_needed(&agent.shared, permit).await;
let stats = agent.get_stats().await;
assert_eq!(stats.events_buffered, 2, "the gate held");
}
#[test]
fn requeue_evicts_the_newest_tail_when_at_capacity() {
let mut deque: VecDeque<BufferedItem<SecurityEvent>> = VecDeque::new();
deque.push_back(BufferedItem {
item: SecurityEvent::new("old"),
redis_key: Some("k_old".to_owned()),
});
deque.push_back(BufferedItem {
item: SecurityEvent::new("newest"),
redis_key: Some("k_new".to_owned()),
});
let mut dropped = 0;
let evicted = requeue(
&mut deque,
2,
vec![SecurityEvent::new("r1"), SecurityEvent::new("r2")],
vec![Some("k_r1".to_owned()), Some("k_r2".to_owned())],
&mut dropped,
);
assert_eq!(dropped, 2, "both tail items were evicted");
assert_eq!(
evicted,
vec![Some("k_new".to_owned()), Some("k_old".to_owned())],
"evictions return newest-first for confirmation"
);
assert_eq!(deque.len(), 2, "capacity holds");
assert_eq!(
deque[0].item.event_type, "r1",
"requeueing in reverse restores the original order"
);
assert_eq!(deque[1].item.event_type, "r2");
}
#[tokio::test]
async fn persist_failures_count_and_land_the_item_in_memory_only() {
let agent = agent_for_test();
agent
.attach_redis_handler(Arc::new(FakeStore::fail_set()) as Arc<dyn RedisHandler>)
.await;
agent.send_event(SecurityEvent::new("x")).await;
let stats = agent.get_stats().await;
assert_eq!(stats.redis_persist_failures, 1);
assert_eq!(stats.events_buffered, 1, "fail-open buffering");
assert!(stats.durability_degraded);
}
#[tokio::test]
async fn start_reuses_orphaned_loop_handles_from_the_stop_window() {
let agent = agent_for_test();
agent.start().await;
agent
.shared
.running
.store(false, std::sync::atomic::Ordering::Release);
agent.start().await;
assert!(agent.get_stats().await.running, "the restart is running");
agent.stop().await;
}
#[cfg(feature = "persistence")]
#[tokio::test]
async fn start_degrades_when_the_redis_url_is_invalid() {
let mut config = AgentConfig::new("test-api-key-1234");
config.endpoint = "http://127.0.0.1:9".to_owned();
config.timeout = 1;
config.retry_attempts = 0;
config.flush_interval = 3_600;
config.status_interval = 3_600;
config.dynamic_rule_interval = 3_600;
config.redis = Some(crate::config::RedisConfig {
url: "not-a-redis-url".to_owned(),
key_prefix: "guard-test".to_owned(),
command_timeout_ms: 1_000,
});
let agent = GuardAgent::new(config).unwrap();
agent.start().await;
assert!(agent.get_stats().await.running);
agent.stop().await;
}
#[cfg(feature = "persistence")]
#[tokio::test]
#[ignore = "requires Redis on 127.0.0.1:6379 (the CI coverage job provides one)"]
async fn start_connects_a_reachable_redis_backend() {
let mut config = AgentConfig::new("test-api-key-1234");
config.endpoint = "http://127.0.0.1:9".to_owned();
config.timeout = 1;
config.retry_attempts = 0;
config.flush_interval = 3_600;
config.status_interval = 3_600;
config.dynamic_rule_interval = 3_600;
config.redis = Some(crate::config::RedisConfig {
url: "redis://127.0.0.1:6379".to_owned(),
key_prefix: "guard-test-live".to_owned(),
command_timeout_ms: 1_000,
});
let agent = GuardAgent::new(config).unwrap();
agent.start().await;
let stats = agent.get_stats().await;
assert!(stats.running);
assert!(
!stats.durability_degraded,
"the reachable backend keeps durability intact"
);
agent.stop().await;
}
#[tokio::test]
async fn compute_status_reports_nearly_full_buffer_and_high_failure_rate() {
let agent = agent_for_test();
{
let mut core = agent.shared.core.lock().await;
while core.events.len() < 9 {
core.events.push_back(BufferedItem {
item: SecurityEvent::new("buffered"),
redis_key: None,
});
}
core.events_failed = 5;
}
let core = agent.shared.core.lock().await;
let status = compute_status(&agent.shared, &core);
assert_eq!(status.status, AgentHealth::Degraded);
assert!(
status
.errors
.iter()
.any(|error| error.contains("Buffer nearly full")),
"{:?}",
status.errors
);
assert!(
status
.errors
.iter()
.any(|error| error.contains("High failure rate")),
"{:?}",
status.errors
);
}
#[tokio::test]
async fn confirm_keys_without_live_keys_returns_quietly() {
let agent = agent_for_test();
confirm_keys(
&agent.shared,
NAMESPACE_EVENTS,
vec![None, Some(String::new())],
)
.await;
let stats = agent.get_stats().await;
assert_eq!(stats.redis_persist_failures, 0);
}
#[tokio::test]
async fn flush_if_needed_gates_on_flush_history_and_flushes_when_open() {
let agent = agent_for_test();
let permit = agent
.shared
.flush_permits
.clone()
.acquire_owned()
.await
.expect("permit free");
flush_if_needed(&agent.shared, permit).await;
assert_eq!(agent.get_stats().await.events_buffered, 0);
{
let mut core = agent.shared.core.lock().await;
core.events.push_back(BufferedItem {
item: SecurityEvent::new("gated"),
redis_key: None,
});
core.last_flush_instant = Some(Instant::now());
}
let permit = agent
.shared
.flush_permits
.clone()
.acquire_owned()
.await
.expect("permit free");
flush_if_needed(&agent.shared, permit).await;
let stats = agent.get_stats().await;
assert_eq!(stats.events_buffered, 1, "the closed gate kept the item");
assert_eq!(stats.events_failed, 0, "no flush attempt was made");
agent.shared.core.lock().await.last_flush_instant = None;
let permit = agent
.shared
.flush_permits
.clone()
.acquire_owned()
.await
.expect("permit free");
flush_if_needed(&agent.shared, permit).await;
let stats = agent.get_stats().await;
assert_eq!(
stats.events_failed, 1,
"the gated flush hit the dead endpoint"
);
assert_eq!(stats.events_buffered, 1, "the failure requeued the item");
}
#[tokio::test]
async fn flush_metrics_reports_the_full_outcome_surface() {
use wiremock::matchers::{method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
crate::test_support::install_trace_logger();
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/v1/metrics"))
.respond_with(|_request: &wiremock::Request| {
ResponseTemplate::new(200).set_body_string("{\"success\": true}")
})
.mount(&server)
.await;
let mut config = AgentConfig::new("test-api-key-1234");
config.endpoint = server.uri();
config.timeout = 5;
config.retry_attempts = 0;
config.compression_enabled = false;
let agent = GuardAgent::new(config).unwrap();
agent
.send_metric(SecurityMetric::new(MetricType::ResponseTime, 1.0))
.await;
flush_metrics(&agent.shared).await;
let stats = agent.get_stats().await;
assert_eq!(stats.metrics_sent, 1, "the healthy flush was accepted");
agent
.send_metric(SecurityMetric::new(MetricType::ResponseTime, 2.0))
.await;
agent.shared.core.lock().await.metrics_retry_after =
Some(Instant::now() + Duration::from_secs(3_600));
flush_metrics(&agent.shared).await;
let stats = agent.get_stats().await;
assert_eq!(stats.metrics_buffered, 1, "the closed gate kept the metric");
let rejecting = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/api/v1/metrics"))
.respond_with(|_request: &wiremock::Request| {
ResponseTemplate::new(413).set_body_string("too large")
})
.mount(&rejecting)
.await;
let mut config = AgentConfig::new("test-api-key-1234");
config.endpoint = rejecting.uri();
config.timeout = 5;
config.retry_attempts = 0;
config.compression_enabled = false;
let agent = GuardAgent::new(config).unwrap();
agent
.send_metric(SecurityMetric::new(MetricType::ResponseTime, 3.0))
.await;
flush_metrics(&agent.shared).await;
let stats = agent.get_stats().await;
assert_eq!(
stats.metrics_buffered, 0,
"the permanent drop emptied the batch"
);
let mut config = AgentConfig::new("test-api-key-1234");
config.endpoint = "http://127.0.0.1:9".to_owned();
config.timeout = 1;
config.retry_attempts = 0;
config.compression_enabled = false;
config.buffer_size = 2;
let agent = GuardAgent::new(config).unwrap();
for value in [1.0, 2.0, 3.0] {
agent
.send_metric(SecurityMetric::new(MetricType::ResponseTime, value))
.await;
}
flush_metrics(&agent.shared).await;
let stats = agent.get_stats().await;
assert!(stats.metrics_failed >= 1, "the flush failed");
assert_eq!(stats.metrics_buffered, 2, "capacity holds after requeue");
assert_eq!(stats.metrics_dropped, 1, "the requeue evicted the newest");
}
#[cfg(not(coverage))]
#[tokio::test]
async fn persist_item_counts_serialization_failures() {
use serde::Serialize;
struct Unserializable;
impl Serialize for Unserializable {
fn serialize<S: serde::Serializer>(&self, _serializer: S) -> Result<S::Ok, S::Error> {
Err(serde::ser::Error::custom("nope"))
}
}
let agent = agent_for_test();
agent
.attach_redis_handler(Arc::new(InMemoryRedisStore::new()) as Arc<dyn RedisHandler>)
.await;
let mut core = agent.shared.core.lock().await;
let outcome = persist_item(&mut core, NAMESPACE_EVENTS, "event", &Unserializable).await;
assert_eq!(outcome, None);
assert_eq!(core.redis_persist_failures, 1);
}
#[tokio::test]
async fn confirm_keys_are_deleted_through_the_handler() {
let store = Arc::new(InMemoryRedisStore::new());
let agent = agent_for_test();
agent
.attach_redis_handler(Arc::clone(&store) as Arc<dyn RedisHandler>)
.await;
confirm_keys(
&agent.shared,
NAMESPACE_EVENTS,
vec![Some("evt_1".to_owned()), Some(String::new()), None],
)
.await;
assert!(
!store.keys(NAMESPACE_EVENTS).contains(&"evt_1".to_owned()),
"the live key was deleted"
);
}
}