use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, HashMap};
use std::sync::{Arc, Mutex, RwLock};
use std::path::Path;
use std::time::SystemTime;
use std::time::{Duration, Instant};
use tracing::{error, info};
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum AlertSeverity {
Critical,
Warning,
Info,
}
impl std::fmt::Display for AlertSeverity {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
AlertSeverity::Critical => write!(f, "critical"),
AlertSeverity::Warning => write!(f, "warning"),
AlertSeverity::Info => write!(f, "info"),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum AlertCondition {
GreaterThan(f64),
LessThan(f64),
EqualTo(f64),
NotEqualTo(f64),
Between(f64, f64),
Outside(f64, f64),
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AlertRule {
pub id: String,
pub name: String,
pub metric_name: String,
pub condition: AlertCondition,
pub threshold: f64,
pub duration: Duration,
pub severity: AlertSeverity,
pub labels: HashMap<String, String>,
pub description: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Alert {
pub rule_id: String,
pub name: String,
pub severity: AlertSeverity,
pub value: f64,
pub threshold: f64,
pub triggered_at: SystemTime,
pub labels: HashMap<String, String>,
pub description: String,
}
pub trait AlertNotifier: Send + Sync {
fn send(&self, alert: &Alert) -> Result<()>;
fn name(&self) -> &str;
}
pub struct HttpWebhookNotifier {
url: String,
headers: HashMap<String, String>,
}
impl HttpWebhookNotifier {
pub fn new(url: String) -> Self {
Self {
url,
headers: HashMap::new(),
}
}
pub fn with_header(mut self, key: String, value: String) -> Self {
self.headers.insert(key, value);
self
}
}
impl AlertNotifier for HttpWebhookNotifier {
fn send(&self, alert: &Alert) -> Result<()> {
let payload: _ = serde_json::to_string(alert).context("Failed to serialize alert")?;
let client: _ = Client::new();
let mut request = client
.post(&self.url)
.body(payload)
.header("Content-Type", "application/json");
for (key, value) in &self.headers {
request = request.header(key, value);
}
let response: _ = request.send().context("Failed to send webhook")?;
if !response.status().is_success() {
return Err(anyhow::anyhow!(
"Webhook returned status: {}",
response.status()
));
}
info!("Alert sent via HTTP webhook to {}", self.url);
Ok(())
}
fn name(&self) -> &str {
"http_webhook"
}
}
pub struct ConsoleNotifier;
impl AlertNotifier for ConsoleNotifier {
fn send(&self, alert: &Alert) -> Result<()> {
println!(
"ALERT [{}] {}: {} (value: {}, threshold: {})",
alert.severity, alert.name, alert.description, alert.value, alert.threshold
);
Ok(())
}
fn name(&self) -> &str {
"console"
}
}
pub struct AlertingSystem {
rules: Arc<RwLock<Vec<AlertRule>>>,
active_alerts: Arc<RwLock<HashMap<String, Alert>>>,
notifiers: Arc<RwLock<Vec<Box<dyn AlertNotifier>>>>,
check_interval: Duration,
last_check: Arc<RwLock<Instant>>,
}
impl AlertingSystem {
pub fn new() -> Self {
let mut notifiers = Vec::new();
notifiers.push(Box::new(ConsoleNotifier) as Box<dyn AlertNotifier>);
Self {
rules: Arc::new(Mutex::new(Vec::new())),
active_alerts: Arc::new(Mutex::new(HashMap::new())),
notifiers: Arc::new(Mutex::new(notifiers)),
check_interval: Duration::from_secs(30),
last_check: Arc::new(Mutex::new(Instant::now())),
}
}
pub fn with_check_interval(interval: Duration) -> Self {
let mut system = Self::new();
system.check_interval = interval;
system
}
pub async fn add_rule(&self, rule: AlertRule) -> Result<()> {
let rule_name: _ = rule.name.clone();
let mut rules = self.rules.write().await;
rules.push(rule);
info!("Added alert rule: {}", rule_name);
Ok(())
}
pub async fn remove_rule(&self, rule_id: &str) -> Result<()> {
let mut rules = self.rules.write().await;
rules.retain(|r| r.id != rule_id);
info!("Removed alert rule: {}", rule_id);
Ok(())
}
pub async fn load_rules_from_file(&self, file_path: &str) -> Result<()> {
let content: _ =
fs::read_to_string(Path::new(file_path)).context("Failed to read alert rules file")?;
let rules: Vec<AlertRule> =
serde_yaml::from_str(&content).context("Failed to parse alert rules")?;
let mut rules_guard = self.rules.write().await;
rules_guard.clear();
rules_guard.extend(rules);
info!(
"Loaded {} alert rules from {}",
rules_guard.len(),
file_path
);
Ok(())
}
pub async fn save_rules_to_file(&self, file_path: &str) -> Result<()> {
let rules: _ = self.rules.read().await;
let content: _ =
serde_yaml::to_string(&*rules).context("Failed to serialize alert rules")?;
fs::write(Path::new(file_path), content).context("Failed to write alert rules file")?;
info!("Saved {} alert rules to {}", rules.len(), file_path);
Ok(())
}
pub async fn add_notifier(&self, notifier: Box<dyn AlertNotifier>) {
let mut notifiers = self.notifiers.write().await;
notifiers.push(notifier);
info!("Added alert notifier");
}
pub async fn check_alerts(&self, metrics: &[MetricFamily]) -> Result<Vec<Alert>> {
let rules: _ = self.rules.read().await;
let mut triggered_alerts = Vec::new();
for rule in rules.iter() {
if let Some(metric) = self.find_metric(metrics, &rule.metric_name) {
if let Some(alert) = self.check_rule(rule, metric).await? {
triggered_alerts.push(alert);
}
}
}
Ok(triggered_alerts)
}
pub async fn should_check(&self) -> bool {
let last_check: _ = *self.last_check.read().await;
last_check.elapsed() >= self.check_interval
}
pub async fn run_check_loop(&self, metrics_provider: impl Fn() -> Vec<MetricFamily>) {
let mut interval = tokio::time::interval(self.check_interval);
loop {
interval.tick().await;
if !self.should_check().await {
continue;
}
let metrics: _ = metrics_provider();
match self.check_alerts(&metrics).await {
Ok(alerts) => {
for alert in alerts {
self.trigger_alert(&alert).await;
}
}
Err(e) => {
error!("Failed to check alerts: {}", e);
}
}
*self.last_check.write().await = Instant::now();
}
}
async fn trigger_alert(&self, alert: &Alert) {
let mut active_alerts = self.active_alerts.write().await;
if active_alerts.contains_key(&alert.rule_id) {
active_alerts.insert(alert.rule_id.clone(), alert.clone());
info!("Updated alert: {}", alert.name);
} else {
active_alerts.insert(alert.rule_id.clone(), alert.clone());
info!("Triggered new alert: {}", alert.name);
let notifiers: _ = self.notifiers.read().await;
for notifier in notifiers.iter() {
if let Err(e) = notifier.send(alert) {
error!("Failed to send alert via {}: {}", notifier.name(), e);
}
}
}
}
pub async fn get_active_alerts(&self) -> HashMap<String, Alert> {
self.active_alerts.read().await.clone()
}
pub async fn resolve_alert(&self, rule_id: &str) {
let mut active_alerts = self.active_alerts.write().await;
if let Some(alert) = active_alerts.remove(rule_id) {
info!("Resolved alert: {}", alert.name);
}
}
pub async fn shutdown(&self) -> Result<()> {
info!("Shutting down alerting system...");
Ok(())
}
fn find_metric<'a>(
&self,
metrics: &'a [MetricFamily],
metric_name: &str,
) -> Option<&'a MetricFamily> {
metrics.iter().find(|m| m.get_name() == metric_name)
}
async fn check_rule(&self, rule: &AlertRule, metric: &MetricFamily) -> Result<Option<Alert>> {
if metric.get_metric().is_empty() {
return Ok(None);
}
let metric_point: _ = &metric.get_metric()[0];
let value: _ = if metric_point.has_gauge() {
0.0
} else if metric_point.has_counter() {
0.0
} else {
return Ok(None);
};
let condition_met: _ = match &rule.condition {
AlertCondition::GreaterThan(threshold) => value > *threshold,
AlertCondition::LessThan(threshold) => value < *threshold,
AlertCondition::EqualTo(threshold) => (value - *threshold).abs() < f64::EPSILON,
AlertCondition::NotEqualTo(threshold) => (value - *threshold).abs() >= f64::EPSILON,
AlertCondition::Between(min, max) => value >= *min && value <= *max,
AlertCondition::Outside(min, max) => value < *min || value > *max,
};
if condition_met {
let alert: _ = Alert {
rule_id: rule.id.clone(),
name: rule.name.clone(),
severity: rule.severity.clone(),
value,
threshold: rule.threshold,
triggered_at: SystemTime::now(),
labels: rule.labels.clone(),
description: rule.description.clone(),
};
Ok(Some(alert))
} else {
Ok(None)
}
}
}
impl Default for AlertingSystem {
fn default() -> Self {
Self::new()
}
}
pub struct BuiltInAlertRules;
impl BuiltInAlertRules {
pub fn get_default_rules() -> Vec<AlertRule> {
vec![
AlertRule {
id: "high_error_rate".to_string(),
name: "High Error Rate".to_string(),
metric_name: "amberjs_script_error_rate_percent".to_string(),
condition: AlertCondition::GreaterThan(10.0),
threshold: 10.0,
duration: Duration::from_secs(60),
severity: AlertSeverity::Critical,
labels: HashMap::new(),
description: "Script error rate is above 10%".to_string(),
},
AlertRule {
id: "high_memory_usage".to_string(),
name: "High Memory Usage".to_string(),
metric_name: "amberjs_memory_usage_bytes".to_string(),
condition: AlertCondition::GreaterThan(1073741824.0), threshold: 1073741824.0,
duration: Duration::from_secs(120),
severity: AlertSeverity::Warning,
labels: HashMap::new(),
description: "Memory usage is above 1GB".to_string(),
},
AlertRule {
id: "high_latency".to_string(),
name: "High Script Latency".to_string(),
metric_name: "amberjs_script_execution_duration_seconds".to_string(),
condition: AlertCondition::GreaterThan(1.0), threshold: 1.0,
duration: Duration::from_secs(60),
severity: AlertSeverity::Warning,
labels: HashMap::new(),
description: "Script execution latency is above 1 second".to_string(),
},
]
}
}
#[cfg(test)]
mod tests {
#[tokio::test]
async fn test_alerting_system_creation() {
let system: _ = AlertingSystem::new();
assert!(system.rules.read().await.is_empty());
}
#[tokio::test]
async fn test_add_rule() {
let system: _ = AlertingSystem::new();
let rule: _ = AlertRule {
id: "test_rule".to_string(),
name: "Test Rule".to_string(),
metric_name: "test_metric".to_string(),
condition: AlertCondition::GreaterThan(100.0),
threshold: 100.0,
duration: Duration::from_secs(60),
severity: AlertSeverity::Warning,
labels: HashMap::new(),
description: "Test alert rule".to_string(),
};
system.add_rule(rule).await.unwrap();
assert_eq!(system.rules.read().await.len(), 1);
}
#[test]
fn test_alert_condition() {
assert!(AlertCondition::GreaterThan(100.0).is_triggered(150.0));
assert!(!AlertCondition::GreaterThan(100.0).is_triggered(50.0));
assert!(AlertCondition::LessThan(100.0).is_triggered(50.0));
assert!(!AlertCondition::LessThan(100.0).is_triggered(150.0));
assert!(AlertCondition::Between(50.0, 100.0).is_triggered(75.0));
assert!(!AlertCondition::Between(50.0, 100.0).is_triggered(25.0));
}
}
impl AlertCondition {
fn is_triggered(&self, value: f64) -> bool {
match self {
AlertCondition::GreaterThan(threshold) => value > *threshold,
AlertCondition::LessThan(threshold) => value < *threshold,
AlertCondition::EqualTo(threshold) => (value - *threshold).abs() < f64::EPSILON,
AlertCondition::NotEqualTo(threshold) => (value - *threshold).abs() >= f64::EPSILON,
AlertCondition::Between(min, max) => value >= *min && value <= *max,
AlertCondition::Outside(min, max) => value < *min || value > *max,
}
}
}