Skip to main content

llm_test_bench_core/monitoring/
mod.rs

1// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
2// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
3// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
4// option. This file may not be copied, modified, or distributed
5// except according to those terms.
6
7//! # Real-time Monitoring System
8//!
9//! This module provides enterprise-grade monitoring capabilities including:
10//!
11//! - **Prometheus Metrics**: Industry-standard metrics export for LLM operations
12//! - **WebSocket Dashboards**: Real-time streaming of metrics and events
13//! - **Event System**: Publish/subscribe event bus for distributed monitoring
14//! - **Live Dashboards**: Interactive HTML dashboards with real-time updates
15//!
16//! ## Architecture
17//!
18//! ```text
19//! ┌─────────────────────────────────────────────────────────────┐
20//! │                    LLM Test Bench Core                      │
21//! └─────────────────────────────────────────────────────────────┘
22//!                              │
23//!                              ▼
24//!                    ┌──────────────────┐
25//!                    │   Event Bus      │
26//!                    │  (pub/sub)       │
27//!                    └──────────────────┘
28//!                              │
29//!                 ┌────────────┼────────────┐
30//!                 ▼            ▼            ▼
31//!         ┌──────────┐  ┌──────────┐  ┌──────────┐
32//!         │Prometheus│  │WebSocket │  │ Metrics  │
33//!         │ Exporter │  │  Server  │  │Collector │
34//!         └──────────┘  └──────────┘  └──────────┘
35//!                │            │            │
36//!                ▼            ▼            ▼
37//!         [Prometheus]  [Dashboard]  [Aggregation]
38//! ```
39//!
40//! ## Features
41//!
42//! ### Prometheus Integration
43//! - Counter metrics (requests, tokens, errors)
44//! - Gauge metrics (active requests, queue depth)
45//! - Histogram metrics (latency, cost distribution)
46//! - Summary metrics (percentiles)
47//!
48//! ### WebSocket Streaming
49//! - Real-time event streaming
50//! - Live metric updates
51//! - Benchmark progress tracking
52//! - Connection management
53//!
54//! ### Dashboard
55//! - Interactive charts (Chart.js)
56//! - Real-time updates via WebSocket
57//! - Provider comparison views
58//! - Cost tracking
59//!
60//! ## Usage
61//!
62//! ```rust,no_run
63//! use llm_test_bench_core::monitoring::{MonitoringSystem, MonitoringConfig};
64//!
65//! #[tokio::main]
66//! async fn main() -> anyhow::Result<()> {
67//!     // Initialize monitoring system
68//!     let config = MonitoringConfig::default()
69//!         .with_prometheus_port(9090)
70//!         .with_websocket_port(8080);
71//!
72//!     let monitoring = MonitoringSystem::new(config).await?;
73//!
74//!     // Start monitoring services
75//!     monitoring.start().await?;
76//!
77//!     // Record metrics
78//!     monitoring.record_request("openai", "gpt-4");
79//!     monitoring.record_latency("openai", 1.5);
80//!     monitoring.record_tokens("openai", 150, 50);
81//!
82//!     Ok(())
83//! }
84//! ```
85
86pub mod metrics;
87pub mod events;
88pub mod prometheus;
89pub mod websocket;
90pub mod dashboard;
91pub mod collector;
92pub mod integration;
93
94pub use metrics::{
95    Metric, MetricType, MetricValue, MetricLabels,
96    RequestMetric, LatencyMetric, TokenMetric, CostMetric, ErrorMetric,
97};
98
99pub use events::{
100    MonitoringEvent, EventBus, EventSubscriber, EventType,
101    BenchmarkEvent, ProviderEvent, EvaluationEvent,
102};
103
104pub use prometheus::{PrometheusExporter, PrometheusConfig};
105pub use websocket::{WebSocketServer, WebSocketConfig, WebSocketMessage};
106pub use dashboard::{Dashboard, DashboardConfig};
107pub use collector::{MetricCollector, CollectorConfig};
108pub use integration::{MonitoredProvider, monitor_providers};
109
110use anyhow::Result;
111use std::sync::Arc;
112use parking_lot::RwLock;
113
114/// Main monitoring system that coordinates all monitoring components
115pub struct MonitoringSystem {
116    config: MonitoringConfig,
117    event_bus: Arc<EventBus>,
118    prometheus: Arc<PrometheusExporter>,
119    websocket: Arc<WebSocketServer>,
120    collector: Arc<MetricCollector>,
121}
122
123/// Configuration for the monitoring system
124#[derive(Debug, Clone)]
125pub struct MonitoringConfig {
126    /// Enable Prometheus metrics export
127    pub prometheus_enabled: bool,
128    /// Prometheus metrics port
129    pub prometheus_port: u16,
130    /// Enable WebSocket server
131    pub websocket_enabled: bool,
132    /// WebSocket server port
133    pub websocket_port: u16,
134    /// Enable dashboard
135    pub dashboard_enabled: bool,
136    /// Dashboard port
137    pub dashboard_port: u16,
138    /// Metric retention period (seconds)
139    pub retention_period: u64,
140    /// Enable detailed metrics (may impact performance)
141    pub detailed_metrics: bool,
142}
143
144impl Default for MonitoringConfig {
145    fn default() -> Self {
146        Self {
147            prometheus_enabled: true,
148            prometheus_port: 9090,
149            websocket_enabled: true,
150            websocket_port: 8080,
151            dashboard_enabled: true,
152            dashboard_port: 3000,
153            retention_period: 3600, // 1 hour
154            detailed_metrics: false,
155        }
156    }
157}
158
159impl MonitoringConfig {
160    /// Create a new monitoring configuration
161    pub fn new() -> Self {
162        Self::default()
163    }
164
165    /// Enable Prometheus metrics
166    pub fn with_prometheus(mut self, enabled: bool) -> Self {
167        self.prometheus_enabled = enabled;
168        self
169    }
170
171    /// Set Prometheus port
172    pub fn with_prometheus_port(mut self, port: u16) -> Self {
173        self.prometheus_port = port;
174        self
175    }
176
177    /// Enable WebSocket server
178    pub fn with_websocket(mut self, enabled: bool) -> Self {
179        self.websocket_enabled = enabled;
180        self
181    }
182
183    /// Set WebSocket port
184    pub fn with_websocket_port(mut self, port: u16) -> Self {
185        self.websocket_port = port;
186        self
187    }
188
189    /// Enable dashboard
190    pub fn with_dashboard(mut self, enabled: bool) -> Self {
191        self.dashboard_enabled = enabled;
192        self
193    }
194
195    /// Set dashboard port
196    pub fn with_dashboard_port(mut self, port: u16) -> Self {
197        self.dashboard_port = port;
198        self
199    }
200
201    /// Set metric retention period
202    pub fn with_retention_period(mut self, seconds: u64) -> Self {
203        self.retention_period = seconds;
204        self
205    }
206
207    /// Enable detailed metrics
208    pub fn with_detailed_metrics(mut self, enabled: bool) -> Self {
209        self.detailed_metrics = enabled;
210        self
211    }
212}
213
214impl MonitoringSystem {
215    /// Create a new monitoring system
216    pub async fn new(config: MonitoringConfig) -> Result<Self> {
217        let event_bus = Arc::new(EventBus::new());
218
219        let prometheus_config = PrometheusConfig {
220            port: config.prometheus_port,
221            enabled: config.prometheus_enabled,
222        };
223        let prometheus = Arc::new(PrometheusExporter::new(prometheus_config)?);
224
225        let websocket_config = WebSocketConfig {
226            port: config.websocket_port,
227            enabled: config.websocket_enabled,
228        };
229        let websocket = Arc::new(WebSocketServer::new(websocket_config, event_bus.clone()).await?);
230
231        let collector_config = CollectorConfig {
232            retention_period: config.retention_period,
233            detailed_metrics: config.detailed_metrics,
234        };
235        let collector = Arc::new(MetricCollector::new(collector_config, event_bus.clone()));
236
237        Ok(Self {
238            config,
239            event_bus,
240            prometheus,
241            websocket,
242            collector,
243        })
244    }
245
246    /// Start all monitoring services
247    pub async fn start(&self) -> Result<()> {
248        tracing::info!("Starting monitoring system");
249
250        if self.config.prometheus_enabled {
251            self.prometheus.start().await?;
252            tracing::info!("Prometheus exporter started on port {}", self.config.prometheus_port);
253        }
254
255        if self.config.websocket_enabled {
256            self.websocket.start().await?;
257            tracing::info!("WebSocket server started on port {}", self.config.websocket_port);
258        }
259
260        self.collector.start().await?;
261        tracing::info!("Metric collector started");
262
263        Ok(())
264    }
265
266    /// Stop all monitoring services
267    pub async fn stop(&self) -> Result<()> {
268        tracing::info!("Stopping monitoring system");
269
270        if self.config.prometheus_enabled {
271            self.prometheus.stop().await?;
272        }
273
274        if self.config.websocket_enabled {
275            self.websocket.stop().await?;
276        }
277
278        self.collector.stop().await?;
279
280        Ok(())
281    }
282
283    /// Record a request metric
284    pub fn record_request(&self, provider: &str, model: &str) {
285        let event = MonitoringEvent::request(provider, model);
286        self.event_bus.publish(event);
287    }
288
289    /// Record a latency metric
290    pub fn record_latency(&self, provider: &str, latency: f64) {
291        let event = MonitoringEvent::latency(provider, latency);
292        self.event_bus.publish(event);
293    }
294
295    /// Record token usage
296    pub fn record_tokens(&self, provider: &str, input_tokens: u64, output_tokens: u64) {
297        let event = MonitoringEvent::tokens(provider, input_tokens, output_tokens);
298        self.event_bus.publish(event);
299    }
300
301    /// Record cost
302    pub fn record_cost(&self, provider: &str, cost: f64) {
303        let event = MonitoringEvent::cost(provider, cost);
304        self.event_bus.publish(event);
305    }
306
307    /// Record an error
308    pub fn record_error(&self, provider: &str, error_type: &str) {
309        let event = MonitoringEvent::error(provider, error_type);
310        self.event_bus.publish(event);
311    }
312
313    /// Get the event bus for custom subscriptions
314    pub fn event_bus(&self) -> Arc<EventBus> {
315        self.event_bus.clone()
316    }
317
318    /// Get current metrics snapshot
319    pub async fn get_metrics(&self) -> Vec<Metric> {
320        self.collector.get_metrics().await
321    }
322
323    /// Get metrics for a specific provider
324    pub async fn get_provider_metrics(&self, provider: &str) -> Vec<Metric> {
325        self.collector.get_provider_metrics(provider).await
326    }
327}
328
329#[cfg(test)]
330mod tests {
331    use super::*;
332
333    #[test]
334    fn test_monitoring_config() {
335        let config = MonitoringConfig::new()
336            .with_prometheus_port(9091)
337            .with_websocket_port(8081)
338            .with_detailed_metrics(true);
339
340        assert_eq!(config.prometheus_port, 9091);
341        assert_eq!(config.websocket_port, 8081);
342        assert!(config.detailed_metrics);
343    }
344
345    #[tokio::test]
346    async fn test_monitoring_system_creation() {
347        let config = MonitoringConfig::new()
348            .with_prometheus(false)
349            .with_websocket(false);
350
351        let result = MonitoringSystem::new(config).await;
352        assert!(result.is_ok());
353    }
354}