llm_test_bench_core/monitoring/
mod.rs1pub mod metrics;
87pub mod events;
88pub mod prometheus;
89pub mod websocket;
90pub mod dashboard;
91pub mod collector;
92pub mod integration;
93
94pub use metrics::{
95 Metric, MetricType, MetricValue, MetricLabels,
96 RequestMetric, LatencyMetric, TokenMetric, CostMetric, ErrorMetric,
97};
98
99pub use events::{
100 MonitoringEvent, EventBus, EventSubscriber, EventType,
101 BenchmarkEvent, ProviderEvent, EvaluationEvent,
102};
103
104pub use prometheus::{PrometheusExporter, PrometheusConfig};
105pub use websocket::{WebSocketServer, WebSocketConfig, WebSocketMessage};
106pub use dashboard::{Dashboard, DashboardConfig};
107pub use collector::{MetricCollector, CollectorConfig};
108pub use integration::{MonitoredProvider, monitor_providers};
109
110use anyhow::Result;
111use std::sync::Arc;
112use parking_lot::RwLock;
113
114pub struct MonitoringSystem {
116 config: MonitoringConfig,
117 event_bus: Arc<EventBus>,
118 prometheus: Arc<PrometheusExporter>,
119 websocket: Arc<WebSocketServer>,
120 collector: Arc<MetricCollector>,
121}
122
123#[derive(Debug, Clone)]
125pub struct MonitoringConfig {
126 pub prometheus_enabled: bool,
128 pub prometheus_port: u16,
130 pub websocket_enabled: bool,
132 pub websocket_port: u16,
134 pub dashboard_enabled: bool,
136 pub dashboard_port: u16,
138 pub retention_period: u64,
140 pub detailed_metrics: bool,
142}
143
144impl Default for MonitoringConfig {
145 fn default() -> Self {
146 Self {
147 prometheus_enabled: true,
148 prometheus_port: 9090,
149 websocket_enabled: true,
150 websocket_port: 8080,
151 dashboard_enabled: true,
152 dashboard_port: 3000,
153 retention_period: 3600, detailed_metrics: false,
155 }
156 }
157}
158
159impl MonitoringConfig {
160 pub fn new() -> Self {
162 Self::default()
163 }
164
165 pub fn with_prometheus(mut self, enabled: bool) -> Self {
167 self.prometheus_enabled = enabled;
168 self
169 }
170
171 pub fn with_prometheus_port(mut self, port: u16) -> Self {
173 self.prometheus_port = port;
174 self
175 }
176
177 pub fn with_websocket(mut self, enabled: bool) -> Self {
179 self.websocket_enabled = enabled;
180 self
181 }
182
183 pub fn with_websocket_port(mut self, port: u16) -> Self {
185 self.websocket_port = port;
186 self
187 }
188
189 pub fn with_dashboard(mut self, enabled: bool) -> Self {
191 self.dashboard_enabled = enabled;
192 self
193 }
194
195 pub fn with_dashboard_port(mut self, port: u16) -> Self {
197 self.dashboard_port = port;
198 self
199 }
200
201 pub fn with_retention_period(mut self, seconds: u64) -> Self {
203 self.retention_period = seconds;
204 self
205 }
206
207 pub fn with_detailed_metrics(mut self, enabled: bool) -> Self {
209 self.detailed_metrics = enabled;
210 self
211 }
212}
213
214impl MonitoringSystem {
215 pub async fn new(config: MonitoringConfig) -> Result<Self> {
217 let event_bus = Arc::new(EventBus::new());
218
219 let prometheus_config = PrometheusConfig {
220 port: config.prometheus_port,
221 enabled: config.prometheus_enabled,
222 };
223 let prometheus = Arc::new(PrometheusExporter::new(prometheus_config)?);
224
225 let websocket_config = WebSocketConfig {
226 port: config.websocket_port,
227 enabled: config.websocket_enabled,
228 };
229 let websocket = Arc::new(WebSocketServer::new(websocket_config, event_bus.clone()).await?);
230
231 let collector_config = CollectorConfig {
232 retention_period: config.retention_period,
233 detailed_metrics: config.detailed_metrics,
234 };
235 let collector = Arc::new(MetricCollector::new(collector_config, event_bus.clone()));
236
237 Ok(Self {
238 config,
239 event_bus,
240 prometheus,
241 websocket,
242 collector,
243 })
244 }
245
246 pub async fn start(&self) -> Result<()> {
248 tracing::info!("Starting monitoring system");
249
250 if self.config.prometheus_enabled {
251 self.prometheus.start().await?;
252 tracing::info!("Prometheus exporter started on port {}", self.config.prometheus_port);
253 }
254
255 if self.config.websocket_enabled {
256 self.websocket.start().await?;
257 tracing::info!("WebSocket server started on port {}", self.config.websocket_port);
258 }
259
260 self.collector.start().await?;
261 tracing::info!("Metric collector started");
262
263 Ok(())
264 }
265
266 pub async fn stop(&self) -> Result<()> {
268 tracing::info!("Stopping monitoring system");
269
270 if self.config.prometheus_enabled {
271 self.prometheus.stop().await?;
272 }
273
274 if self.config.websocket_enabled {
275 self.websocket.stop().await?;
276 }
277
278 self.collector.stop().await?;
279
280 Ok(())
281 }
282
283 pub fn record_request(&self, provider: &str, model: &str) {
285 let event = MonitoringEvent::request(provider, model);
286 self.event_bus.publish(event);
287 }
288
289 pub fn record_latency(&self, provider: &str, latency: f64) {
291 let event = MonitoringEvent::latency(provider, latency);
292 self.event_bus.publish(event);
293 }
294
295 pub fn record_tokens(&self, provider: &str, input_tokens: u64, output_tokens: u64) {
297 let event = MonitoringEvent::tokens(provider, input_tokens, output_tokens);
298 self.event_bus.publish(event);
299 }
300
301 pub fn record_cost(&self, provider: &str, cost: f64) {
303 let event = MonitoringEvent::cost(provider, cost);
304 self.event_bus.publish(event);
305 }
306
307 pub fn record_error(&self, provider: &str, error_type: &str) {
309 let event = MonitoringEvent::error(provider, error_type);
310 self.event_bus.publish(event);
311 }
312
313 pub fn event_bus(&self) -> Arc<EventBus> {
315 self.event_bus.clone()
316 }
317
318 pub async fn get_metrics(&self) -> Vec<Metric> {
320 self.collector.get_metrics().await
321 }
322
323 pub async fn get_provider_metrics(&self, provider: &str) -> Vec<Metric> {
325 self.collector.get_provider_metrics(provider).await
326 }
327}
328
329#[cfg(test)]
330mod tests {
331 use super::*;
332
333 #[test]
334 fn test_monitoring_config() {
335 let config = MonitoringConfig::new()
336 .with_prometheus_port(9091)
337 .with_websocket_port(8081)
338 .with_detailed_metrics(true);
339
340 assert_eq!(config.prometheus_port, 9091);
341 assert_eq!(config.websocket_port, 8081);
342 assert!(config.detailed_metrics);
343 }
344
345 #[tokio::test]
346 async fn test_monitoring_system_creation() {
347 let config = MonitoringConfig::new()
348 .with_prometheus(false)
349 .with_websocket(false);
350
351 let result = MonitoringSystem::new(config).await;
352 assert!(result.is_ok());
353 }
354}