Skip to main content

llm_test_bench_core/monitoring/
prometheus.rs

1// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
2// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
3// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
4// option. This file may not be copied, modified, or distributed
5// except according to those terms.
6
7//! Prometheus metrics exporter for LLM Test Bench.
8
9use anyhow::Result;
10use prometheus::{
11    Counter, CounterVec, Gauge, GaugeVec, Histogram, HistogramVec, Opts, Registry,
12    TextEncoder, Encoder, HistogramOpts,
13};
14use std::sync::Arc;
15use std::net::SocketAddr;
16use parking_lot::RwLock;
17use axum::{
18    routing::get,
19    Router,
20    response::{Response, IntoResponse},
21    http::StatusCode,
22};
23use tokio::task::JoinHandle;
24
25/// Prometheus exporter configuration
26#[derive(Debug, Clone)]
27pub struct PrometheusConfig {
28    /// Port to expose metrics on
29    pub port: u16,
30    /// Enable the exporter
31    pub enabled: bool,
32}
33
34impl Default for PrometheusConfig {
35    fn default() -> Self {
36        Self {
37            port: 9090,
38            enabled: true,
39        }
40    }
41}
42
43/// Prometheus metrics exporter
44pub struct PrometheusExporter {
45    config: PrometheusConfig,
46    registry: Arc<Registry>,
47    metrics: Arc<PrometheusMetrics>,
48    server_handle: Arc<RwLock<Option<JoinHandle<()>>>>,
49}
50
51/// Collection of Prometheus metrics
52struct PrometheusMetrics {
53    // Request metrics
54    requests_total: CounterVec,
55    requests_duration: HistogramVec,
56    requests_active: GaugeVec,
57
58    // Token metrics
59    tokens_input_total: CounterVec,
60    tokens_output_total: CounterVec,
61
62    // Cost metrics
63    cost_usd_total: CounterVec,
64
65    // Error metrics
66    errors_total: CounterVec,
67
68    // Evaluation metrics
69    evaluation_score: GaugeVec,
70
71    // Benchmark metrics
72    benchmark_progress: GaugeVec,
73    benchmark_duration: HistogramVec,
74}
75
76impl PrometheusMetrics {
77    fn new(registry: &Registry) -> Result<Self> {
78        // Request metrics
79        let requests_total = CounterVec::new(
80            Opts::new("llm_requests_total", "Total number of LLM requests"),
81            &["provider", "model", "status"],
82        )?;
83        registry.register(Box::new(requests_total.clone()))?;
84
85        let requests_duration = HistogramVec::new(
86            HistogramOpts::new("llm_request_duration_seconds", "Request duration in seconds")
87                .buckets(vec![0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0]),
88            &["provider", "model"],
89        )?;
90        registry.register(Box::new(requests_duration.clone()))?;
91
92        let requests_active = GaugeVec::new(
93            Opts::new("llm_requests_active", "Number of active requests"),
94            &["provider"],
95        )?;
96        registry.register(Box::new(requests_active.clone()))?;
97
98        // Token metrics
99        let tokens_input_total = CounterVec::new(
100            Opts::new("llm_tokens_input_total", "Total input tokens processed"),
101            &["provider", "model"],
102        )?;
103        registry.register(Box::new(tokens_input_total.clone()))?;
104
105        let tokens_output_total = CounterVec::new(
106            Opts::new("llm_tokens_output_total", "Total output tokens generated"),
107            &["provider", "model"],
108        )?;
109        registry.register(Box::new(tokens_output_total.clone()))?;
110
111        // Cost metrics
112        let cost_usd_total = CounterVec::new(
113            Opts::new("llm_cost_usd_total", "Total cost in USD"),
114            &["provider", "model"],
115        )?;
116        registry.register(Box::new(cost_usd_total.clone()))?;
117
118        // Error metrics
119        let errors_total = CounterVec::new(
120            Opts::new("llm_errors_total", "Total number of errors"),
121            &["provider", "model", "error_type"],
122        )?;
123        registry.register(Box::new(errors_total.clone()))?;
124
125        // Evaluation metrics
126        let evaluation_score = GaugeVec::new(
127            Opts::new("llm_evaluation_score", "Evaluation metric score"),
128            &["provider", "model", "metric"],
129        )?;
130        registry.register(Box::new(evaluation_score.clone()))?;
131
132        // Benchmark metrics
133        let benchmark_progress = GaugeVec::new(
134            Opts::new("llm_benchmark_progress", "Benchmark progress percentage"),
135            &["benchmark_id", "name"],
136        )?;
137        registry.register(Box::new(benchmark_progress.clone()))?;
138
139        let benchmark_duration = HistogramVec::new(
140            HistogramOpts::new("llm_benchmark_duration_seconds", "Benchmark duration in seconds")
141                .buckets(vec![1.0, 5.0, 10.0, 30.0, 60.0, 300.0, 600.0, 1800.0]),
142            &["benchmark_id", "name"],
143        )?;
144        registry.register(Box::new(benchmark_duration.clone()))?;
145
146        Ok(Self {
147            requests_total,
148            requests_duration,
149            requests_active,
150            tokens_input_total,
151            tokens_output_total,
152            cost_usd_total,
153            errors_total,
154            evaluation_score,
155            benchmark_progress,
156            benchmark_duration,
157        })
158    }
159
160    /// Record a request
161    fn record_request(&self, provider: &str, model: &str, status: &str) {
162        self.requests_total
163            .with_label_values(&[provider, model, status])
164            .inc();
165    }
166
167    /// Record request duration
168    fn record_duration(&self, provider: &str, model: &str, duration: f64) {
169        self.requests_duration
170            .with_label_values(&[provider, model])
171            .observe(duration);
172    }
173
174    /// Set active requests
175    fn set_active_requests(&self, provider: &str, count: i64) {
176        self.requests_active
177            .with_label_values(&[provider])
178            .set(count as f64);
179    }
180
181    /// Record tokens
182    fn record_tokens(&self, provider: &str, model: &str, input: u64, output: u64) {
183        self.tokens_input_total
184            .with_label_values(&[provider, model])
185            .inc_by(input as f64);
186        self.tokens_output_total
187            .with_label_values(&[provider, model])
188            .inc_by(output as f64);
189    }
190
191    /// Record cost
192    fn record_cost(&self, provider: &str, model: &str, cost: f64) {
193        self.cost_usd_total
194            .with_label_values(&[provider, model])
195            .inc_by(cost);
196    }
197
198    /// Record error
199    fn record_error(&self, provider: &str, model: &str, error_type: &str) {
200        self.errors_total
201            .with_label_values(&[provider, model, error_type])
202            .inc();
203    }
204
205    /// Record evaluation score
206    fn record_evaluation(&self, provider: &str, model: &str, metric: &str, score: f64) {
207        self.evaluation_score
208            .with_label_values(&[provider, model, metric])
209            .set(score);
210    }
211
212    /// Record benchmark progress
213    fn record_benchmark_progress(&self, benchmark_id: &str, name: &str, progress: f64) {
214        self.benchmark_progress
215            .with_label_values(&[benchmark_id, name])
216            .set(progress);
217    }
218
219    /// Record benchmark duration
220    fn record_benchmark_duration(&self, benchmark_id: &str, name: &str, duration: f64) {
221        self.benchmark_duration
222            .with_label_values(&[benchmark_id, name])
223            .observe(duration);
224    }
225}
226
227impl PrometheusExporter {
228    /// Create a new Prometheus exporter
229    pub fn new(config: PrometheusConfig) -> Result<Self> {
230        let registry = Registry::new();
231        let metrics = PrometheusMetrics::new(&registry)?;
232
233        Ok(Self {
234            config,
235            registry: Arc::new(registry),
236            metrics: Arc::new(metrics),
237            server_handle: Arc::new(RwLock::new(None)),
238        })
239    }
240
241    /// Start the Prometheus HTTP server
242    pub async fn start(&self) -> Result<()> {
243        if !self.config.enabled {
244            tracing::debug!("Prometheus exporter is disabled");
245            return Ok(());
246        }
247
248        let registry = self.registry.clone();
249        let port = self.config.port;
250
251        let app = Router::new()
252            .route("/metrics", get(move || Self::metrics_handler(registry.clone())));
253
254        let addr: SocketAddr = format!("0.0.0.0:{}", port).parse()?;
255        tracing::info!("Starting Prometheus exporter on {}", addr);
256
257        let server = tokio::spawn(async move {
258            let listener = tokio::net::TcpListener::bind(addr)
259                .await
260                .expect("Failed to bind Prometheus server");
261
262            axum::serve(listener, app)
263                .await
264                .expect("Prometheus server error");
265        });
266
267        let mut handle = self.server_handle.write();
268        *handle = Some(server);
269
270        Ok(())
271    }
272
273    /// Stop the Prometheus HTTP server
274    pub async fn stop(&self) -> Result<()> {
275        let mut handle = self.server_handle.write();
276        if let Some(h) = handle.take() {
277            h.abort();
278        }
279        Ok(())
280    }
281
282    /// Metrics endpoint handler
283    async fn metrics_handler(registry: Arc<Registry>) -> Response {
284        let encoder = TextEncoder::new();
285        let metric_families = registry.gather();
286
287        let mut buffer = Vec::new();
288        match encoder.encode(&metric_families, &mut buffer) {
289            Ok(_) => {
290                Response::builder()
291                    .status(StatusCode::OK)
292                    .header("Content-Type", encoder.format_type())
293                    .body(buffer.into())
294                    .unwrap()
295            }
296            Err(e) => {
297                tracing::error!("Failed to encode metrics: {}", e);
298                (StatusCode::INTERNAL_SERVER_ERROR, "Failed to encode metrics").into_response()
299            }
300        }
301    }
302
303    /// Record a request
304    pub fn record_request(&self, provider: &str, model: &str, status: &str) {
305        self.metrics.record_request(provider, model, status);
306    }
307
308    /// Record request duration
309    pub fn record_duration(&self, provider: &str, model: &str, duration: f64) {
310        self.metrics.record_duration(provider, model, duration);
311    }
312
313    /// Set active requests
314    pub fn set_active_requests(&self, provider: &str, count: i64) {
315        self.metrics.set_active_requests(provider, count);
316    }
317
318    /// Record tokens
319    pub fn record_tokens(&self, provider: &str, model: &str, input: u64, output: u64) {
320        self.metrics.record_tokens(provider, model, input, output);
321    }
322
323    /// Record cost
324    pub fn record_cost(&self, provider: &str, model: &str, cost: f64) {
325        self.metrics.record_cost(provider, model, cost);
326    }
327
328    /// Record error
329    pub fn record_error(&self, provider: &str, model: &str, error_type: &str) {
330        self.metrics.record_error(provider, model, error_type);
331    }
332
333    /// Record evaluation score
334    pub fn record_evaluation(&self, provider: &str, model: &str, metric: &str, score: f64) {
335        self.metrics.record_evaluation(provider, model, metric, score);
336    }
337
338    /// Record benchmark progress
339    pub fn record_benchmark_progress(&self, benchmark_id: &str, name: &str, progress: f64) {
340        self.metrics.record_benchmark_progress(benchmark_id, name, progress);
341    }
342
343    /// Record benchmark duration
344    pub fn record_benchmark_duration(&self, benchmark_id: &str, name: &str, duration: f64) {
345        self.metrics.record_benchmark_duration(benchmark_id, name, duration);
346    }
347}
348
349#[cfg(test)]
350mod tests {
351    use super::*;
352
353    #[test]
354    fn test_prometheus_config() {
355        let config = PrometheusConfig {
356            port: 9091,
357            enabled: true,
358        };
359        assert_eq!(config.port, 9091);
360        assert!(config.enabled);
361    }
362
363    #[tokio::test]
364    async fn test_prometheus_exporter_creation() {
365        let config = PrometheusConfig {
366            port: 9092,
367            enabled: false,
368        };
369        let exporter = PrometheusExporter::new(config);
370        assert!(exporter.is_ok());
371    }
372
373    #[tokio::test]
374    async fn test_record_metrics() {
375        let config = PrometheusConfig {
376            port: 9093,
377            enabled: false,
378        };
379        let exporter = PrometheusExporter::new(config).unwrap();
380
381        exporter.record_request("openai", "gpt-4", "success");
382        exporter.record_duration("openai", "gpt-4", 1.5);
383        exporter.record_tokens("openai", "gpt-4", 100, 50);
384        exporter.record_cost("openai", "gpt-4", 0.05);
385        exporter.record_error("openai", "gpt-4", "rate_limit");
386    }
387}