Skip to main content

fetchium_core/telemetry/
metrics.rs

1//! Lock-free pipeline metrics — latency histograms, counters, gauges.
2//!
3//! Uses atomic operations throughout for zero-contention in concurrent pipelines.
4//! Metrics are organized by operation type (search, fetch, extract, rank, validate).
5
6use dashmap::DashMap;
7use std::sync::atomic::{AtomicU64, Ordering};
8use std::sync::Arc;
9use std::time::{Duration, Instant};
10
11/// Global pipeline metrics collector.
12///
13/// Thread-safe, lock-free, designed for concurrent access from
14/// multiple search backends and pipeline stages simultaneously.
15#[derive(Clone)]
16pub struct PipelineMetrics {
17    /// Per-operation latency tracking.
18    operations: Arc<DashMap<String, OperationStats>>,
19    /// Global counters.
20    counters: Arc<GlobalCounters>,
21    /// When the metrics collector was created.
22    started_at: Instant,
23}
24
25/// Atomic global counters for pipeline-wide metrics.
26struct GlobalCounters {
27    total_searches: AtomicU64,
28    total_fetches: AtomicU64,
29    total_extractions: AtomicU64,
30    total_cache_hits: AtomicU64,
31    total_cache_misses: AtomicU64,
32    total_errors: AtomicU64,
33    total_tokens_processed: AtomicU64,
34    total_bytes_fetched: AtomicU64,
35}
36
37impl GlobalCounters {
38    fn new() -> Self {
39        Self {
40            total_searches: AtomicU64::new(0),
41            total_fetches: AtomicU64::new(0),
42            total_extractions: AtomicU64::new(0),
43            total_cache_hits: AtomicU64::new(0),
44            total_cache_misses: AtomicU64::new(0),
45            total_errors: AtomicU64::new(0),
46            total_tokens_processed: AtomicU64::new(0),
47            total_bytes_fetched: AtomicU64::new(0),
48        }
49    }
50}
51
52/// Per-operation latency and count tracking using HDR-style bucketing.
53#[derive(Debug)]
54struct OperationStats {
55    count: AtomicU64,
56    total_duration_us: AtomicU64,
57    min_duration_us: AtomicU64,
58    max_duration_us: AtomicU64,
59    errors: AtomicU64,
60}
61
62impl Default for OperationStats {
63    fn default() -> Self {
64        Self {
65            count: AtomicU64::new(0),
66            total_duration_us: AtomicU64::new(0),
67            min_duration_us: AtomicU64::new(u64::MAX),
68            max_duration_us: AtomicU64::new(0),
69            errors: AtomicU64::new(0),
70        }
71    }
72}
73
74impl OperationStats {
75    fn record(&self, duration: Duration) {
76        let us = duration.as_micros() as u64;
77        self.count.fetch_add(1, Ordering::Relaxed);
78        self.total_duration_us.fetch_add(us, Ordering::Relaxed);
79        self.min_duration_us.fetch_min(us, Ordering::Relaxed);
80        self.max_duration_us.fetch_max(us, Ordering::Relaxed);
81    }
82
83    fn record_error(&self) {
84        self.errors.fetch_add(1, Ordering::Relaxed);
85    }
86
87    fn snapshot(&self) -> OperationSnapshot {
88        let count = self.count.load(Ordering::Relaxed);
89        let total_us = self.total_duration_us.load(Ordering::Relaxed);
90        let min_us = self.min_duration_us.load(Ordering::Relaxed);
91        let max_us = self.max_duration_us.load(Ordering::Relaxed);
92        let errors = self.errors.load(Ordering::Relaxed);
93
94        OperationSnapshot {
95            count,
96            avg_duration: if count > 0 {
97                Duration::from_micros(total_us / count)
98            } else {
99                Duration::ZERO
100            },
101            min_duration: if min_us == u64::MAX {
102                Duration::ZERO
103            } else {
104                Duration::from_micros(min_us)
105            },
106            max_duration: Duration::from_micros(max_us),
107            errors,
108            error_rate: if count > 0 {
109                errors as f64 / count as f64
110            } else {
111                0.0
112            },
113        }
114    }
115}
116
117impl PipelineMetrics {
118    /// Create a new metrics collector.
119    pub fn new() -> Self {
120        Self {
121            operations: Arc::new(DashMap::new()),
122            counters: Arc::new(GlobalCounters::new()),
123            started_at: Instant::now(),
124        }
125    }
126
127    /// Start timing an operation. Returns a timer that records on drop.
128    pub fn start_operation(&self, operation: &str) -> OperationTimer {
129        OperationTimer {
130            operation: operation.to_string(),
131            start: Instant::now(),
132            metrics: self.clone(),
133            errored: false,
134        }
135    }
136
137    /// Record a completed operation with explicit duration.
138    pub fn record_operation(&self, operation: &str, duration: Duration) {
139        let entry = self.operations.entry(operation.to_string()).or_default();
140        entry.value().record(duration);
141    }
142
143    /// Record an error for an operation.
144    pub fn record_error(&self, operation: &str) {
145        let entry = self.operations.entry(operation.to_string()).or_default();
146        entry.value().record_error();
147        self.counters.total_errors.fetch_add(1, Ordering::Relaxed);
148    }
149
150    // ─── Counter Helpers ──────────────────────────────────────────
151
152    /// Increment the search counter.
153    pub fn inc_searches(&self) {
154        self.counters.total_searches.fetch_add(1, Ordering::Relaxed);
155    }
156
157    /// Increment the fetch counter.
158    pub fn inc_fetches(&self) {
159        self.counters.total_fetches.fetch_add(1, Ordering::Relaxed);
160    }
161
162    /// Increment the extraction counter.
163    pub fn inc_extractions(&self) {
164        self.counters
165            .total_extractions
166            .fetch_add(1, Ordering::Relaxed);
167    }
168
169    /// Record a cache hit.
170    pub fn inc_cache_hits(&self) {
171        self.counters
172            .total_cache_hits
173            .fetch_add(1, Ordering::Relaxed);
174    }
175
176    /// Record a cache miss.
177    pub fn inc_cache_misses(&self) {
178        self.counters
179            .total_cache_misses
180            .fetch_add(1, Ordering::Relaxed);
181    }
182
183    /// Add to total tokens processed.
184    pub fn add_tokens(&self, count: u64) {
185        self.counters
186            .total_tokens_processed
187            .fetch_add(count, Ordering::Relaxed);
188    }
189
190    /// Add to total bytes fetched.
191    pub fn add_bytes(&self, count: u64) {
192        self.counters
193            .total_bytes_fetched
194            .fetch_add(count, Ordering::Relaxed);
195    }
196
197    // ─── Snapshots ────────────────────────────────────────────────
198
199    /// Get a full metrics snapshot for reporting.
200    pub fn snapshot(&self) -> MetricsSnapshot {
201        let uptime = self.started_at.elapsed();
202        let operations: std::collections::HashMap<String, OperationSnapshot> = self
203            .operations
204            .iter()
205            .map(|entry| (entry.key().clone(), entry.value().snapshot()))
206            .collect();
207
208        let total_searches = self.counters.total_searches.load(Ordering::Relaxed);
209        let total_fetches = self.counters.total_fetches.load(Ordering::Relaxed);
210
211        MetricsSnapshot {
212            uptime,
213            total_searches,
214            total_fetches,
215            total_extractions: self.counters.total_extractions.load(Ordering::Relaxed),
216            total_cache_hits: self.counters.total_cache_hits.load(Ordering::Relaxed),
217            total_cache_misses: self.counters.total_cache_misses.load(Ordering::Relaxed),
218            total_errors: self.counters.total_errors.load(Ordering::Relaxed),
219            total_tokens_processed: self.counters.total_tokens_processed.load(Ordering::Relaxed),
220            total_bytes_fetched: self.counters.total_bytes_fetched.load(Ordering::Relaxed),
221            searches_per_sec: if uptime.as_secs() > 0 {
222                total_searches as f64 / uptime.as_secs_f64()
223            } else {
224                0.0
225            },
226            fetches_per_sec: if uptime.as_secs() > 0 {
227                total_fetches as f64 / uptime.as_secs_f64()
228            } else {
229                0.0
230            },
231            cache_hit_rate: {
232                let hits = self.counters.total_cache_hits.load(Ordering::Relaxed);
233                let misses = self.counters.total_cache_misses.load(Ordering::Relaxed);
234                let total = hits + misses;
235                if total > 0 {
236                    hits as f64 / total as f64
237                } else {
238                    0.0
239                }
240            },
241            operations,
242        }
243    }
244
245    /// Get a summary suitable for CLI display.
246    pub fn summary_line(&self) -> String {
247        let snap = self.snapshot();
248        format!(
249            "searches={} fetches={} extractions={} errors={} cache_hit_rate={:.0}% uptime={:.1}s",
250            snap.total_searches,
251            snap.total_fetches,
252            snap.total_extractions,
253            snap.total_errors,
254            snap.cache_hit_rate * 100.0,
255            snap.uptime.as_secs_f64(),
256        )
257    }
258}
259
260impl Default for PipelineMetrics {
261    fn default() -> Self {
262        Self::new()
263    }
264}
265
266/// RAII timer — records operation duration on drop.
267pub struct OperationTimer {
268    operation: String,
269    start: Instant,
270    metrics: PipelineMetrics,
271    errored: bool,
272}
273
274impl OperationTimer {
275    /// Mark this operation as failed.
276    pub fn mark_error(&mut self) {
277        self.errored = true;
278    }
279
280    /// Get elapsed time so far.
281    pub fn elapsed(&self) -> Duration {
282        self.start.elapsed()
283    }
284}
285
286impl Drop for OperationTimer {
287    fn drop(&mut self) {
288        let duration = self.start.elapsed();
289        self.metrics.record_operation(&self.operation, duration);
290        if self.errored {
291            self.metrics.record_error(&self.operation);
292        }
293    }
294}
295
296/// Point-in-time snapshot of all metrics.
297#[derive(Debug, Clone)]
298pub struct MetricsSnapshot {
299    pub uptime: Duration,
300    pub total_searches: u64,
301    pub total_fetches: u64,
302    pub total_extractions: u64,
303    pub total_cache_hits: u64,
304    pub total_cache_misses: u64,
305    pub total_errors: u64,
306    pub total_tokens_processed: u64,
307    pub total_bytes_fetched: u64,
308    pub searches_per_sec: f64,
309    pub fetches_per_sec: f64,
310    pub cache_hit_rate: f64,
311    pub operations: std::collections::HashMap<String, OperationSnapshot>,
312}
313
314/// Snapshot of a single operation's metrics.
315#[derive(Debug, Clone)]
316pub struct OperationSnapshot {
317    pub count: u64,
318    pub avg_duration: Duration,
319    pub min_duration: Duration,
320    pub max_duration: Duration,
321    pub errors: u64,
322    pub error_rate: f64,
323}
324
325#[cfg(test)]
326mod tests {
327    use super::*;
328
329    #[test]
330    fn metrics_basic_counting() {
331        let m = PipelineMetrics::new();
332        m.inc_searches();
333        m.inc_searches();
334        m.inc_fetches();
335
336        let snap = m.snapshot();
337        assert_eq!(snap.total_searches, 2);
338        assert_eq!(snap.total_fetches, 1);
339    }
340
341    #[test]
342    fn operation_timing() {
343        let m = PipelineMetrics::new();
344        m.record_operation("test_op", Duration::from_millis(100));
345        m.record_operation("test_op", Duration::from_millis(200));
346        m.record_operation("test_op", Duration::from_millis(300));
347
348        let snap = m.snapshot();
349        let op = snap.operations.get("test_op").unwrap();
350        assert_eq!(op.count, 3);
351        assert!(op.avg_duration.as_millis() >= 150 && op.avg_duration.as_millis() <= 250);
352        assert!(op.min_duration.as_millis() >= 90 && op.min_duration.as_millis() <= 110);
353        assert!(op.max_duration.as_millis() >= 290 && op.max_duration.as_millis() <= 310);
354    }
355
356    #[test]
357    fn operation_timer_raii() {
358        let m = PipelineMetrics::new();
359        {
360            let _timer = m.start_operation("timed_op");
361            std::thread::sleep(Duration::from_millis(10));
362        }
363        let snap = m.snapshot();
364        let op = snap.operations.get("timed_op").unwrap();
365        assert_eq!(op.count, 1);
366        assert!(op.avg_duration.as_millis() >= 5);
367    }
368
369    #[test]
370    fn error_tracking() {
371        let m = PipelineMetrics::new();
372        m.record_operation("fail_op", Duration::from_millis(50));
373        m.record_error("fail_op");
374
375        let snap = m.snapshot();
376        assert_eq!(snap.total_errors, 1);
377        let op = snap.operations.get("fail_op").unwrap();
378        assert_eq!(op.errors, 1);
379    }
380
381    #[test]
382    fn cache_hit_rate() {
383        let m = PipelineMetrics::new();
384        m.inc_cache_hits();
385        m.inc_cache_hits();
386        m.inc_cache_hits();
387        m.inc_cache_misses();
388
389        let snap = m.snapshot();
390        assert!((snap.cache_hit_rate - 0.75).abs() < 0.01);
391    }
392
393    #[test]
394    fn summary_line_format() {
395        let m = PipelineMetrics::new();
396        m.inc_searches();
397        let line = m.summary_line();
398        assert!(line.contains("searches=1"));
399    }
400}