Skip to main content

ipfrs_interface/
metrics.rs

1//! Prometheus metrics for observability
2//!
3//! This module provides comprehensive metrics collection for monitoring
4//! IPFRS interface performance, usage patterns, and system health.
5//!
6//! # Global registry (lazy_static)
7//!
8//! The module exposes a set of well-known `lazy_static` metrics that register
9//! themselves in the **default** prometheus registry.  These are used by the
10//! HTTP middleware and route helpers.
11//!
12//! # Per-node registry (`IpfrsMetrics`)
13//!
14//! For fine-grained per-node observability (block operations, DHT, inference
15//! sessions, GossipSub, storage and GC) each `Node` or gateway instance
16//! can hold an [`IpfrsMetrics`] value.  Metrics are registered in a
17//! **private** [`prometheus::Registry`] so multiple test instances do not
18//! collide.
19
20use lazy_static::lazy_static;
21use prometheus::{
22    register_counter_vec, register_gauge_vec, register_histogram_vec, register_int_counter_vec,
23    register_int_gauge_vec, Counter, CounterVec, Encoder, Gauge, GaugeVec, Histogram,
24    HistogramOpts, HistogramVec, IntCounterVec, IntGaugeVec, Opts, Registry, TextEncoder,
25};
26use std::sync::Arc;
27use std::time::Instant;
28
29lazy_static! {
30    // HTTP Request Metrics
31
32    /// Total number of HTTP requests by endpoint and method
33    pub static ref HTTP_REQUESTS_TOTAL: IntCounterVec = register_int_counter_vec!(
34        "ipfrs_http_requests_total",
35        "Total number of HTTP requests",
36        &["endpoint", "method", "status"]
37    )
38    .expect("prometheus metric registration failed");
39
40    /// HTTP request duration in seconds
41    pub static ref HTTP_REQUEST_DURATION_SECONDS: HistogramVec = register_histogram_vec!(
42        "ipfrs_http_request_duration_seconds",
43        "HTTP request duration in seconds",
44        &["endpoint", "method"],
45        vec![0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0]
46    )
47    .expect("prometheus metric registration failed");
48
49    /// HTTP request body size in bytes
50    pub static ref HTTP_REQUEST_SIZE_BYTES: HistogramVec = register_histogram_vec!(
51        "ipfrs_http_request_size_bytes",
52        "HTTP request body size in bytes",
53        &["endpoint", "method"],
54        vec![
55            100.0,
56            1_000.0,
57            10_000.0,
58            100_000.0,
59            1_000_000.0,
60            10_000_000.0,
61            100_000_000.0
62        ]
63    )
64    .expect("prometheus metric registration failed");
65
66    /// HTTP response size in bytes
67    pub static ref HTTP_RESPONSE_SIZE_BYTES: HistogramVec = register_histogram_vec!(
68        "ipfrs_http_response_size_bytes",
69        "HTTP response body size in bytes",
70        &["endpoint", "method"],
71        vec![
72            100.0,
73            1_000.0,
74            10_000.0,
75            100_000.0,
76            1_000_000.0,
77            10_000_000.0,
78            100_000_000.0
79        ]
80    )
81    .expect("prometheus metric registration failed");
82
83    /// Currently active HTTP connections
84    pub static ref HTTP_CONNECTIONS_ACTIVE: IntGaugeVec = register_int_gauge_vec!(
85        "ipfrs_http_connections_active",
86        "Currently active HTTP connections",
87        &["endpoint"]
88    )
89    .expect("prometheus metric registration failed");
90
91    // Block Operations Metrics
92
93    /// Total blocks retrieved
94    pub static ref BLOCKS_RETRIEVED_TOTAL: IntCounterVec = register_int_counter_vec!(
95        "ipfrs_blocks_retrieved_total",
96        "Total number of blocks retrieved",
97        &["source"]
98    )
99    .expect("prometheus metric registration failed");
100
101    /// Total blocks stored
102    pub static ref BLOCKS_STORED_TOTAL: IntCounterVec = register_int_counter_vec!(
103        "ipfrs_blocks_stored_total",
104        "Total number of blocks stored",
105        &["destination"]
106    )
107    .expect("prometheus metric registration failed");
108
109    /// Block operation errors
110    pub static ref BLOCK_ERRORS_TOTAL: IntCounterVec = register_int_counter_vec!(
111        "ipfrs_block_errors_total",
112        "Total number of block operation errors",
113        &["operation", "error_type"]
114    )
115    .expect("prometheus metric registration failed");
116
117    /// Block retrieval duration
118    pub static ref BLOCK_RETRIEVAL_DURATION_SECONDS: HistogramVec = register_histogram_vec!(
119        "ipfrs_block_retrieval_duration_seconds",
120        "Block retrieval duration in seconds",
121        &["source"],
122        vec![0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0]
123    )
124    .expect("prometheus metric registration failed");
125
126    // Batch Operations Metrics
127
128    /// Batch operation size (number of items)
129    pub static ref BATCH_OPERATION_SIZE: HistogramVec = register_histogram_vec!(
130        "ipfrs_batch_operation_size",
131        "Number of items in batch operations",
132        &["operation"],
133        vec![1.0, 10.0, 50.0, 100.0, 500.0, 1000.0]
134    )
135    .expect("prometheus metric registration failed");
136
137    /// Batch operation duration
138    pub static ref BATCH_OPERATION_DURATION_SECONDS: HistogramVec = register_histogram_vec!(
139        "ipfrs_batch_operation_duration_seconds",
140        "Batch operation duration in seconds",
141        &["operation"],
142        vec![0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0, 30.0]
143    )
144    .expect("prometheus metric registration failed");
145
146    // Streaming Metrics
147
148    /// Total bytes uploaded
149    pub static ref UPLOAD_BYTES_TOTAL: CounterVec = register_counter_vec!(
150        "ipfrs_upload_bytes_total",
151        "Total bytes uploaded",
152        &["endpoint"]
153    )
154    .expect("prometheus metric registration failed");
155
156    /// Total bytes downloaded
157    pub static ref DOWNLOAD_BYTES_TOTAL: CounterVec = register_counter_vec!(
158        "ipfrs_download_bytes_total",
159        "Total bytes downloaded",
160        &["endpoint"]
161    )
162    .expect("prometheus metric registration failed");
163
164    /// Active streaming operations
165    pub static ref STREAMING_OPERATIONS_ACTIVE: IntGaugeVec = register_int_gauge_vec!(
166        "ipfrs_streaming_operations_active",
167        "Currently active streaming operations",
168        &["type"]
169    )
170    .expect("prometheus metric registration failed");
171
172    /// Streaming chunk size
173    pub static ref STREAMING_CHUNK_SIZE_BYTES: HistogramVec = register_histogram_vec!(
174        "ipfrs_streaming_chunk_size_bytes",
175        "Streaming chunk size in bytes",
176        &["operation"],
177        vec![
178            1024.0,
179            4096.0,
180            16384.0,
181            65536.0,
182            262144.0,
183            1048576.0
184        ]
185    )
186    .expect("prometheus metric registration failed");
187
188    // Cache Metrics
189
190    /// Cache hits
191    pub static ref CACHE_HITS_TOTAL: IntCounterVec = register_int_counter_vec!(
192        "ipfrs_cache_hits_total",
193        "Total cache hits",
194        &["cache_type"]
195    )
196    .expect("prometheus metric registration failed");
197
198    /// Cache misses
199    pub static ref CACHE_MISSES_TOTAL: IntCounterVec = register_int_counter_vec!(
200        "ipfrs_cache_misses_total",
201        "Total cache misses",
202        &["cache_type"]
203    )
204    .expect("prometheus metric registration failed");
205
206    /// Current cache size
207    pub static ref CACHE_SIZE_BYTES: GaugeVec = register_gauge_vec!(
208        "ipfrs_cache_size_bytes",
209        "Current cache size in bytes",
210        &["cache_type"]
211    )
212    .expect("prometheus metric registration failed");
213
214    // Authentication Metrics
215
216    /// Authentication attempts
217    pub static ref AUTH_ATTEMPTS_TOTAL: IntCounterVec = register_int_counter_vec!(
218        "ipfrs_auth_attempts_total",
219        "Total authentication attempts",
220        &["method", "result"]
221    )
222    .expect("prometheus metric registration failed");
223
224    /// Active authenticated sessions
225    pub static ref AUTH_SESSIONS_ACTIVE: IntGaugeVec = register_int_gauge_vec!(
226        "ipfrs_auth_sessions_active",
227        "Currently active authenticated sessions",
228        &["user"]
229    )
230    .expect("prometheus metric registration failed");
231
232    // Rate Limiting Metrics
233
234    /// Rate limit hits (requests blocked)
235    pub static ref RATE_LIMIT_HITS_TOTAL: IntCounterVec = register_int_counter_vec!(
236        "ipfrs_rate_limit_hits_total",
237        "Total rate limit hits (requests blocked)",
238        &["endpoint", "client_ip"]
239    )
240    .expect("prometheus metric registration failed");
241
242    /// Available rate limit tokens
243    pub static ref RATE_LIMIT_TOKENS_AVAILABLE: GaugeVec = register_gauge_vec!(
244        "ipfrs_rate_limit_tokens_available",
245        "Available rate limit tokens",
246        &["client_ip"]
247    )
248    .expect("prometheus metric registration failed");
249
250    // WebSocket Metrics
251
252    /// Active WebSocket connections
253    pub static ref WEBSOCKET_CONNECTIONS_ACTIVE: IntGaugeVec = register_int_gauge_vec!(
254        "ipfrs_websocket_connections_active",
255        "Currently active WebSocket connections",
256        &["topic"]
257    )
258    .expect("prometheus metric registration failed");
259
260    /// WebSocket messages sent
261    pub static ref WEBSOCKET_MESSAGES_SENT_TOTAL: IntCounterVec = register_int_counter_vec!(
262        "ipfrs_websocket_messages_sent_total",
263        "Total WebSocket messages sent",
264        &["topic", "event_type"]
265    )
266    .expect("prometheus metric registration failed");
267
268    /// WebSocket messages received
269    pub static ref WEBSOCKET_MESSAGES_RECEIVED_TOTAL: IntCounterVec = register_int_counter_vec!(
270        "ipfrs_websocket_messages_received_total",
271        "Total WebSocket messages received",
272        &["message_type"]
273    )
274    .expect("prometheus metric registration failed");
275
276    // gRPC Metrics
277
278    /// gRPC requests total
279    pub static ref GRPC_REQUESTS_TOTAL: IntCounterVec = register_int_counter_vec!(
280        "ipfrs_grpc_requests_total",
281        "Total gRPC requests",
282        &["service", "method", "status"]
283    )
284    .expect("prometheus metric registration failed");
285
286    /// gRPC request duration
287    pub static ref GRPC_REQUEST_DURATION_SECONDS: HistogramVec = register_histogram_vec!(
288        "ipfrs_grpc_request_duration_seconds",
289        "gRPC request duration in seconds",
290        &["service", "method"],
291        vec![0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0]
292    )
293    .expect("prometheus metric registration failed");
294
295    // Tensor Operations Metrics
296
297    /// Tensor operations total
298    pub static ref TENSOR_OPERATIONS_TOTAL: IntCounterVec = register_int_counter_vec!(
299        "ipfrs_tensor_operations_total",
300        "Total tensor operations",
301        &["operation", "dtype"]
302    )
303    .expect("prometheus metric registration failed");
304
305    /// Tensor slice operations
306    pub static ref TENSOR_SLICE_OPERATIONS_TOTAL: IntCounterVec = register_int_counter_vec!(
307        "ipfrs_tensor_slice_operations_total",
308        "Total tensor slice operations",
309        &["dimensions"]
310    )
311    .expect("prometheus metric registration failed");
312
313    /// Tensor size in bytes
314    pub static ref TENSOR_SIZE_BYTES: HistogramVec = register_histogram_vec!(
315        "ipfrs_tensor_size_bytes",
316        "Tensor size in bytes",
317        &["dtype"],
318        vec![
319            1000.0,
320            10_000.0,
321            100_000.0,
322            1_000_000.0,
323            10_000_000.0,
324            100_000_000.0,
325            1_000_000_000.0
326        ]
327    )
328    .expect("prometheus metric registration failed");
329
330    // System Metrics
331
332    /// Total memory allocated (in bytes)
333    pub static ref MEMORY_ALLOCATED_BYTES: IntGaugeVec = register_int_gauge_vec!(
334        "ipfrs_memory_allocated_bytes",
335        "Total memory allocated in bytes",
336        &["component"]
337    )
338    .expect("prometheus metric registration failed");
339
340    /// Number of goroutines (async tasks)
341    pub static ref ASYNC_TASKS_ACTIVE: IntGaugeVec = register_int_gauge_vec!(
342        "ipfrs_async_tasks_active",
343        "Currently active async tasks",
344        &["type"]
345    )
346    .expect("prometheus metric registration failed");
347}
348
349/// Helper struct for timing operations
350pub struct Timer {
351    start: Instant,
352    labels: Vec<String>,
353}
354
355impl Timer {
356    /// Create a new timer with labels
357    pub fn new(labels: Vec<String>) -> Self {
358        Self {
359            start: Instant::now(),
360            labels,
361        }
362    }
363
364    /// Observe the duration and record it to the given histogram
365    pub fn observe_duration(self, histogram: &HistogramVec) {
366        let duration = self.start.elapsed().as_secs_f64();
367        histogram
368            .with_label_values(&self.labels.iter().map(|s| s.as_str()).collect::<Vec<_>>())
369            .observe(duration);
370    }
371}
372
373/// Record an HTTP request
374#[allow(dead_code)]
375pub fn record_http_request(endpoint: &str, method: &str, status: u16) {
376    HTTP_REQUESTS_TOTAL
377        .with_label_values(&[endpoint, method, &status.to_string()])
378        .inc();
379}
380
381/// Start timing an HTTP request
382#[allow(dead_code)]
383pub fn start_http_request_timer(endpoint: &str, method: &str) -> Timer {
384    HTTP_CONNECTIONS_ACTIVE.with_label_values(&[endpoint]).inc();
385    Timer::new(vec![endpoint.to_string(), method.to_string()])
386}
387
388/// Finish timing an HTTP request
389#[allow(dead_code)]
390pub fn finish_http_request_timer(timer: Timer, endpoint: &str) {
391    timer.observe_duration(&HTTP_REQUEST_DURATION_SECONDS);
392    HTTP_CONNECTIONS_ACTIVE.with_label_values(&[endpoint]).dec();
393}
394
395/// Record HTTP request size
396#[allow(dead_code)]
397pub fn record_http_request_size(endpoint: &str, method: &str, size: usize) {
398    HTTP_REQUEST_SIZE_BYTES
399        .with_label_values(&[endpoint, method])
400        .observe(size as f64);
401}
402
403/// Record HTTP response size
404#[allow(dead_code)]
405pub fn record_http_response_size(endpoint: &str, method: &str, size: usize) {
406    HTTP_RESPONSE_SIZE_BYTES
407        .with_label_values(&[endpoint, method])
408        .observe(size as f64);
409}
410
411/// Record block retrieval
412#[allow(dead_code)]
413pub fn record_block_retrieved(source: &str) {
414    BLOCKS_RETRIEVED_TOTAL.with_label_values(&[source]).inc();
415}
416
417/// Record block storage
418#[allow(dead_code)]
419pub fn record_block_stored(destination: &str) {
420    BLOCKS_STORED_TOTAL.with_label_values(&[destination]).inc();
421}
422
423/// Record block error
424#[allow(dead_code)]
425pub fn record_block_error(operation: &str, error_type: &str) {
426    BLOCK_ERRORS_TOTAL
427        .with_label_values(&[operation, error_type])
428        .inc();
429}
430
431/// Record upload bytes
432#[allow(dead_code)]
433pub fn record_upload_bytes(endpoint: &str, bytes: u64) {
434    UPLOAD_BYTES_TOTAL
435        .with_label_values(&[endpoint])
436        .inc_by(bytes as f64);
437}
438
439/// Record download bytes
440#[allow(dead_code)]
441pub fn record_download_bytes(endpoint: &str, bytes: u64) {
442    DOWNLOAD_BYTES_TOTAL
443        .with_label_values(&[endpoint])
444        .inc_by(bytes as f64);
445}
446
447/// Record cache hit
448#[allow(dead_code)]
449pub fn record_cache_hit(cache_type: &str) {
450    CACHE_HITS_TOTAL.with_label_values(&[cache_type]).inc();
451}
452
453/// Record cache miss
454#[allow(dead_code)]
455pub fn record_cache_miss(cache_type: &str) {
456    CACHE_MISSES_TOTAL.with_label_values(&[cache_type]).inc();
457}
458
459/// Record authentication attempt
460#[allow(dead_code)]
461pub fn record_auth_attempt(method: &str, result: &str) {
462    AUTH_ATTEMPTS_TOTAL
463        .with_label_values(&[method, result])
464        .inc();
465}
466
467/// Record rate limit hit
468#[allow(dead_code)]
469pub fn record_rate_limit_hit(endpoint: &str, client_ip: &str) {
470    RATE_LIMIT_HITS_TOTAL
471        .with_label_values(&[endpoint, client_ip])
472        .inc();
473}
474
475// ============================================================================
476// IpfrsMetrics — per-node registry
477// ============================================================================
478
479/// Global metrics registry for IPFRS.
480///
481/// Each field is a Prometheus primitive registered in a **private** registry
482/// so that multiple instances (e.g., in tests) do not conflict with each other.
483///
484/// ```rust
485/// use ipfrs_interface::metrics::IpfrsMetrics;
486///
487/// let m = IpfrsMetrics::new().expect("failed to create metrics registry");
488/// m.blocks_added.inc();
489/// let text = m.render();
490/// assert!(text.contains("ipfrs_blocks_added_total"));
491/// ```
492pub struct IpfrsMetrics {
493    /// Private registry — metrics exported via `render()`.
494    pub registry: Registry,
495
496    // ----- Block operations --------------------------------------------------
497    /// Total blocks added to local storage.
498    pub blocks_added: Counter,
499    /// Total blocks fetched from local storage or network.
500    pub blocks_fetched: Counter,
501    /// Total blocks deleted.
502    pub blocks_deleted: Counter,
503    /// Total bytes written via `add_bytes` / `add_file`.
504    pub block_add_bytes: Counter,
505    /// Latency (seconds) for block fetch operations.
506    pub block_fetch_latency: Histogram,
507    /// Rolling cache hit-rate (0.0 – 1.0).
508    pub cache_hit_rate: Gauge,
509
510    // ----- DHT ---------------------------------------------------------------
511    /// Number of DHT `provide` calls.
512    pub dht_provide_calls: Counter,
513    /// Number of DHT `find_providers` calls.
514    pub dht_find_providers_calls: Counter,
515    /// Current number of provider records held by this node.
516    pub dht_provider_records: Gauge,
517
518    // ----- Inference sessions ------------------------------------------------
519    /// Inference sessions started.
520    pub inference_sessions_started: Counter,
521    /// Inference sessions completed successfully.
522    pub inference_sessions_completed: Counter,
523    /// End-to-end latency (seconds) for completed inference sessions.
524    pub inference_session_latency: Histogram,
525    /// Depth of proof trees emitted during inference.
526    pub proof_tree_depth: Histogram,
527
528    // ----- GossipSub ---------------------------------------------------------
529    /// Messages published to GossipSub topics.
530    pub gossipsub_messages_sent: Counter,
531    /// Messages received from GossipSub topics.
532    pub gossipsub_messages_received: Counter,
533    /// Current number of mesh peers across all topics.
534    pub gossipsub_mesh_peers: Gauge,
535
536    // ----- Storage / GC ------------------------------------------------------
537    /// Total bytes currently stored on disk (updated periodically).
538    pub storage_bytes_total: Gauge,
539    /// Total number of blocks currently stored on disk.
540    pub storage_blocks_total: Gauge,
541    /// Number of GC runs completed.
542    pub gc_runs: Counter,
543    /// Total blocks collected (deleted) across all GC runs.
544    pub gc_blocks_collected: Counter,
545}
546
547impl IpfrsMetrics {
548    /// Create a new metrics registry with all counters, gauges, and histograms
549    /// initialised to zero.
550    ///
551    /// # Errors
552    ///
553    /// Returns a [`prometheus::Error`] if any metric registration fails (which
554    /// should only happen if two metrics share the same name).
555    pub fn new() -> Result<Self, prometheus::Error> {
556        let registry = Registry::new();
557
558        // ---- Block operations ----
559        let blocks_added = Counter::with_opts(Opts::new(
560            "ipfrs_blocks_added_total",
561            "Total number of blocks added to local storage",
562        ))?;
563        registry.register(Box::new(blocks_added.clone()))?;
564
565        let blocks_fetched = Counter::with_opts(Opts::new(
566            "ipfrs_blocks_fetched_total",
567            "Total number of blocks fetched from storage or network",
568        ))?;
569        registry.register(Box::new(blocks_fetched.clone()))?;
570
571        let blocks_deleted = Counter::with_opts(Opts::new(
572            "ipfrs_blocks_deleted_total",
573            "Total number of blocks deleted from local storage",
574        ))?;
575        registry.register(Box::new(blocks_deleted.clone()))?;
576
577        let block_add_bytes = Counter::with_opts(Opts::new(
578            "ipfrs_block_add_bytes_total",
579            "Total bytes written to local storage via add operations",
580        ))?;
581        registry.register(Box::new(block_add_bytes.clone()))?;
582
583        let block_fetch_latency = Histogram::with_opts(
584            HistogramOpts::new(
585                "ipfrs_block_fetch_latency_seconds",
586                "Latency of block fetch operations in seconds",
587            )
588            .buckets(vec![
589                0.0001, 0.0005, 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5,
590            ]),
591        )?;
592        registry.register(Box::new(block_fetch_latency.clone()))?;
593
594        let cache_hit_rate = Gauge::with_opts(Opts::new(
595            "ipfrs_cache_hit_rate",
596            "Rolling cache hit rate in the range [0.0, 1.0]",
597        ))?;
598        registry.register(Box::new(cache_hit_rate.clone()))?;
599
600        // ---- DHT ----
601        let dht_provide_calls = Counter::with_opts(Opts::new(
602            "ipfrs_dht_provide_calls_total",
603            "Total number of DHT provide calls",
604        ))?;
605        registry.register(Box::new(dht_provide_calls.clone()))?;
606
607        let dht_find_providers_calls = Counter::with_opts(Opts::new(
608            "ipfrs_dht_find_providers_calls_total",
609            "Total number of DHT find_providers calls",
610        ))?;
611        registry.register(Box::new(dht_find_providers_calls.clone()))?;
612
613        let dht_provider_records = Gauge::with_opts(Opts::new(
614            "ipfrs_dht_provider_records",
615            "Current number of DHT provider records held by this node",
616        ))?;
617        registry.register(Box::new(dht_provider_records.clone()))?;
618
619        // ---- Inference sessions ----
620        let inference_sessions_started = Counter::with_opts(Opts::new(
621            "ipfrs_inference_sessions_started_total",
622            "Total number of inference sessions started",
623        ))?;
624        registry.register(Box::new(inference_sessions_started.clone()))?;
625
626        let inference_sessions_completed = Counter::with_opts(Opts::new(
627            "ipfrs_inference_sessions_completed_total",
628            "Total number of inference sessions completed successfully",
629        ))?;
630        registry.register(Box::new(inference_sessions_completed.clone()))?;
631
632        let inference_session_latency = Histogram::with_opts(
633            HistogramOpts::new(
634                "ipfrs_inference_session_latency_seconds",
635                "End-to-end latency of completed inference sessions in seconds",
636            )
637            .buckets(vec![0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0]),
638        )?;
639        registry.register(Box::new(inference_session_latency.clone()))?;
640
641        let proof_tree_depth = Histogram::with_opts(
642            HistogramOpts::new(
643                "ipfrs_proof_tree_depth",
644                "Depth of proof trees emitted during inference",
645            )
646            .buckets(vec![1.0, 2.0, 3.0, 5.0, 8.0, 13.0, 21.0, 34.0, 55.0]),
647        )?;
648        registry.register(Box::new(proof_tree_depth.clone()))?;
649
650        // ---- GossipSub ----
651        let gossipsub_messages_sent = Counter::with_opts(Opts::new(
652            "ipfrs_gossipsub_messages_sent_total",
653            "Total number of messages published to GossipSub topics",
654        ))?;
655        registry.register(Box::new(gossipsub_messages_sent.clone()))?;
656
657        let gossipsub_messages_received = Counter::with_opts(Opts::new(
658            "ipfrs_gossipsub_messages_received_total",
659            "Total number of messages received from GossipSub topics",
660        ))?;
661        registry.register(Box::new(gossipsub_messages_received.clone()))?;
662
663        let gossipsub_mesh_peers = Gauge::with_opts(Opts::new(
664            "ipfrs_gossipsub_mesh_peers",
665            "Current number of GossipSub mesh peers across all topics",
666        ))?;
667        registry.register(Box::new(gossipsub_mesh_peers.clone()))?;
668
669        // ---- Storage / GC ----
670        let storage_bytes_total = Gauge::with_opts(Opts::new(
671            "ipfrs_storage_bytes_total",
672            "Total bytes currently stored on disk",
673        ))?;
674        registry.register(Box::new(storage_bytes_total.clone()))?;
675
676        let storage_blocks_total = Gauge::with_opts(Opts::new(
677            "ipfrs_storage_blocks_total",
678            "Total number of blocks currently stored on disk",
679        ))?;
680        registry.register(Box::new(storage_blocks_total.clone()))?;
681
682        let gc_runs = Counter::with_opts(Opts::new(
683            "ipfrs_gc_runs_total",
684            "Total number of garbage collection runs completed",
685        ))?;
686        registry.register(Box::new(gc_runs.clone()))?;
687
688        let gc_blocks_collected = Counter::with_opts(Opts::new(
689            "ipfrs_gc_blocks_collected_total",
690            "Total number of blocks collected (deleted) across all GC runs",
691        ))?;
692        registry.register(Box::new(gc_blocks_collected.clone()))?;
693
694        Ok(Self {
695            registry,
696            blocks_added,
697            blocks_fetched,
698            blocks_deleted,
699            block_add_bytes,
700            block_fetch_latency,
701            cache_hit_rate,
702            dht_provide_calls,
703            dht_find_providers_calls,
704            dht_provider_records,
705            inference_sessions_started,
706            inference_sessions_completed,
707            inference_session_latency,
708            proof_tree_depth,
709            gossipsub_messages_sent,
710            gossipsub_messages_received,
711            gossipsub_mesh_peers,
712            storage_bytes_total,
713            storage_blocks_total,
714            gc_runs,
715            gc_blocks_collected,
716        })
717    }
718
719    /// Render all registered metrics in Prometheus text exposition format.
720    ///
721    /// Returns an empty string if encoding fails.
722    pub fn render(&self) -> String {
723        let encoder = TextEncoder::new();
724        let metric_families = self.registry.gather();
725        let mut buffer = Vec::new();
726        if encoder.encode(&metric_families, &mut buffer).is_err() {
727            return String::new();
728        }
729        String::from_utf8(buffer).unwrap_or_default()
730    }
731}
732
733impl Default for IpfrsMetrics {
734    fn default() -> Self {
735        Self::new().expect("IpfrsMetrics::default() failed to create registry")
736    }
737}
738
739/// A thread-safe shared handle to an [`IpfrsMetrics`] instance.
740pub type SharedMetrics = Arc<IpfrsMetrics>;
741
742/// Create a new [`SharedMetrics`] instance.
743///
744/// # Errors
745///
746/// Returns a [`prometheus::Error`] if metric registration fails.
747pub fn new_shared_metrics() -> Result<SharedMetrics, prometheus::Error> {
748    IpfrsMetrics::new().map(Arc::new)
749}
750
751// ============================================================================
752// Global encode helper
753// ============================================================================
754
755/// Encode all metrics in Prometheus text format
756pub fn encode_metrics() -> Result<String, prometheus::Error> {
757    let encoder = TextEncoder::new();
758    let metric_families = prometheus::gather();
759    let mut buffer = Vec::new();
760    encoder.encode(&metric_families, &mut buffer)?;
761    String::from_utf8(buffer)
762        .map_err(|e| prometheus::Error::Msg(format!("Failed to encode metrics as UTF-8: {}", e)))
763}
764
765#[cfg(test)]
766mod tests {
767    use super::*;
768
769    #[test]
770    fn test_record_http_request() {
771        record_http_request("/api/v0/add", "POST", 200);
772        let metrics = encode_metrics().expect("test: metrics encoding should succeed");
773        assert!(metrics.contains("ipfrs_http_requests_total"));
774    }
775
776    #[test]
777    fn test_timer() {
778        let timer = Timer::new(vec!["test".to_string(), "GET".to_string()]);
779        std::thread::sleep(std::time::Duration::from_millis(10));
780        timer.observe_duration(&HTTP_REQUEST_DURATION_SECONDS);
781
782        let metrics = encode_metrics().expect("test: metrics encoding should succeed");
783        assert!(metrics.contains("ipfrs_http_request_duration_seconds"));
784    }
785
786    #[test]
787    fn test_record_block_operations() {
788        record_block_retrieved("local");
789        record_block_stored("blockstore");
790        record_block_error("get", "not_found");
791
792        let metrics = encode_metrics().expect("test: metrics encoding should succeed");
793        assert!(metrics.contains("ipfrs_blocks_retrieved_total"));
794        assert!(metrics.contains("ipfrs_blocks_stored_total"));
795        assert!(metrics.contains("ipfrs_block_errors_total"));
796    }
797
798    #[test]
799    fn test_record_cache_operations() {
800        record_cache_hit("block_cache");
801        record_cache_miss("block_cache");
802
803        let metrics = encode_metrics().expect("test: metrics encoding should succeed");
804        assert!(metrics.contains("ipfrs_cache_hits_total"));
805        assert!(metrics.contains("ipfrs_cache_misses_total"));
806    }
807
808    #[test]
809    fn test_encode_metrics() {
810        // Record some metrics to ensure encoder has data
811        record_http_request("/test", "GET", 200);
812        record_block_retrieved("test_store");
813
814        let result = encode_metrics();
815        assert!(result.is_ok());
816
817        let metrics = result.expect("test: encode_metrics should return valid UTF-8 metrics");
818        // Metrics should include at least the recorded ones
819        assert!(
820            metrics.contains("ipfrs_http_requests_total")
821                || metrics.contains("ipfrs_blocks_retrieved_total")
822        );
823    }
824
825    // ---- IpfrsMetrics (private registry) tests ----
826
827    /// All counters, gauges and histograms must start at zero after construction.
828    #[test]
829    fn test_metrics_default_zero() {
830        let m = IpfrsMetrics::new().expect("should create metrics");
831
832        // Counters start at 0
833        assert_eq!(m.blocks_added.get() as u64, 0);
834        assert_eq!(m.blocks_fetched.get() as u64, 0);
835        assert_eq!(m.blocks_deleted.get() as u64, 0);
836        assert_eq!(m.block_add_bytes.get() as u64, 0);
837        assert_eq!(m.dht_provide_calls.get() as u64, 0);
838        assert_eq!(m.dht_find_providers_calls.get() as u64, 0);
839        assert_eq!(m.inference_sessions_started.get() as u64, 0);
840        assert_eq!(m.inference_sessions_completed.get() as u64, 0);
841        assert_eq!(m.gossipsub_messages_sent.get() as u64, 0);
842        assert_eq!(m.gossipsub_messages_received.get() as u64, 0);
843        assert_eq!(m.gc_runs.get() as u64, 0);
844        assert_eq!(m.gc_blocks_collected.get() as u64, 0);
845
846        // Gauges start at 0
847        assert_eq!(m.cache_hit_rate.get(), 0.0_f64);
848        assert_eq!(m.dht_provider_records.get(), 0.0_f64);
849        assert_eq!(m.gossipsub_mesh_peers.get(), 0.0_f64);
850        assert_eq!(m.storage_bytes_total.get(), 0.0_f64);
851        assert_eq!(m.storage_blocks_total.get(), 0.0_f64);
852    }
853
854    /// Incrementing `blocks_added` must be reflected in `render()`.
855    #[test]
856    fn test_metrics_increment() {
857        let m = IpfrsMetrics::new().expect("should create metrics");
858        m.blocks_added.inc();
859        m.blocks_added.inc();
860
861        let text = m.render();
862        // The rendered text must contain the metric name and the value "2"
863        assert!(
864            text.contains("ipfrs_blocks_added_total"),
865            "render() must include blocks_added counter"
866        );
867        assert!(
868            text.contains("ipfrs_blocks_added_total 2"),
869            "render() value should be 2 but got:\n{text}"
870        );
871    }
872
873    /// `render()` must produce valid Prometheus text format output.
874    #[test]
875    fn test_metrics_render_format() {
876        let m = IpfrsMetrics::new().expect("should create metrics");
877        // Touch at least one metric so the output is non-empty.
878        m.blocks_added.inc();
879
880        let text = m.render();
881        assert!(
882            text.starts_with("# HELP"),
883            "render() should start with '# HELP' but got:\n{text}"
884        );
885        assert!(
886            text.contains("ipfrs_"),
887            "render() should contain 'ipfrs_' prefix metrics"
888        );
889    }
890
891    /// Observing a latency value must make histogram sum positive.
892    #[test]
893    fn test_histogram_observe() {
894        let m = IpfrsMetrics::new().expect("should create metrics");
895        m.block_fetch_latency.observe(0.042);
896
897        let text = m.render();
898        assert!(
899            text.contains("ipfrs_block_fetch_latency_seconds_sum"),
900            "render() must include histogram sum"
901        );
902        // The sum should be > 0, encoded as a non-zero float
903        // Find the sum line and verify it is not "0"
904        let sum_line = text
905            .lines()
906            .find(|l| l.contains("ipfrs_block_fetch_latency_seconds_sum"))
907            .unwrap_or("");
908        assert!(
909            !sum_line.ends_with(" 0"),
910            "histogram sum should be non-zero after observe(0.042)"
911        );
912    }
913
914    /// Setting `storage_bytes_total` gauge must be reflected in `render()`.
915    #[test]
916    fn test_gauge_set() {
917        let m = IpfrsMetrics::new().expect("should create metrics");
918        m.storage_bytes_total.set(12345.0);
919
920        let text = m.render();
921        assert!(
922            text.contains("ipfrs_storage_bytes_total"),
923            "render() must include storage_bytes_total gauge"
924        );
925        assert!(
926            text.contains("ipfrs_storage_bytes_total 12345"),
927            "render() gauge value should be 12345 but got:\n{text}"
928        );
929    }
930}