pg_exporter 0.11.1

PostgreSQL metric exporter for Prometheus
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
use anyhow::Result;
use prometheus::{CounterVec, GaugeVec, HistogramVec, IntGauge, Opts, Registry};
use std::time::Instant;

/// Tracks scrape performance and metrics cardinality
///
/// This collector monitors the health and performance of all other collectors,
/// helping operators identify slow collectors, detect failures, and track
/// metric cardinality (critical for Cortex/Mimir with strict limits).
///
/// # Metrics Exported
///
/// ## Per-Collector Performance
///
/// - `pg_exporter_collector_scrape_duration_seconds{collector}` (Histogram)
///   - Time spent scraping each collector
///   - Buckets: 1ms, 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 2.5s, 5s
///   - Use `histogram_quantile()` for percentiles (p50, p95, p99)
///   - Example: `histogram_quantile(0.99, rate(pg_exporter_collector_scrape_duration_seconds_bucket[5m]))`
///
/// - `pg_exporter_collector_scrape_errors_total{collector}` (Counter)
///   - Total errors per collector since start
///   - Alert if rate > 0: collector is failing
///   - Example: `rate(pg_exporter_collector_scrape_errors_total[5m]) > 0`
///
/// - `pg_exporter_collector_last_scrape_timestamp_seconds{collector}` (Gauge)
///   - Unix timestamp of last scrape attempt
///   - Detect stale collectors (stuck or disabled)
///   - Example: `time() - pg_exporter_collector_last_scrape_timestamp_seconds > 120`
///
/// - `pg_exporter_collector_last_scrape_success{collector}` (Gauge)
///   - 1 = last scrape succeeded, 0 = failed
///   - Simple success/failure indicator per collector
///
/// ## Global Metrics
///
/// - `pg_exporter_metrics_total` (`IntGauge`)
///   - **Total active time series / cardinality currently exported**
///   - Matches: `curl -s 0:9432/metrics | grep -vEc '^(#|\s*$)'`
///   - Counts non-comment, non-empty metric lines
///   - Critical for Cortex/Mimir operators with series limits
///   - Alert if approaching your cardinality limit
///   - Example: `pg_exporter_metrics_total > 10000`
///
/// - `pg_exporter_scrapes_total` (`IntGauge`)
///   - Total scrapes performed since start
///   - Used to detect if exporter is active
///
/// # Usage Pattern with `ScrapeTimer`
///
/// The `ScrapeTimer` is an RAII (Resource Acquisition Is Initialization) timer
/// that automatically records scrape duration and status when dropped:
///
/// ```no_run
/// # use pg_exporter::collectors::exporter::ScraperCollector;
/// # use anyhow::Result;
/// # async fn example() -> Result<()> {
/// let scraper = ScraperCollector::new();
///
/// // Start timing a collector scrape
/// let timer = scraper.start_scrape("database");
///
/// // Simulate collector work
/// match collect_database_metrics().await {
///     Ok(_) => timer.success(),  // Records duration, marks success
///     Err(e) => timer.error(),   // Records error, marks failure
/// }
///
/// // If timer is dropped without calling success()/error(),
/// // it defaults to success (optimistic)
/// # Ok(())
/// # }
/// # async fn collect_database_metrics() -> Result<()> { Ok(()) }
/// ```
///
/// # Thread Safety
///
/// The collector updates `prometheus` metric types directly. Their internal
/// synchronization is sufficient here, so scrape bookkeeping does not need an
/// extra lock in the exporter hot path.
///
/// # Example `Prometheus` Queries
///
/// ```promql
/// # Slowest collector (p99 latency)
/// topk(5,
///   histogram_quantile(0.99,
///     rate(pg_exporter_collector_scrape_duration_seconds_bucket[5m])
///   )
/// ) by (collector)
///
/// # Failed collectors
/// sum by (collector) (
///   rate(pg_exporter_collector_scrape_errors_total[5m])
/// ) > 0
///
/// # Metric cardinality trend
/// delta(pg_exporter_metrics_total[1h])
/// ```
#[derive(Clone)]
pub struct ScraperCollector {
    // Per-collector metrics
    scrape_duration_seconds: HistogramVec,
    scrape_errors_total: CounterVec,
    last_scrape_timestamp: GaugeVec,
    last_scrape_success: GaugeVec,

    // Global metrics
    metrics_total: IntGauge,
    scrapes_total: IntGauge,
}

impl Default for ScraperCollector {
    fn default() -> Self {
        Self::new()
    }
}

impl ScraperCollector {
    /// Creates a new `ScraperCollector`
    ///
    /// # Panics
    ///
    /// Panics if metric creation fails (should never happen with valid metric names)
    #[must_use]
    #[allow(clippy::expect_used)]
    pub fn new() -> Self {
        let scrape_duration_seconds = HistogramVec::new(
            prometheus::HistogramOpts::new(
                "pg_exporter_collector_scrape_duration_seconds",
                "Time spent scraping each collector in seconds",
            )
            .buckets(vec![0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0]),
            &["collector"],
        )
        .expect("pg_exporter_collector_scrape_duration_seconds");

        let scrape_errors_total = CounterVec::new(
            Opts::new(
                "pg_exporter_collector_scrape_errors_total",
                "Total number of scrape errors per collector",
            ),
            &["collector"],
        )
        .expect("pg_exporter_collector_scrape_errors_total");

        let last_scrape_timestamp = GaugeVec::new(
            Opts::new(
                "pg_exporter_collector_last_scrape_timestamp_seconds",
                "Unix timestamp of the last scrape attempt per collector",
            ),
            &["collector"],
        )
        .expect("pg_exporter_collector_last_scrape_timestamp_seconds");

        let last_scrape_success = GaugeVec::new(
            Opts::new(
                "pg_exporter_collector_last_scrape_success",
                "Whether the last scrape was successful (1=success, 0=failure)",
            ),
            &["collector"],
        )
        .expect("pg_exporter_collector_last_scrape_success");

        let metrics_total = IntGauge::with_opts(Opts::new(
            "pg_exporter_metrics_total",
            "Total active time series / cardinality (non-comment, non-empty lines)",
        ))
        .expect("pg_exporter_metrics_total");

        let scrapes_total = IntGauge::with_opts(Opts::new(
            "pg_exporter_scrapes_total",
            "Total number of scrapes performed since start",
        ))
        .expect("pg_exporter_scrapes_total");

        Self {
            scrape_duration_seconds,
            scrape_errors_total,
            last_scrape_timestamp,
            last_scrape_success,
            metrics_total,
            scrapes_total,
        }
    }

    /// Returns the total number of scrapes performed
    #[must_use]
    pub fn scrapes_total(&self) -> i64 {
        self.scrapes_total.get()
    }

    /// Record the start of a collector scrape
    #[must_use]
    pub fn start_scrape(&self, collector_name: &'static str) -> ScrapeTimer {
        ScrapeTimer {
            collector_name,
            start: Instant::now(),
            scraper: self.clone(),
            recorded: false,
        }
    }

    /// Update total metrics count
    /// Call this after each scrape to track cardinality
    pub fn update_metrics_count(&self, count: i64) {
        self.metrics_total.set(count);
    }

    /// Increment total scrapes counter
    pub fn increment_scrapes(&self) {
        self.scrapes_total.inc();
    }

    /// Record a successful scrape
    fn record_success(&self, collector_name: &'static str, duration: f64) {
        let timestamp = std::time::SystemTime::now()
            .duration_since(std::time::UNIX_EPOCH)
            .unwrap_or_default()
            .as_secs_f64();

        self.scrape_duration_seconds
            .with_label_values(&[collector_name])
            .observe(duration);

        self.last_scrape_timestamp
            .with_label_values(&[collector_name])
            .set(timestamp);

        self.last_scrape_success
            .with_label_values(&[collector_name])
            .set(1.0);
    }

    /// Record a failed scrape
    fn record_error(&self, collector_name: &'static str) {
        let timestamp = std::time::SystemTime::now()
            .duration_since(std::time::UNIX_EPOCH)
            .unwrap_or_default()
            .as_secs_f64();

        self.scrape_errors_total
            .with_label_values(&[collector_name])
            .inc();

        self.last_scrape_timestamp
            .with_label_values(&[collector_name])
            .set(timestamp);

        self.last_scrape_success
            .with_label_values(&[collector_name])
            .set(0.0);
    }

    /// Register all metrics with the registry
    ///
    /// # Errors
    ///
    /// Returns an error if any metric fails to register
    pub fn register(&self, registry: &Registry) -> Result<()> {
        registry.register(Box::new(self.scrape_duration_seconds.clone()))?;
        registry.register(Box::new(self.scrape_errors_total.clone()))?;
        registry.register(Box::new(self.last_scrape_timestamp.clone()))?;
        registry.register(Box::new(self.last_scrape_success.clone()))?;
        registry.register(Box::new(self.metrics_total.clone()))?;
        registry.register(Box::new(self.scrapes_total.clone()))?;
        Ok(())
    }
}

impl crate::collectors::Collector for ScraperCollector {
    fn name(&self) -> &'static str {
        "scraper"
    }

    fn register_metrics(&self, registry: &Registry) -> Result<()> {
        self.register(registry)
    }

    fn collect<'a>(&'a self, _pool: &'a sqlx::PgPool) -> futures::future::BoxFuture<'a, Result<()>> {
        // ScraperCollector doesn't scrape from PostgreSQL
        // It's updated by other collectors via start_scrape(), update_metrics_count(), etc.
        Box::pin(async move { Ok(()) })
    }

    fn enabled_by_default(&self) -> bool {
        false
    }
}

/// RAII timer for recording scrape duration
///
/// Automatically records duration and success/failure on drop
pub struct ScrapeTimer {
    collector_name: &'static str,
    start: Instant,
    scraper: ScraperCollector,
    recorded: bool,
}

impl ScrapeTimer {
    /// Mark scrape as successful
    /// Call this before timer drops if scrape succeeded
    pub fn success(mut self) {
        self.recorded = true;
        let duration = self.start.elapsed().as_secs_f64();
        self.scraper.record_success(self.collector_name, duration);
    }

    /// Mark scrape as failed
    /// Call this before timer drops if scrape failed
    pub fn error(mut self) {
        self.recorded = true;
        self.scraper.record_error(self.collector_name);
    }
}

impl Drop for ScrapeTimer {
    fn drop(&mut self) {
        if self.recorded {
            return;
        }
        // If neither success() nor error() was called explicitly,
        // default to success (optimistic)
        let duration = self.start.elapsed().as_secs_f64();
        self.scraper.record_success(self.collector_name, duration);
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::thread;
    use std::time::Duration;

    #[test]
    #[allow(clippy::unwrap_used)]
    fn test_scraper_collector_new() {
        let scraper = ScraperCollector::new();
        assert_eq!(scraper.metrics_total.get(), 0);
        assert_eq!(scraper.scrapes_total.get(), 0);
    }

    #[test]
    #[allow(clippy::unwrap_used)]
    fn test_scraper_collector_registers_without_error() {
        let scraper = ScraperCollector::new();
        let registry = Registry::new();
        assert!(scraper.register(&registry).is_ok());
    }

    #[test]
    #[allow(clippy::unwrap_used)]
    #[allow(clippy::expect_used)]
    fn test_scrape_timer_records_duration() {
        let scraper = ScraperCollector::new();
        let registry = Registry::new();
        scraper.register(&registry).unwrap();

        {
            let timer = scraper.start_scrape("test_collector");
            thread::sleep(Duration::from_millis(10));
            timer.success();
        }

        // Check that metrics were recorded
        let metrics = registry.gather();
        let duration_metric = metrics
            .iter()
            .find(|m| m.name() == "pg_exporter_collector_scrape_duration_seconds")
            .expect("duration metric should exist");

        let metric = duration_metric.get_metric().first().expect("metric should have at least one sample");
        assert_eq!(
            metric.get_histogram().get_sample_count(),
            1,
            "Should record exactly one sample"
        );
    }

    #[test]
    #[allow(clippy::unwrap_used)]
    #[allow(clippy::expect_used)]
    fn test_scrape_timer_records_error() {
        let scraper = ScraperCollector::new();
        let registry = Registry::new();
        scraper.register(&registry).unwrap();

        {
            let timer = scraper.start_scrape("test_collector");
            timer.error();
        }

        // Check that error was recorded
        let metrics = registry.gather();
        let error_metric = metrics
            .iter()
            .find(|m| m.name() == "pg_exporter_collector_scrape_errors_total")
            .expect("error metric should exist");

        let metric = error_metric.get_metric().first().expect("metric should have at least one sample");
        assert!((metric.get_counter().value() - 1.0).abs() < f64::EPSILON, "Should record exactly one error");

        // Verify NO success duration was recorded (bug fix check)
        let duration_metric = metrics
            .iter()
            .find(|m| m.name() == "pg_exporter_collector_scrape_duration_seconds");

        if let Some(m) = duration_metric {
            let metrics = m.get_metric();
            if !metrics.is_empty() {
                assert_eq!(
                    metrics.first().expect("metric should have at least one sample").get_histogram().get_sample_count(),
                    0,
                    "Should not record duration on error"
                );
            }
        }
    }

    #[test]
    fn test_update_metrics_count() {
        let scraper = ScraperCollector::new();
        scraper.update_metrics_count(42);
        assert_eq!(scraper.metrics_total.get(), 42);
    }

    #[test]
    fn test_increment_scrapes() {
        let scraper = ScraperCollector::new();
        scraper.increment_scrapes();
        assert_eq!(scraper.scrapes_total.get(), 1);
        scraper.increment_scrapes();
        assert_eq!(scraper.scrapes_total.get(), 2);
    }
}