Skip to main content

azums/jobs/
metrics.rs

1use chrono::{DateTime, Utc};
2use serde::Serialize;
3use sqlx::PgPool;
4
5#[derive(Debug, Serialize)]
6/// Point-in-time PostgreSQL queue health and throughput summary.
7/// # Examples
8///
9/// ```rust,no_run
10/// use azums::MetricsRepo;
11/// use sqlx::postgres::PgPoolOptions;
12///
13/// let pool = PgPoolOptions::new()
14///     .connect_lazy("postgres://postgres:postgres@localhost/azums")?;
15/// let metrics = MetricsRepo::new(pool);
16/// let _ = metrics;
17/// # Ok::<(), sqlx::Error>(())
18/// ```
19pub struct Metrics {
20    /// Time at which the snapshot was calculated.
21    pub at: DateTime<Utc>,
22
23    /// Queue represented by the snapshot.
24    pub queue: String,
25    /// Number of queued jobs currently eligible to run.
26    pub runnable_queue_depth: i64,
27
28    // last 60s window
29    /// Completed attempts per second over the latest 60-second window.
30    pub jobs_per_sec: f64,
31    /// Fraction of finished attempts that succeeded.
32    pub success_rate: f64,
33    /// Fraction of started attempts whose number is at least two.
34    pub retry_rate: f64,
35    /// Mean finished-attempt latency in milliseconds.
36    pub mean_latency_ms: f64,
37}
38
39#[derive(Clone)]
40/// PostgreSQL repository for queue-level operational metrics.
41/// # Examples
42///
43/// ```rust,no_run
44/// use azums::MetricsRepo;
45/// use sqlx::postgres::PgPoolOptions;
46///
47/// let pool = PgPoolOptions::new()
48///     .connect_lazy("postgres://postgres:postgres@localhost/azums")?;
49/// let metrics = MetricsRepo::new(pool);
50/// let _ = metrics;
51/// # Ok::<(), sqlx::Error>(())
52/// ```
53pub struct MetricsRepo {
54    pool: PgPool,
55}
56
57/// # Examples
58///
59/// ```rust,no_run
60/// use azums::MetricsRepo;
61/// use sqlx::postgres::PgPoolOptions;
62///
63/// let pool = PgPoolOptions::new()
64///     .connect_lazy("postgres://postgres:postgres@localhost/azums")?;
65/// let metrics = MetricsRepo::new(pool);
66/// let _ = metrics;
67/// # Ok::<(), sqlx::Error>(())
68/// ```
69impl MetricsRepo {
70    /// Creates a metrics repository backed by `pool`.
71    /// # Examples
72    ///
73    /// ```rust,no_run
74    /// use azums::MetricsRepo;
75    /// use sqlx::postgres::PgPoolOptions;
76    ///
77    /// let pool = PgPoolOptions::new()
78    ///     .connect_lazy("postgres://postgres:postgres@localhost/azums")?;
79    /// let metrics = MetricsRepo::new(pool);
80    /// let _ = metrics;
81    /// # Ok::<(), sqlx::Error>(())
82    /// ```
83    pub fn new(pool: PgPool) -> Self {
84        Self { pool }
85    }
86
87    /// Returns one metrics snapshot for every known queue.
88    /// # Examples
89    ///
90    /// ```rust,no_run
91    /// use azums::MetricsRepo;
92    /// use sqlx::postgres::PgPoolOptions;
93    ///
94    /// let pool = PgPoolOptions::new()
95    ///     .connect_lazy("postgres://postgres:postgres@localhost/azums")?;
96    /// let metrics = MetricsRepo::new(pool);
97    /// let _ = metrics;
98    /// # Ok::<(), sqlx::Error>(())
99    /// ```
100    pub async fn snapshot_all(&self) -> anyhow::Result<Vec<Metrics>> {
101        let queues: Vec<String> = sqlx::query_scalar(
102            r#"
103            SELECT DISTINCT queue
104            FROM jobs
105            ORDER BY queue
106            "#,
107        )
108        .fetch_all(&self.pool)
109        .await?;
110
111        let mut out = Vec::with_capacity(queues.len());
112        for queue in queues {
113            out.push(self.snapshot_for_queue(&queue).await?);
114        }
115
116        Ok(out)
117    }
118
119    /// Calculates a metrics snapshot for one queue.
120    /// # Examples
121    ///
122    /// ```rust,no_run
123    /// use azums::MetricsRepo;
124    /// use sqlx::postgres::PgPoolOptions;
125    ///
126    /// let pool = PgPoolOptions::new()
127    ///     .connect_lazy("postgres://postgres:postgres@localhost/azums")?;
128    /// let metrics = MetricsRepo::new(pool);
129    /// let _ = metrics;
130    /// # Ok::<(), sqlx::Error>(())
131    /// ```
132    pub async fn snapshot_for_queue(&self, queue: &str) -> anyhow::Result<Metrics> {
133        // Depth (runnable queued)
134        let depth: i64 = sqlx::query_scalar(
135            r#"
136            SELECT COUNT(*)
137            FROM jobs
138            WHERE queue = $1
139              AND status = 'queued'
140              AND run_at <= now()
141            "#,
142        )
143        .bind(queue)
144        .fetch_one(&self.pool)
145        .await?;
146
147        // Attempts window stats (last 60 seconds)
148        // - throughput ~ attempts finished per sec
149        // - success_rate = succeeded / finished
150        // - retry_rate = attempts with attempt_no >=2 / total attempts started
151        // - mean latency = avg(latency_ms) for finished attempts
152        let row = sqlx::query_as::<
153            _,
154            (
155                Option<f64>,
156                Option<f64>,
157                Option<f64>,
158                Option<f64>,
159                Option<f64>,
160            ),
161        >(
162            r#"
163            WITH a AS (
164              SELECT a.*
165              FROM job_attempts a
166              JOIN jobs j ON j.id = a.job_id AND j.dataset_id = a.dataset_id
167              WHERE j.queue = $1
168                AND a.started_at >= now() - interval '60 seconds'
169            ),
170            finished AS (
171              SELECT *
172              FROM a
173              WHERE finished_at IS NOT NULL
174            )
175            SELECT
176              (SELECT COUNT(*) FROM finished)::float8 AS finished_count,
177              (SELECT COUNT(*) FROM finished WHERE status = 'succeeded')::float8 AS succeeded_count,
178              (SELECT COUNT(*) FROM a WHERE attempt_no >= 2)::float8 AS retry_count,
179              (SELECT COUNT(*) FROM a)::float8 AS started_count,
180              COALESCE((SELECT AVG(latency_ms)::float8 FROM finished), 0.0) AS mean_latency_ms
181            "#,
182        )
183        .bind(queue)
184        .fetch_one(&self.pool)
185        .await?;
186
187        let finished_count = row.0.unwrap_or(0.0);
188        let succeeded_count = row.1.unwrap_or(0.0);
189        let retry_count = row.2.unwrap_or(0.0);
190        let started_count = row.3.unwrap_or(0.0);
191        let mean_latency_ms = row.4.unwrap_or(0.0);
192
193        let jobs_per_sec = finished_count / 60.0;
194
195        let success_rate = if finished_count > 0.0 {
196            succeeded_count / finished_count
197        } else {
198            0.0
199        };
200
201        let retry_rate = if started_count > 0.0 {
202            retry_count / started_count
203        } else {
204            0.0
205        };
206
207        Ok(Metrics {
208            at: Utc::now(),
209            queue: queue.to_string(),
210            runnable_queue_depth: depth,
211            jobs_per_sec,
212            success_rate,
213            retry_rate,
214            mean_latency_ms,
215        })
216    }
217}