azums/jobs/metrics.rs
1use chrono::{DateTime, Utc};
2use serde::Serialize;
3use sqlx::PgPool;
4
5#[derive(Debug, Serialize)]
6/// Point-in-time PostgreSQL queue health and throughput summary.
7/// # Examples
8///
9/// ```rust,no_run
10/// use azums::MetricsRepo;
11/// use sqlx::postgres::PgPoolOptions;
12///
13/// let pool = PgPoolOptions::new()
14/// .connect_lazy("postgres://postgres:postgres@localhost/azums")?;
15/// let metrics = MetricsRepo::new(pool);
16/// let _ = metrics;
17/// # Ok::<(), sqlx::Error>(())
18/// ```
19pub struct Metrics {
20 /// Time at which the snapshot was calculated.
21 pub at: DateTime<Utc>,
22
23 /// Queue represented by the snapshot.
24 pub queue: String,
25 /// Number of queued jobs currently eligible to run.
26 pub runnable_queue_depth: i64,
27
28 // last 60s window
29 /// Completed attempts per second over the latest 60-second window.
30 pub jobs_per_sec: f64,
31 /// Fraction of finished attempts that succeeded.
32 pub success_rate: f64,
33 /// Fraction of started attempts whose number is at least two.
34 pub retry_rate: f64,
35 /// Mean finished-attempt latency in milliseconds.
36 pub mean_latency_ms: f64,
37}
38
39#[derive(Clone)]
40/// PostgreSQL repository for queue-level operational metrics.
41/// # Examples
42///
43/// ```rust,no_run
44/// use azums::MetricsRepo;
45/// use sqlx::postgres::PgPoolOptions;
46///
47/// let pool = PgPoolOptions::new()
48/// .connect_lazy("postgres://postgres:postgres@localhost/azums")?;
49/// let metrics = MetricsRepo::new(pool);
50/// let _ = metrics;
51/// # Ok::<(), sqlx::Error>(())
52/// ```
53pub struct MetricsRepo {
54 pool: PgPool,
55}
56
57/// # Examples
58///
59/// ```rust,no_run
60/// use azums::MetricsRepo;
61/// use sqlx::postgres::PgPoolOptions;
62///
63/// let pool = PgPoolOptions::new()
64/// .connect_lazy("postgres://postgres:postgres@localhost/azums")?;
65/// let metrics = MetricsRepo::new(pool);
66/// let _ = metrics;
67/// # Ok::<(), sqlx::Error>(())
68/// ```
69impl MetricsRepo {
70 /// Creates a metrics repository backed by `pool`.
71 /// # Examples
72 ///
73 /// ```rust,no_run
74 /// use azums::MetricsRepo;
75 /// use sqlx::postgres::PgPoolOptions;
76 ///
77 /// let pool = PgPoolOptions::new()
78 /// .connect_lazy("postgres://postgres:postgres@localhost/azums")?;
79 /// let metrics = MetricsRepo::new(pool);
80 /// let _ = metrics;
81 /// # Ok::<(), sqlx::Error>(())
82 /// ```
83 pub fn new(pool: PgPool) -> Self {
84 Self { pool }
85 }
86
87 /// Returns one metrics snapshot for every known queue.
88 /// # Examples
89 ///
90 /// ```rust,no_run
91 /// use azums::MetricsRepo;
92 /// use sqlx::postgres::PgPoolOptions;
93 ///
94 /// let pool = PgPoolOptions::new()
95 /// .connect_lazy("postgres://postgres:postgres@localhost/azums")?;
96 /// let metrics = MetricsRepo::new(pool);
97 /// let _ = metrics;
98 /// # Ok::<(), sqlx::Error>(())
99 /// ```
100 pub async fn snapshot_all(&self) -> anyhow::Result<Vec<Metrics>> {
101 let queues: Vec<String> = sqlx::query_scalar(
102 r#"
103 SELECT DISTINCT queue
104 FROM jobs
105 ORDER BY queue
106 "#,
107 )
108 .fetch_all(&self.pool)
109 .await?;
110
111 let mut out = Vec::with_capacity(queues.len());
112 for queue in queues {
113 out.push(self.snapshot_for_queue(&queue).await?);
114 }
115
116 Ok(out)
117 }
118
119 /// Calculates a metrics snapshot for one queue.
120 /// # Examples
121 ///
122 /// ```rust,no_run
123 /// use azums::MetricsRepo;
124 /// use sqlx::postgres::PgPoolOptions;
125 ///
126 /// let pool = PgPoolOptions::new()
127 /// .connect_lazy("postgres://postgres:postgres@localhost/azums")?;
128 /// let metrics = MetricsRepo::new(pool);
129 /// let _ = metrics;
130 /// # Ok::<(), sqlx::Error>(())
131 /// ```
132 pub async fn snapshot_for_queue(&self, queue: &str) -> anyhow::Result<Metrics> {
133 // Depth (runnable queued)
134 let depth: i64 = sqlx::query_scalar(
135 r#"
136 SELECT COUNT(*)
137 FROM jobs
138 WHERE queue = $1
139 AND status = 'queued'
140 AND run_at <= now()
141 "#,
142 )
143 .bind(queue)
144 .fetch_one(&self.pool)
145 .await?;
146
147 // Attempts window stats (last 60 seconds)
148 // - throughput ~ attempts finished per sec
149 // - success_rate = succeeded / finished
150 // - retry_rate = attempts with attempt_no >=2 / total attempts started
151 // - mean latency = avg(latency_ms) for finished attempts
152 let row = sqlx::query_as::<
153 _,
154 (
155 Option<f64>,
156 Option<f64>,
157 Option<f64>,
158 Option<f64>,
159 Option<f64>,
160 ),
161 >(
162 r#"
163 WITH a AS (
164 SELECT a.*
165 FROM job_attempts a
166 JOIN jobs j ON j.id = a.job_id AND j.dataset_id = a.dataset_id
167 WHERE j.queue = $1
168 AND a.started_at >= now() - interval '60 seconds'
169 ),
170 finished AS (
171 SELECT *
172 FROM a
173 WHERE finished_at IS NOT NULL
174 )
175 SELECT
176 (SELECT COUNT(*) FROM finished)::float8 AS finished_count,
177 (SELECT COUNT(*) FROM finished WHERE status = 'succeeded')::float8 AS succeeded_count,
178 (SELECT COUNT(*) FROM a WHERE attempt_no >= 2)::float8 AS retry_count,
179 (SELECT COUNT(*) FROM a)::float8 AS started_count,
180 COALESCE((SELECT AVG(latency_ms)::float8 FROM finished), 0.0) AS mean_latency_ms
181 "#,
182 )
183 .bind(queue)
184 .fetch_one(&self.pool)
185 .await?;
186
187 let finished_count = row.0.unwrap_or(0.0);
188 let succeeded_count = row.1.unwrap_or(0.0);
189 let retry_count = row.2.unwrap_or(0.0);
190 let started_count = row.3.unwrap_or(0.0);
191 let mean_latency_ms = row.4.unwrap_or(0.0);
192
193 let jobs_per_sec = finished_count / 60.0;
194
195 let success_rate = if finished_count > 0.0 {
196 succeeded_count / finished_count
197 } else {
198 0.0
199 };
200
201 let retry_rate = if started_count > 0.0 {
202 retry_count / started_count
203 } else {
204 0.0
205 };
206
207 Ok(Metrics {
208 at: Utc::now(),
209 queue: queue.to_string(),
210 runnable_queue_depth: depth,
211 jobs_per_sec,
212 success_rate,
213 retry_rate,
214 mean_latency_ms,
215 })
216 }
217}