everruns_durable/persistence/postgres/
workers.rs1use super::*;
4use crate::persistence::store::{
5 WORKER_META_DRAINING_SINCE, WORKER_META_SELF_CHECKED_AT, WORKER_META_SELF_CHECKS,
6};
7
8#[async_trait]
9impl WorkerRegistry for PostgresWorkflowEventStore {
10 #[instrument(skip(self, worker))]
11 async fn register_worker(&self, worker: WorkerInfo) -> Result<(), StoreError> {
12 let worker_group = worker.worker_group.as_deref().unwrap_or("default");
14
15 sqlx::query(
16 r#"
17 INSERT INTO durable_workers (
18 id, worker_group, activity_types, max_concurrency, current_load,
19 status, started_at, last_heartbeat_at, accepting_tasks, hostname, version,
20 metadata
21 )
22 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
23 ON CONFLICT (id) DO UPDATE SET
24 worker_group = EXCLUDED.worker_group,
25 activity_types = EXCLUDED.activity_types,
26 max_concurrency = EXCLUDED.max_concurrency,
27 current_load = EXCLUDED.current_load,
28 status = EXCLUDED.status,
29 last_heartbeat_at = EXCLUDED.last_heartbeat_at,
30 accepting_tasks = EXCLUDED.accepting_tasks,
31 hostname = EXCLUDED.hostname,
32 version = EXCLUDED.version,
33 -- A re-registered worker starts fresh: no drain, no old report.
34 metadata = EXCLUDED.metadata
35 "#,
36 )
37 .bind(&worker.id)
38 .bind(worker_group)
39 .bind(&worker.activity_types)
40 .bind(worker.max_concurrency as i32)
41 .bind(worker.current_load as i32)
42 .bind(&worker.status)
43 .bind(worker.started_at)
44 .bind(worker.last_heartbeat_at)
45 .bind(worker.accepting_tasks)
46 .bind(&worker.hostname)
47 .bind(&worker.version)
48 .bind(&worker.metadata)
49 .execute(&self.pool)
50 .await
51 .map_err(|e| {
52 error!(error = %e, "Failed to register worker");
53 StoreError::Database(e.to_string())
54 })?;
55
56 debug!(worker_id = %worker.id, "registered worker");
57 Ok(())
58 }
59
60 #[instrument(skip(self))]
61 async fn worker_heartbeat(
62 &self,
63 worker_id: &str,
64 current_load: usize,
65 accepting_tasks: bool,
66 ) -> Result<WorkerHeartbeat, StoreError> {
67 let draining: Option<bool> = sqlx::query_scalar(
70 r#"
71 UPDATE durable_workers
72 SET last_heartbeat_at = NOW(),
73 current_load = $2,
74 accepting_tasks = CASE
75 WHEN status = 'draining' THEN false
76 ELSE $3
77 END
78 WHERE id = $1
79 RETURNING status = 'draining'
80 "#,
81 )
82 .bind(worker_id)
83 .bind(current_load as i32)
84 .bind(accepting_tasks)
85 .fetch_optional(&self.pool)
86 .await
87 .map_err(|e| {
88 store_failure(
89 "durable.workers.heartbeat",
90 "Failed to update worker heartbeat",
91 e,
92 )
93 })?;
94
95 Ok(WorkerHeartbeat {
96 draining: draining.unwrap_or(false),
97 })
98 }
99
100 #[instrument(skip(self))]
101 async fn list_workers(&self, filter: WorkerFilter) -> Result<Vec<WorkerInfo>, StoreError> {
102 let heartbeat_threshold =
105 Utc::now() - chrono::Duration::seconds(WORKER_HEARTBEAT_TIMEOUT_SECS);
106
107 let query = r#"
108 SELECT w.id, w.worker_group, w.activity_types, w.max_concurrency, w.current_load,
109 w.status, w.started_at, w.last_heartbeat_at, w.accepting_tasks,
110 w.backpressure_reason, w.hostname, w.version, w.metadata,
111 COALESCE(stats.tasks_completed, 0) AS tasks_completed,
112 COALESCE(stats.tasks_failed, 0) AS tasks_failed,
113 stats.avg_task_duration_ms
114 FROM durable_workers w
115 LEFT JOIN (
116 SELECT claimed_by,
117 COUNT(*) FILTER (WHERE status = 'completed') AS tasks_completed,
118 COUNT(*) FILTER (WHERE status IN ('failed', 'dead')) AS tasks_failed,
119 (AVG(EXTRACT(EPOCH FROM (heartbeat_at - claimed_at)) * 1000)
120 FILTER (WHERE status = 'completed' AND claimed_at IS NOT NULL AND heartbeat_at IS NOT NULL))::FLOAT8
121 AS avg_task_duration_ms
122 FROM durable_task_queue
123 -- Aggregate only the live workers' tasks, not the whole queue
124 -- (1.1 s per dashboard tick in production, EVERRUNS-2F).
125 WHERE claimed_by IN (
126 SELECT id FROM durable_workers WHERE last_heartbeat_at > $1
127 )
128 GROUP BY claimed_by
129 ) stats ON stats.claimed_by = w.id
130 WHERE w.last_heartbeat_at > $1
131 AND ($2::text IS NULL OR w.status = $2)
132 AND ($3::text IS NULL OR w.worker_group = $3)
133 "#;
134
135 let rows = sqlx::query(query)
136 .bind(heartbeat_threshold)
137 .bind(&filter.status)
138 .bind(&filter.worker_group)
139 .fetch_all(&self.pool)
140 .await
141 .map_err(|e| {
142 error!(error = %e, "Failed to list workers");
143 StoreError::Database(e.to_string())
144 })?;
145
146 let workers = rows
147 .into_iter()
148 .map(|row| WorkerInfo {
149 id: row.get("id"),
150 worker_group: row.get("worker_group"),
151 activity_types: row.get("activity_types"),
152 max_concurrency: row.get::<i32, _>("max_concurrency") as u32,
153 current_load: row.get::<i32, _>("current_load") as u32,
154 status: row.get("status"),
155 accepting_tasks: row.get("accepting_tasks"),
156 backpressure_reason: row.get("backpressure_reason"),
157 started_at: row.get("started_at"),
158 last_heartbeat_at: row.get("last_heartbeat_at"),
159 hostname: row.get("hostname"),
160 version: row.get("version"),
161 metadata: row.get("metadata"),
162 tasks_completed: row.get::<i64, _>("tasks_completed") as u64,
163 tasks_failed: row.get::<i64, _>("tasks_failed") as u64,
164 avg_task_duration_ms: row
165 .get::<Option<f64>, _>("avg_task_duration_ms")
166 .map(|v| v as u64),
167 })
168 .collect();
169
170 Ok(workers)
171 }
172
173 #[instrument(skip(self))]
174 async fn deregister_worker(&self, worker_id: &str) -> Result<usize, StoreError> {
175 let reclaimed = sqlx::query(
178 r#"
179 UPDATE durable_task_queue
180 SET status = 'pending',
181 claimed_by = NULL,
182 claimed_at = NULL
183 WHERE status = 'claimed'
184 AND claimed_by = $1
185 "#,
186 )
187 .bind(worker_id)
188 .execute(&self.pool)
189 .await
190 .map_err(|e| {
191 error!(error = %e, "Failed to reclaim worker tasks");
192 StoreError::Database(e.to_string())
193 })?
194 .rows_affected() as usize;
195
196 sqlx::query(
198 r#"
199 UPDATE durable_workers
200 SET status = 'stopped'
201 WHERE id = $1
202 "#,
203 )
204 .bind(worker_id)
205 .execute(&self.pool)
206 .await
207 .map_err(|e| {
208 error!(error = %e, "Failed to deregister worker");
209 StoreError::Database(e.to_string())
210 })?;
211
212 if reclaimed > 0 {
213 info!(
214 worker_id,
215 reclaimed, "deregistered worker and reclaimed tasks"
216 );
217 } else {
218 debug!(worker_id, "deregistered worker (no tasks to reclaim)");
219 }
220
221 Ok(reclaimed)
222 }
223
224 #[instrument(skip(self))]
225 async fn get_capacity_snapshot(&self) -> Result<CapacitySnapshot, StoreError> {
226 let row = sqlx::query(
227 r#"
228 SELECT
229 COALESCE(SUM(max_concurrency - current_load), 0)::INT AS total_available,
230 COUNT(*)::INT AS active_workers
231 FROM durable_workers
232 WHERE status = 'active'
233 AND accepting_tasks = true
234 AND last_heartbeat_at > NOW() - INTERVAL '30 seconds'
235 "#,
236 )
237 .fetch_one(&self.pool)
238 .await
239 .map_err(|e| {
240 error!(error = %e, "Failed to get capacity snapshot");
241 StoreError::Database(e.to_string())
242 })?;
243
244 Ok(CapacitySnapshot {
245 total_available: row.get::<i32, _>("total_available") as u32,
246 active_workers: row.get::<i32, _>("active_workers") as u32,
247 })
248 }
249
250 #[instrument(skip(self))]
251 async fn drain_worker(&self, worker_id: &str) -> Result<(), StoreError> {
252 sqlx::query(
253 r#"
254 UPDATE durable_workers
255 SET metadata = CASE
256 WHEN status = 'draining' THEN metadata
257 ELSE COALESCE(metadata, '{}'::jsonb) || jsonb_build_object($2::text, NOW())
258 END,
259 status = 'draining',
260 accepting_tasks = false
261 WHERE id = $1
262 "#,
263 )
264 .bind(worker_id)
265 .bind(WORKER_META_DRAINING_SINCE)
266 .execute(&self.pool)
267 .await
268 .map_err(|e| {
269 error!(error = %e, "Failed to drain worker");
270 StoreError::Database(e.to_string())
271 })?;
272
273 debug!(worker_id, "drained worker");
274 Ok(())
275 }
276
277 #[instrument(skip(self))]
278 async fn resume_worker(&self, worker_id: &str) -> Result<(), StoreError> {
279 sqlx::query(
280 r#"
281 UPDATE durable_workers
282 SET status = 'active',
283 accepting_tasks = true,
284 metadata = metadata - $2::text
285 WHERE id = $1 AND status = 'draining'
286 "#,
287 )
288 .bind(worker_id)
289 .bind(WORKER_META_DRAINING_SINCE)
290 .execute(&self.pool)
291 .await
292 .map_err(|e| {
293 error!(error = %e, "Failed to resume worker");
294 StoreError::Database(e.to_string())
295 })?;
296
297 debug!(worker_id, "resumed worker");
298 Ok(())
299 }
300
301 #[instrument(skip(self, checks))]
302 async fn record_worker_checks(
303 &self,
304 worker_id: &str,
305 checks: serde_json::Value,
306 ) -> Result<(), StoreError> {
307 sqlx::query(
308 r#"
309 UPDATE durable_workers
310 SET metadata = COALESCE(metadata, '{}'::jsonb)
311 || jsonb_build_object($2::text, $3::jsonb, $4::text, NOW())
312 WHERE id = $1
313 "#,
314 )
315 .bind(worker_id)
316 .bind(WORKER_META_SELF_CHECKS)
317 .bind(checks)
318 .bind(WORKER_META_SELF_CHECKED_AT)
319 .execute(&self.pool)
320 .await
321 .map_err(|e| {
322 error!(error = %e, "Failed to record worker checks");
323 StoreError::Database(e.to_string())
324 })?;
325 Ok(())
326 }
327}