Skip to main content

everruns_durable/persistence/postgres/
workers.rs

1//! Implementation of [`crate::WorkerRegistry`].
2
3use super::*;
4use crate::persistence::store::{
5    WORKER_META_DRAINING_SINCE, WORKER_META_SELF_CHECKED_AT, WORKER_META_SELF_CHECKS,
6};
7
8#[async_trait]
9impl WorkerRegistry for PostgresWorkflowEventStore {
10    #[instrument(skip(self, worker))]
11    async fn register_worker(&self, worker: WorkerInfo) -> Result<(), StoreError> {
12        // Default worker_group to "default" if not specified
13        let worker_group = worker.worker_group.as_deref().unwrap_or("default");
14
15        sqlx::query(
16            r#"
17            INSERT INTO durable_workers (
18                id, worker_group, activity_types, max_concurrency, current_load,
19                status, started_at, last_heartbeat_at, accepting_tasks, hostname, version,
20                metadata
21            )
22            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
23            ON CONFLICT (id) DO UPDATE SET
24                worker_group = EXCLUDED.worker_group,
25                activity_types = EXCLUDED.activity_types,
26                max_concurrency = EXCLUDED.max_concurrency,
27                current_load = EXCLUDED.current_load,
28                status = EXCLUDED.status,
29                last_heartbeat_at = EXCLUDED.last_heartbeat_at,
30                accepting_tasks = EXCLUDED.accepting_tasks,
31                hostname = EXCLUDED.hostname,
32                version = EXCLUDED.version,
33                -- A re-registered worker starts fresh: no drain, no old report.
34                metadata = EXCLUDED.metadata
35            "#,
36        )
37        .bind(&worker.id)
38        .bind(worker_group)
39        .bind(&worker.activity_types)
40        .bind(worker.max_concurrency as i32)
41        .bind(worker.current_load as i32)
42        .bind(&worker.status)
43        .bind(worker.started_at)
44        .bind(worker.last_heartbeat_at)
45        .bind(worker.accepting_tasks)
46        .bind(&worker.hostname)
47        .bind(&worker.version)
48        .bind(&worker.metadata)
49        .execute(&self.pool)
50        .await
51        .map_err(|e| {
52            error!(error = %e, "Failed to register worker");
53            StoreError::Database(e.to_string())
54        })?;
55
56        debug!(worker_id = %worker.id, "registered worker");
57        Ok(())
58    }
59
60    #[instrument(skip(self))]
61    async fn worker_heartbeat(
62        &self,
63        worker_id: &str,
64        current_load: usize,
65        accepting_tasks: bool,
66    ) -> Result<WorkerHeartbeat, StoreError> {
67        // Only update accepting_tasks if worker is NOT draining.
68        // When draining, we preserve accepting_tasks = false set by drain_worker.
69        let draining: Option<bool> = sqlx::query_scalar(
70            r#"
71            UPDATE durable_workers
72            SET last_heartbeat_at = NOW(),
73                current_load = $2,
74                accepting_tasks = CASE
75                    WHEN status = 'draining' THEN false
76                    ELSE $3
77                END
78            WHERE id = $1
79            RETURNING status = 'draining'
80            "#,
81        )
82        .bind(worker_id)
83        .bind(current_load as i32)
84        .bind(accepting_tasks)
85        .fetch_optional(&self.pool)
86        .await
87        .map_err(|e| {
88            store_failure(
89                "durable.workers.heartbeat",
90                "Failed to update worker heartbeat",
91                e,
92            )
93        })?;
94
95        Ok(WorkerHeartbeat {
96            draining: draining.unwrap_or(false),
97        })
98    }
99
100    #[instrument(skip(self))]
101    async fn list_workers(&self, filter: WorkerFilter) -> Result<Vec<WorkerInfo>, StoreError> {
102        // Only show workers with recent heartbeat (within WORKER_HEARTBEAT_TIMEOUT_SECS)
103        // Join with task queue to compute completed/failed counts
104        let heartbeat_threshold =
105            Utc::now() - chrono::Duration::seconds(WORKER_HEARTBEAT_TIMEOUT_SECS);
106
107        let query = r#"
108            SELECT w.id, w.worker_group, w.activity_types, w.max_concurrency, w.current_load,
109                   w.status, w.started_at, w.last_heartbeat_at, w.accepting_tasks,
110                   w.backpressure_reason, w.hostname, w.version, w.metadata,
111                   COALESCE(stats.tasks_completed, 0) AS tasks_completed,
112                   COALESCE(stats.tasks_failed, 0) AS tasks_failed,
113                   stats.avg_task_duration_ms
114            FROM durable_workers w
115            LEFT JOIN (
116                SELECT claimed_by,
117                       COUNT(*) FILTER (WHERE status = 'completed') AS tasks_completed,
118                       COUNT(*) FILTER (WHERE status IN ('failed', 'dead')) AS tasks_failed,
119                       (AVG(EXTRACT(EPOCH FROM (heartbeat_at - claimed_at)) * 1000)
120                           FILTER (WHERE status = 'completed' AND claimed_at IS NOT NULL AND heartbeat_at IS NOT NULL))::FLOAT8
121                           AS avg_task_duration_ms
122                FROM durable_task_queue
123                -- Aggregate only the live workers' tasks, not the whole queue
124                -- (1.1 s per dashboard tick in production, EVERRUNS-2F).
125                WHERE claimed_by IN (
126                    SELECT id FROM durable_workers WHERE last_heartbeat_at > $1
127                )
128                GROUP BY claimed_by
129            ) stats ON stats.claimed_by = w.id
130            WHERE w.last_heartbeat_at > $1
131              AND ($2::text IS NULL OR w.status = $2)
132              AND ($3::text IS NULL OR w.worker_group = $3)
133            "#;
134
135        let rows = sqlx::query(query)
136            .bind(heartbeat_threshold)
137            .bind(&filter.status)
138            .bind(&filter.worker_group)
139            .fetch_all(&self.pool)
140            .await
141            .map_err(|e| {
142                error!(error = %e, "Failed to list workers");
143                StoreError::Database(e.to_string())
144            })?;
145
146        let workers = rows
147            .into_iter()
148            .map(|row| WorkerInfo {
149                id: row.get("id"),
150                worker_group: row.get("worker_group"),
151                activity_types: row.get("activity_types"),
152                max_concurrency: row.get::<i32, _>("max_concurrency") as u32,
153                current_load: row.get::<i32, _>("current_load") as u32,
154                status: row.get("status"),
155                accepting_tasks: row.get("accepting_tasks"),
156                backpressure_reason: row.get("backpressure_reason"),
157                started_at: row.get("started_at"),
158                last_heartbeat_at: row.get("last_heartbeat_at"),
159                hostname: row.get("hostname"),
160                version: row.get("version"),
161                metadata: row.get("metadata"),
162                tasks_completed: row.get::<i64, _>("tasks_completed") as u64,
163                tasks_failed: row.get::<i64, _>("tasks_failed") as u64,
164                avg_task_duration_ms: row
165                    .get::<Option<f64>, _>("avg_task_duration_ms")
166                    .map(|v| v as u64),
167            })
168            .collect();
169
170        Ok(workers)
171    }
172
173    #[instrument(skip(self))]
174    async fn deregister_worker(&self, worker_id: &str) -> Result<usize, StoreError> {
175        // Reclaim all tasks claimed by this worker (set back to pending)
176        // This allows immediate task reassignment instead of waiting for heartbeat timeout
177        let reclaimed = sqlx::query(
178            r#"
179            UPDATE durable_task_queue
180            SET status = 'pending',
181                claimed_by = NULL,
182                claimed_at = NULL
183            WHERE status = 'claimed'
184              AND claimed_by = $1
185            "#,
186        )
187        .bind(worker_id)
188        .execute(&self.pool)
189        .await
190        .map_err(|e| {
191            error!(error = %e, "Failed to reclaim worker tasks");
192            StoreError::Database(e.to_string())
193        })?
194        .rows_affected() as usize;
195
196        // Mark worker as stopped
197        sqlx::query(
198            r#"
199            UPDATE durable_workers
200            SET status = 'stopped'
201            WHERE id = $1
202            "#,
203        )
204        .bind(worker_id)
205        .execute(&self.pool)
206        .await
207        .map_err(|e| {
208            error!(error = %e, "Failed to deregister worker");
209            StoreError::Database(e.to_string())
210        })?;
211
212        if reclaimed > 0 {
213            info!(
214                worker_id,
215                reclaimed, "deregistered worker and reclaimed tasks"
216            );
217        } else {
218            debug!(worker_id, "deregistered worker (no tasks to reclaim)");
219        }
220
221        Ok(reclaimed)
222    }
223
224    #[instrument(skip(self))]
225    async fn get_capacity_snapshot(&self) -> Result<CapacitySnapshot, StoreError> {
226        let row = sqlx::query(
227            r#"
228            SELECT
229                COALESCE(SUM(max_concurrency - current_load), 0)::INT AS total_available,
230                COUNT(*)::INT AS active_workers
231            FROM durable_workers
232            WHERE status = 'active'
233              AND accepting_tasks = true
234              AND last_heartbeat_at > NOW() - INTERVAL '30 seconds'
235            "#,
236        )
237        .fetch_one(&self.pool)
238        .await
239        .map_err(|e| {
240            error!(error = %e, "Failed to get capacity snapshot");
241            StoreError::Database(e.to_string())
242        })?;
243
244        Ok(CapacitySnapshot {
245            total_available: row.get::<i32, _>("total_available") as u32,
246            active_workers: row.get::<i32, _>("active_workers") as u32,
247        })
248    }
249
250    #[instrument(skip(self))]
251    async fn drain_worker(&self, worker_id: &str) -> Result<(), StoreError> {
252        sqlx::query(
253            r#"
254            UPDATE durable_workers
255            SET metadata = CASE
256                    WHEN status = 'draining' THEN metadata
257                    ELSE COALESCE(metadata, '{}'::jsonb) || jsonb_build_object($2::text, NOW())
258                END,
259                status = 'draining',
260                accepting_tasks = false
261            WHERE id = $1
262            "#,
263        )
264        .bind(worker_id)
265        .bind(WORKER_META_DRAINING_SINCE)
266        .execute(&self.pool)
267        .await
268        .map_err(|e| {
269            error!(error = %e, "Failed to drain worker");
270            StoreError::Database(e.to_string())
271        })?;
272
273        debug!(worker_id, "drained worker");
274        Ok(())
275    }
276
277    #[instrument(skip(self))]
278    async fn resume_worker(&self, worker_id: &str) -> Result<(), StoreError> {
279        sqlx::query(
280            r#"
281            UPDATE durable_workers
282            SET status = 'active',
283                accepting_tasks = true,
284                metadata = metadata - $2::text
285            WHERE id = $1 AND status = 'draining'
286            "#,
287        )
288        .bind(worker_id)
289        .bind(WORKER_META_DRAINING_SINCE)
290        .execute(&self.pool)
291        .await
292        .map_err(|e| {
293            error!(error = %e, "Failed to resume worker");
294            StoreError::Database(e.to_string())
295        })?;
296
297        debug!(worker_id, "resumed worker");
298        Ok(())
299    }
300
301    #[instrument(skip(self, checks))]
302    async fn record_worker_checks(
303        &self,
304        worker_id: &str,
305        checks: serde_json::Value,
306    ) -> Result<(), StoreError> {
307        sqlx::query(
308            r#"
309            UPDATE durable_workers
310            SET metadata = COALESCE(metadata, '{}'::jsonb)
311                || jsonb_build_object($2::text, $3::jsonb, $4::text, NOW())
312            WHERE id = $1
313            "#,
314        )
315        .bind(worker_id)
316        .bind(WORKER_META_SELF_CHECKS)
317        .bind(checks)
318        .bind(WORKER_META_SELF_CHECKED_AT)
319        .execute(&self.pool)
320        .await
321        .map_err(|e| {
322            error!(error = %e, "Failed to record worker checks");
323            StoreError::Database(e.to_string())
324        })?;
325        Ok(())
326    }
327}