Skip to main content

everruns_durable/persistence/postgres/
admin.rs

1//! Implementation of [`crate::DurableAdmin`].
2
3use super::*;
4
5#[async_trait]
6impl DurableAdmin for PostgresWorkflowEventStore {
7    #[instrument(skip(self))]
8    async fn count_active_workflows(&self) -> Result<i64, StoreError> {
9        let row = sqlx::query(
10            r#"
11            SELECT COUNT(*) as count
12            FROM durable_workflow_instances
13            WHERE status IN ('pending', 'running')
14            "#,
15        )
16        .fetch_one(&self.pool)
17        .await
18        .map_err(|e| {
19            error!(error = %e, "Failed to count active workflows");
20            StoreError::Database(e.to_string())
21        })?;
22
23        Ok(row.get("count"))
24    }
25
26    #[instrument(skip(self))]
27    async fn get_workflow_extended(
28        &self,
29        workflow_id: Uuid,
30    ) -> Result<Option<WorkflowInfoExtended>, StoreError> {
31        // EVE-455: direct id lookup. The previous implementation scanned the
32        // first 1000 rows of `list_workflows`, which silently dropped older
33        // workflows once a deployment grew past that page.
34        let row = sqlx::query(
35            r#"
36            SELECT id, workflow_type, status, input, result, error,
37                   created_at, started_at, completed_at, continued_as_new_id
38            FROM durable_workflow_instances
39            WHERE id = $1
40            "#,
41        )
42        .bind(workflow_id)
43        .fetch_optional(&self.pool)
44        .await
45        .map_err(|e| {
46            error!(error = %e, workflow_id = %workflow_id, "Failed to load workflow");
47            StoreError::Database(e.to_string())
48        })?;
49
50        let Some(row) = row else { return Ok(None) };
51        let status_str: String = row.get("status");
52        let error_json: Option<serde_json::Value> = row.get("error");
53        Ok(Some(WorkflowInfoExtended {
54            id: row.get("id"),
55            workflow_type: row.get("workflow_type"),
56            status: parse_workflow_status(&status_str)?,
57            input: row.get("input"),
58            result: row.get("result"),
59            error: error_json.and_then(|v| serde_json::from_value(v).ok()),
60            created_at: row.get("created_at"),
61            started_at: row.get("started_at"),
62            completed_at: row.get("completed_at"),
63            continued_as_new_id: row.try_get("continued_as_new_id").ok().flatten(),
64        }))
65    }
66
67    async fn list_workflows(
68        &self,
69        filter: WorkflowFilter,
70        pagination: Pagination,
71    ) -> Result<Vec<WorkflowInfoExtended>, StoreError> {
72        let status_str = filter.status.map(|s| s.to_string());
73
74        let rows = sqlx::query(
75            r#"
76            SELECT id, workflow_type, status, input, result, error,
77                   created_at, started_at, completed_at, continued_as_new_id
78            FROM durable_workflow_instances
79            WHERE ($1::text IS NULL OR status = $1)
80              AND ($2::text IS NULL OR workflow_type = $2)
81            ORDER BY created_at DESC
82            OFFSET $3
83            LIMIT $4
84            "#,
85        )
86        .bind(&status_str)
87        .bind(&filter.workflow_type)
88        .bind(pagination.offset as i64)
89        .bind(pagination.limit as i64)
90        .fetch_all(&self.pool)
91        .await
92        .map_err(|e| {
93            error!(error = %e, "Failed to list workflows");
94            StoreError::Database(e.to_string())
95        })?;
96
97        let mut workflows = Vec::with_capacity(rows.len());
98        for row in rows {
99            let status_str: String = row.get("status");
100            let error_json: Option<serde_json::Value> = row.get("error");
101
102            workflows.push(WorkflowInfoExtended {
103                id: row.get("id"),
104                workflow_type: row.get("workflow_type"),
105                status: parse_workflow_status(&status_str)?,
106                input: row.get("input"),
107                result: row.get("result"),
108                error: error_json.and_then(|v| serde_json::from_value(v).ok()),
109                created_at: row.get("created_at"),
110                started_at: row.get("started_at"),
111                completed_at: row.get("completed_at"),
112                continued_as_new_id: row.try_get("continued_as_new_id").ok().flatten(),
113            });
114        }
115
116        Ok(workflows)
117    }
118
119    #[instrument(skip(self))]
120    async fn get_system_health(&self) -> Result<SystemHealth, StoreError> {
121        // Single query to get all health stats.
122        // $1 = heartbeat threshold: only workers with heartbeat after this are "active".
123        let heartbeat_threshold =
124            Utc::now() - chrono::Duration::seconds(WORKER_HEARTBEAT_TIMEOUT_SECS);
125
126        // Live gauges (pending/claimed tasks, running/pending workflows, workers,
127        // DLQ) are bounded by current work and backed by partial indexes, so they
128        // stay as direct queries. The cumulative totals (completed/failed/started
129        // for tasks and workflows) are read from `durable_stat_counters`, which is
130        // maintained incrementally by triggers (migration 082) and sharded so
131        // writers do not queue on one row (migration 182); a total is the sum of
132        // its shards. This keeps the health path O(1) instead of scanning the
133        // unbounded historical tables on every 10s metrics sample. See EVE-605.
134        let row = sqlx::query(
135            r#"
136            SELECT
137                (SELECT COUNT(*) FROM durable_workers) as total_workers,
138                (SELECT COUNT(*) FROM durable_workers WHERE status = 'active' AND last_heartbeat_at > $1) as active_workers,
139                (SELECT COUNT(*) FROM durable_workers WHERE status = 'active' AND accepting_tasks = true AND last_heartbeat_at > $1) as workers_accepting,
140                (SELECT COALESCE(SUM(max_concurrency), 0) FROM durable_workers WHERE status = 'active' AND last_heartbeat_at > $1) as total_capacity,
141                (SELECT COALESCE(SUM(current_load), 0) FROM durable_workers WHERE status = 'active' AND last_heartbeat_at > $1) as current_load,
142                (SELECT COUNT(*) FROM durable_task_queue WHERE status = 'pending') as pending_tasks,
143                (SELECT COUNT(*) FROM durable_task_queue WHERE status = 'claimed') as claimed_tasks,
144                COALESCE((SELECT SUM(value) FROM durable_stat_counters WHERE name = 'tasks_completed'), 0)::BIGINT as completed_tasks,
145                COALESCE((SELECT SUM(value) FROM durable_stat_counters WHERE name = 'tasks_failed'), 0)::BIGINT as failed_tasks,
146                COALESCE((SELECT SUM(value) FROM durable_stat_counters WHERE name = 'tasks_started'), 0)::BIGINT as started_tasks,
147                (SELECT COUNT(*) FROM durable_workflow_instances WHERE status = 'running') as running_workflows,
148                (SELECT COUNT(*) FROM durable_workflow_instances WHERE status = 'pending') as pending_workflows,
149                COALESCE((SELECT SUM(value) FROM durable_stat_counters WHERE name = 'workflows_completed'), 0)::BIGINT as completed_workflows,
150                COALESCE((SELECT SUM(value) FROM durable_stat_counters WHERE name = 'workflows_failed'), 0)::BIGINT as failed_workflows,
151                COALESCE((SELECT SUM(value) FROM durable_stat_counters WHERE name = 'workflows_started'), 0)::BIGINT as started_workflows,
152                (SELECT COUNT(*) FROM durable_dead_letter_queue WHERE requeued_at IS NULL) as dlq_size
153            "#,
154        )
155        .bind(heartbeat_threshold)
156        .fetch_one(&self.pool)
157        .await
158        .map_err(|e| store_failure("durable.admin.system_health", "Failed to get system health", e))?;
159
160        Ok(SystemHealth {
161            total_workers: row.get::<i64, _>("total_workers") as usize,
162            active_workers: row.get::<i64, _>("active_workers") as usize,
163            workers_accepting: row.get::<i64, _>("workers_accepting") as usize,
164            total_capacity: row.get::<i64, _>("total_capacity") as usize,
165            current_load: row.get::<i64, _>("current_load") as usize,
166            pending_tasks: row.get::<i64, _>("pending_tasks") as usize,
167            claimed_tasks: row.get::<i64, _>("claimed_tasks") as usize,
168            completed_tasks: row.get::<i64, _>("completed_tasks") as usize,
169            failed_tasks: row.get::<i64, _>("failed_tasks") as usize,
170            started_tasks: row.get::<i64, _>("started_tasks") as usize,
171            running_workflows: row.get::<i64, _>("running_workflows") as usize,
172            pending_workflows: row.get::<i64, _>("pending_workflows") as usize,
173            completed_workflows: row.get::<i64, _>("completed_workflows") as usize,
174            failed_workflows: row.get::<i64, _>("failed_workflows") as usize,
175            started_workflows: row.get::<i64, _>("started_workflows") as usize,
176            dlq_size: row.get::<i64, _>("dlq_size") as usize,
177        })
178    }
179
180    #[instrument(skip(self))]
181    async fn get_workflow_events(
182        &self,
183        workflow_id: Uuid,
184    ) -> Result<Vec<WorkflowEventInfo>, StoreError> {
185        let rows = sqlx::query(
186            r#"
187            SELECT id, workflow_id, sequence_num, event_type, event_data, created_at
188            FROM durable_workflow_events
189            WHERE workflow_id = $1
190            ORDER BY sequence_num
191            "#,
192        )
193        .bind(workflow_id)
194        .fetch_all(&self.pool)
195        .await
196        .map_err(|e| {
197            error!(error = %e, "Failed to get workflow events");
198            StoreError::Database(e.to_string())
199        })?;
200
201        let events = rows
202            .into_iter()
203            .map(|row| WorkflowEventInfo {
204                id: row.get("id"),
205                workflow_id: row.get("workflow_id"),
206                sequence_num: row.get("sequence_num"),
207                event_type: row.get("event_type"),
208                event_data: row.get("event_data"),
209                created_at: row.get("created_at"),
210            })
211            .collect();
212
213        Ok(events)
214    }
215
216    #[instrument(skip(self))]
217    async fn count_workflow_events(&self, workflow_id: Uuid) -> Result<i64, StoreError> {
218        let (count,): (i64,) =
219            sqlx::query_as("SELECT COUNT(*) FROM durable_workflow_events WHERE workflow_id = $1")
220                .bind(workflow_id)
221                .fetch_one(&self.pool)
222                .await
223                .map_err(|e| {
224                    error!(error = %e, "Failed to count workflow events");
225                    StoreError::Database(e.to_string())
226                })?;
227
228        Ok(count)
229    }
230}