Skip to main content

assay_workflow/store/
postgres.rs

1use anyhow::Result;
2use sqlx::PgPool;
3
4use crate::store::WorkflowStore;
5use crate::types::*;
6
7const SCHEMA: &str = r#"
8CREATE TABLE IF NOT EXISTS namespaces (
9    name            TEXT PRIMARY KEY,
10    created_at      DOUBLE PRECISION NOT NULL
11);
12INSERT INTO namespaces (name, created_at)
13    VALUES ('main', EXTRACT(EPOCH FROM NOW()))
14    ON CONFLICT DO NOTHING;
15
16CREATE TABLE IF NOT EXISTS workflows (
17    id              TEXT PRIMARY KEY,
18    namespace       TEXT NOT NULL DEFAULT 'main',
19    run_id          TEXT NOT NULL,
20    workflow_type   TEXT NOT NULL,
21    task_queue      TEXT NOT NULL DEFAULT 'main',
22    status          TEXT NOT NULL DEFAULT 'PENDING',
23    input           TEXT,
24    result          TEXT,
25    error           TEXT,
26    parent_id       TEXT,
27    claimed_by      TEXT,
28    search_attributes TEXT,
29    archived_at     DOUBLE PRECISION,
30    archive_uri     TEXT,
31    -- Workflow-task dispatch (Phase 9): see sqlite.rs for the full comment.
32    needs_dispatch  BOOLEAN NOT NULL DEFAULT FALSE,
33    dispatch_claimed_by    TEXT,
34    dispatch_last_heartbeat DOUBLE PRECISION,
35    created_at      DOUBLE PRECISION NOT NULL,
36    updated_at      DOUBLE PRECISION NOT NULL,
37    completed_at    DOUBLE PRECISION
38);
39CREATE INDEX IF NOT EXISTS idx_wf_status_queue ON workflows(status, task_queue);
40CREATE INDEX IF NOT EXISTS idx_wf_namespace ON workflows(namespace);
41CREATE INDEX IF NOT EXISTS idx_wf_dispatch ON workflows(task_queue, needs_dispatch, dispatch_claimed_by);
42
43CREATE TABLE IF NOT EXISTS workflow_events (
44    id              BIGSERIAL PRIMARY KEY,
45    workflow_id     TEXT NOT NULL REFERENCES workflows(id),
46    seq             INTEGER NOT NULL,
47    event_type      TEXT NOT NULL,
48    payload         TEXT,
49    timestamp       DOUBLE PRECISION NOT NULL
50);
51CREATE INDEX IF NOT EXISTS idx_wf_events_lookup ON workflow_events(workflow_id, seq);
52
53CREATE TABLE IF NOT EXISTS workflow_activities (
54    id              BIGSERIAL PRIMARY KEY,
55    workflow_id     TEXT NOT NULL REFERENCES workflows(id),
56    seq             INTEGER NOT NULL,
57    name            TEXT NOT NULL,
58    task_queue      TEXT NOT NULL DEFAULT 'main',
59    input           TEXT,
60    status          TEXT NOT NULL DEFAULT 'PENDING',
61    result          TEXT,
62    error           TEXT,
63    attempt         INTEGER NOT NULL DEFAULT 1,
64    max_attempts    INTEGER NOT NULL DEFAULT 3,
65    initial_interval_secs   DOUBLE PRECISION NOT NULL DEFAULT 1,
66    backoff_coefficient     DOUBLE PRECISION NOT NULL DEFAULT 2,
67    start_to_close_secs     DOUBLE PRECISION NOT NULL DEFAULT 300,
68    heartbeat_timeout_secs  DOUBLE PRECISION,
69    claimed_by      TEXT,
70    scheduled_at    DOUBLE PRECISION NOT NULL,
71    started_at      DOUBLE PRECISION,
72    completed_at    DOUBLE PRECISION,
73    last_heartbeat  DOUBLE PRECISION,
74    UNIQUE (workflow_id, seq)
75);
76CREATE INDEX IF NOT EXISTS idx_wf_act_pending ON workflow_activities(task_queue, status, scheduled_at);
77
78CREATE TABLE IF NOT EXISTS workflow_timers (
79    id              BIGSERIAL PRIMARY KEY,
80    workflow_id     TEXT NOT NULL REFERENCES workflows(id),
81    seq             INTEGER NOT NULL,
82    fire_at         DOUBLE PRECISION NOT NULL,
83    fired           BOOLEAN NOT NULL DEFAULT FALSE,
84    UNIQUE (workflow_id, seq)
85);
86CREATE INDEX IF NOT EXISTS idx_wf_timers_due ON workflow_timers(fire_at) WHERE fired = FALSE;
87
88CREATE TABLE IF NOT EXISTS workflow_signals (
89    id              BIGSERIAL PRIMARY KEY,
90    workflow_id     TEXT NOT NULL REFERENCES workflows(id),
91    name            TEXT NOT NULL,
92    payload         TEXT,
93    consumed        BOOLEAN NOT NULL DEFAULT FALSE,
94    received_at     DOUBLE PRECISION NOT NULL
95);
96CREATE INDEX IF NOT EXISTS idx_wf_signals_lookup ON workflow_signals(workflow_id, name, consumed);
97
98CREATE TABLE IF NOT EXISTS workflow_schedules (
99    namespace       TEXT NOT NULL DEFAULT 'main',
100    name            TEXT NOT NULL,
101    workflow_type   TEXT NOT NULL,
102    cron_expr       TEXT NOT NULL,
103    timezone        TEXT NOT NULL DEFAULT 'UTC',
104    input           TEXT,
105    task_queue      TEXT NOT NULL DEFAULT 'main',
106    overlap_policy  TEXT NOT NULL DEFAULT 'skip',
107    paused          BOOLEAN NOT NULL DEFAULT FALSE,
108    last_run_at     DOUBLE PRECISION,
109    next_run_at     DOUBLE PRECISION,
110    last_workflow_id TEXT,
111    created_at      DOUBLE PRECISION NOT NULL,
112    PRIMARY KEY (namespace, name)
113);
114
115CREATE TABLE IF NOT EXISTS workflow_workers (
116    id              TEXT PRIMARY KEY,
117    namespace       TEXT NOT NULL DEFAULT 'main',
118    identity        TEXT NOT NULL,
119    task_queue      TEXT NOT NULL,
120    workflows       TEXT,
121    activities      TEXT,
122    max_concurrent_workflows  INTEGER NOT NULL DEFAULT 10,
123    max_concurrent_activities INTEGER NOT NULL DEFAULT 10,
124    active_tasks    INTEGER NOT NULL DEFAULT 0,
125    last_heartbeat  DOUBLE PRECISION NOT NULL,
126    registered_at   DOUBLE PRECISION NOT NULL
127);
128
129CREATE TABLE IF NOT EXISTS workflow_snapshots (
130    workflow_id     TEXT NOT NULL REFERENCES workflows(id),
131    event_seq       INTEGER NOT NULL,
132    state_json      TEXT NOT NULL,
133    created_at      DOUBLE PRECISION NOT NULL,
134    PRIMARY KEY (workflow_id, event_seq)
135);
136
137CREATE TABLE IF NOT EXISTS api_keys (
138    key_hash        TEXT PRIMARY KEY,
139    prefix          TEXT NOT NULL,
140    label           TEXT,
141    created_at      DOUBLE PRECISION NOT NULL
142);
143CREATE INDEX IF NOT EXISTS idx_api_keys_prefix ON api_keys(prefix);
144
145"#;
146
147/// Trigger DDL for LISTEN/NOTIFY push streams (Task 3.17).
148///
149/// Executed as raw SQL (not through `sanitise_schema`) because the PL/pgSQL
150/// function bodies use `$$` dollar-quoting which contains semicolons that would
151/// be incorrectly split by `sanitise_schema`'s naive `;` splitter.
152///
153/// Idempotent: `CREATE OR REPLACE FUNCTION` and `DROP TRIGGER IF EXISTS` /
154/// `CREATE TRIGGER` are safe to run on every startup.
155const TRIGGER_DDL: &str = r#"
156CREATE OR REPLACE FUNCTION assay_notify_runnable() RETURNS trigger AS $$
157BEGIN
158  IF NEW.needs_dispatch = TRUE
159     AND NEW.status IN ('PENDING', 'RUNNING')
160     AND NEW.dispatch_claimed_by IS NULL
161  THEN
162    PERFORM pg_notify('assay_runnable_' || NEW.namespace, NEW.id);
163  END IF;
164  RETURN NEW;
165END;
166$$ LANGUAGE plpgsql;
167
168DROP TRIGGER IF EXISTS workflow_runnable_notify ON workflows;
169CREATE TRIGGER workflow_runnable_notify
170  AFTER INSERT OR UPDATE OF status, needs_dispatch, dispatch_claimed_by ON workflows
171  FOR EACH ROW EXECUTE FUNCTION assay_notify_runnable();
172
173CREATE OR REPLACE FUNCTION assay_notify_task() RETURNS trigger AS $$
174BEGIN
175  PERFORM pg_notify('assay_task_' || NEW.task_queue, NEW.id::text);
176  RETURN NEW;
177END;
178$$ LANGUAGE plpgsql;
179
180DROP TRIGGER IF EXISTS workflow_task_notify ON workflow_activities;
181CREATE TRIGGER workflow_task_notify
182  AFTER INSERT ON workflow_activities
183  FOR EACH ROW EXECUTE FUNCTION assay_notify_task();
184"#;
185
186/// Split a Postgres DDL script into individual statements ready for `sqlx::query`.
187///
188/// Drops pure-comment lines (those starting with `--` after optional whitespace)
189/// *before* splitting on `;`. Without this step, a semicolon inside a line comment
190/// (e.g. `-- Idempotent across startups; fresh installs pick the column up`) would
191/// split the surrounding comment into fragments — one of which is naked prose that
192/// Postgres tries to parse as SQL and rejects with `syntax error at or near "<word>"`.
193///
194/// The filter only drops *pure-comment* lines (leading whitespace then `--`), leaving
195/// `--`-after-code untouched. That keeps string literals safe (could legally contain
196/// `--`) and is conservative enough to remain correct if the SCHEMA grows more prose.
197fn sanitise_schema(schema: &str) -> Vec<String> {
198    let without_comments: String = schema
199        .lines()
200        .filter(|line| !line.trim_start().starts_with("--"))
201        .collect::<Vec<_>>()
202        .join("\n");
203
204    without_comments
205        .split(';')
206        .map(|s| s.trim().to_string())
207        .filter(|s| !s.is_empty())
208        .collect()
209}
210
211pub struct PostgresStore {
212    pool: PgPool,
213}
214
215impl PostgresStore {
216    pub async fn new(url: &str) -> Result<Self> {
217        let pool = PgPool::connect(url).await?;
218        Self::from_pool(pool).await
219    }
220
221    /// Build a store from an existing pool. Runs migrations on the target
222    /// database. Useful when the engine owns the pool (shared with other
223    /// modules) and hands a clone to the workflow module, or for tests that
224    /// point many stores at different databases in the same Postgres server.
225    pub async fn from_pool(pool: PgPool) -> Result<Self> {
226        let store = Self { pool };
227        store.migrate().await?;
228        Ok(store)
229    }
230
231    async fn migrate(&self) -> Result<()> {
232        // Apply the base schema (tables + indexes) statement-by-statement.
233        for statement in sanitise_schema(SCHEMA) {
234            sqlx::query(&statement).execute(&self.pool).await?;
235        }
236        // Apply LISTEN/NOTIFY trigger DDL as a single raw execution.
237        // Can't use sanitise_schema here because PL/pgSQL $$ bodies contain
238        // semicolons that would be incorrectly split.
239        sqlx::raw_sql(TRIGGER_DDL).execute(&self.pool).await?;
240        Ok(())
241    }
242
243    /// Try to acquire pg_advisory_lock for leader election.
244    /// Returns true if this instance is the leader (scheduler should run).
245    pub async fn try_acquire_leader_lock(&self) -> Result<bool> {
246        let row: (bool,) =
247            sqlx::query_as("SELECT pg_try_advisory_lock(1)")
248                .fetch_one(&self.pool)
249                .await?;
250        Ok(row.0)
251    }
252}
253
254impl WorkflowStore for PostgresStore {
255    // ── Namespaces ─────────────────────────────────────────
256
257    async fn create_namespace(&self, name: &str) -> Result<()> {
258        sqlx::query("INSERT INTO namespaces (name, created_at) VALUES ($1, EXTRACT(EPOCH FROM NOW()))")
259            .bind(name)
260            .execute(&self.pool)
261            .await?;
262        Ok(())
263    }
264
265    async fn list_namespaces(&self) -> Result<Vec<crate::store::NamespaceRecord>> {
266        let rows = sqlx::query_as::<_, (String, f64)>(
267            "SELECT name, created_at FROM namespaces ORDER BY name",
268        )
269        .fetch_all(&self.pool)
270        .await?;
271        Ok(rows
272            .into_iter()
273            .map(|(name, created_at)| crate::store::NamespaceRecord { name, created_at })
274            .collect())
275    }
276
277    async fn delete_namespace(&self, name: &str) -> Result<bool> {
278        let res = sqlx::query("DELETE FROM namespaces WHERE name = $1 AND name != 'main'")
279            .bind(name)
280            .execute(&self.pool)
281            .await?;
282        Ok(res.rows_affected() > 0)
283    }
284
285    async fn get_namespace_stats(&self, namespace: &str) -> Result<crate::store::NamespaceStats> {
286        let total: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM workflows WHERE namespace = $1")
287            .bind(namespace)
288            .fetch_one(&self.pool)
289            .await?;
290        let running: (i64,) = sqlx::query_as(
291            "SELECT COUNT(*) FROM workflows WHERE namespace = $1 AND status = 'RUNNING'",
292        )
293        .bind(namespace)
294        .fetch_one(&self.pool)
295        .await?;
296        let pending: (i64,) = sqlx::query_as(
297            "SELECT COUNT(*) FROM workflows WHERE namespace = $1 AND status = 'PENDING'",
298        )
299        .bind(namespace)
300        .fetch_one(&self.pool)
301        .await?;
302        let completed: (i64,) = sqlx::query_as(
303            "SELECT COUNT(*) FROM workflows WHERE namespace = $1 AND status = 'COMPLETED'",
304        )
305        .bind(namespace)
306        .fetch_one(&self.pool)
307        .await?;
308        let failed: (i64,) = sqlx::query_as(
309            "SELECT COUNT(*) FROM workflows WHERE namespace = $1 AND status = 'FAILED'",
310        )
311        .bind(namespace)
312        .fetch_one(&self.pool)
313        .await?;
314        let schedules: (i64,) =
315            sqlx::query_as("SELECT COUNT(*) FROM workflow_schedules WHERE namespace = $1")
316                .bind(namespace)
317                .fetch_one(&self.pool)
318                .await?;
319        let workers: (i64,) =
320            sqlx::query_as("SELECT COUNT(*) FROM workflow_workers WHERE namespace = $1")
321                .bind(namespace)
322                .fetch_one(&self.pool)
323                .await?;
324
325        Ok(crate::store::NamespaceStats {
326            namespace: namespace.to_string(),
327            total_workflows: total.0,
328            running: running.0,
329            pending: pending.0,
330            completed: completed.0,
331            failed: failed.0,
332            schedules: schedules.0,
333            workers: workers.0,
334        })
335    }
336
337    // ── Workflows ──────────────────────────────────────────
338
339    async fn create_workflow(&self, wf: &WorkflowRecord) -> Result<()> {
340        sqlx::query(
341            "INSERT INTO workflows (id, namespace, run_id, workflow_type, task_queue, status, input, result, error, parent_id, claimed_by, search_attributes, archived_at, archive_uri, created_at, updated_at, completed_at)
342             VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)",
343        )
344        .bind(&wf.id)
345        .bind(&wf.namespace)
346        .bind(&wf.run_id)
347        .bind(&wf.workflow_type)
348        .bind(&wf.task_queue)
349        .bind(&wf.status)
350        .bind(&wf.input)
351        .bind(&wf.result)
352        .bind(&wf.error)
353        .bind(&wf.parent_id)
354        .bind(&wf.claimed_by)
355        .bind(&wf.search_attributes)
356        .bind(wf.archived_at)
357        .bind(&wf.archive_uri)
358        .bind(wf.created_at)
359        .bind(wf.updated_at)
360        .bind(wf.completed_at)
361        .execute(&self.pool)
362        .await?;
363        Ok(())
364    }
365
366    async fn get_workflow(&self, id: &str) -> Result<Option<WorkflowRecord>> {
367        let row = sqlx::query_as::<_, PgWorkflowRow>(
368            "SELECT id, namespace, run_id, workflow_type, task_queue, status, input, result, error, parent_id, claimed_by, search_attributes, archived_at, archive_uri, created_at, updated_at, completed_at FROM workflows WHERE id = $1",
369        )
370        .bind(id)
371        .fetch_optional(&self.pool)
372        .await?;
373        Ok(row.map(Into::into))
374    }
375
376    async fn list_workflows(
377        &self,
378        namespace: &str,
379        status: Option<WorkflowStatus>,
380        workflow_type: Option<&str>,
381        search_attrs_filter: Option<&str>,
382        limit: i64,
383        offset: i64,
384    ) -> Result<Vec<WorkflowRecord>> {
385        let status_str = status.map(|s| s.to_string());
386
387        let filter_pairs: Vec<(String, serde_json::Value)> = search_attrs_filter
388            .and_then(|s| serde_json::from_str::<serde_json::Value>(s).ok())
389            .and_then(|v| v.as_object().cloned())
390            .map(|m| m.into_iter().collect())
391            .unwrap_or_default();
392
393        let mut sql = String::from(
394            "SELECT id, namespace, run_id, workflow_type, task_queue, status, input, result, error, parent_id, claimed_by, search_attributes, archived_at, archive_uri, created_at, updated_at, completed_at
395             FROM workflows
396             WHERE namespace = $1
397               AND ($2::TEXT IS NULL OR status = $2)
398               AND ($3::TEXT IS NULL OR workflow_type = $3)",
399        );
400        // Bind placeholders for the filter follow $3; next index is 4.
401        let mut idx = 4usize;
402        for _ in &filter_pairs {
403            sql.push_str(&format!(
404                " AND (search_attributes::jsonb)->>${} = ${}",
405                idx,
406                idx + 1
407            ));
408            idx += 2;
409        }
410        sql.push_str(&format!(" ORDER BY created_at DESC LIMIT ${} OFFSET ${}", idx, idx + 1));
411
412        let mut q = sqlx::query_as::<_, PgWorkflowRow>(&sql)
413            .bind(namespace)
414            .bind(&status_str)
415            .bind(workflow_type);
416        for (key, value) in &filter_pairs {
417            q = q.bind(key.clone());
418            // JSONB ->> always returns TEXT; compare by stringified value.
419            let as_text = match value {
420                serde_json::Value::String(s) => s.clone(),
421                other => other.to_string(),
422            };
423            q = q.bind(as_text);
424        }
425        let rows = q.bind(limit).bind(offset).fetch_all(&self.pool).await?;
426        Ok(rows.into_iter().map(Into::into).collect())
427    }
428
429    async fn update_workflow_status(
430        &self,
431        id: &str,
432        status: WorkflowStatus,
433        result: Option<&str>,
434        error: Option<&str>,
435    ) -> Result<()> {
436        let now = timestamp_now();
437        let completed_at = if status.is_terminal() { Some(now) } else { None };
438        sqlx::query(
439            "UPDATE workflows SET status = $1, result = COALESCE($2, result), error = COALESCE($3, error), updated_at = $4, completed_at = COALESCE($5, completed_at) WHERE id = $6",
440        )
441        .bind(status.to_string())
442        .bind(result)
443        .bind(error)
444        .bind(now)
445        .bind(completed_at)
446        .bind(id)
447        .execute(&self.pool)
448        .await?;
449        Ok(())
450    }
451
452    async fn claim_workflow(&self, id: &str, worker_id: &str) -> Result<bool> {
453        let res = sqlx::query(
454            "UPDATE workflows SET claimed_by = $1, status = 'RUNNING', updated_at = $2 WHERE id = $3 AND claimed_by IS NULL",
455        )
456        .bind(worker_id)
457        .bind(timestamp_now())
458        .bind(id)
459        .execute(&self.pool)
460        .await?;
461        Ok(res.rows_affected() > 0)
462    }
463
464    async fn mark_workflow_dispatchable(&self, workflow_id: &str) -> Result<()> {
465        sqlx::query("UPDATE workflows SET needs_dispatch = TRUE WHERE id = $1")
466            .bind(workflow_id)
467            .execute(&self.pool)
468            .await?;
469        Ok(())
470    }
471
472    async fn claim_workflow_task(
473        &self,
474        task_queue: &str,
475        worker_id: &str,
476    ) -> Result<Option<WorkflowRecord>> {
477        let now = timestamp_now();
478        // Atomic claim with FOR UPDATE SKIP LOCKED so multiple engine
479        // replicas don't fight over the same workflow task.
480        let row = sqlx::query_as::<_, PgWorkflowRow>(
481            "UPDATE workflows
482             SET dispatch_claimed_by = $1, dispatch_last_heartbeat = $2, needs_dispatch = FALSE
483             WHERE id = (
484                SELECT id FROM workflows
485                WHERE task_queue = $3
486                  AND needs_dispatch = TRUE
487                  AND dispatch_claimed_by IS NULL
488                  AND status NOT IN ('COMPLETED', 'FAILED', 'CANCELLED', 'TIMED_OUT')
489                ORDER BY updated_at ASC
490                FOR UPDATE SKIP LOCKED
491                LIMIT 1
492             )
493             RETURNING id, namespace, run_id, workflow_type, task_queue, status, input, result, error, parent_id, claimed_by, search_attributes, archived_at, archive_uri, created_at, updated_at, completed_at",
494        )
495        .bind(worker_id)
496        .bind(now)
497        .bind(task_queue)
498        .fetch_optional(&self.pool)
499        .await?;
500        Ok(row.map(Into::into))
501    }
502
503    async fn release_workflow_task(&self, workflow_id: &str, worker_id: &str) -> Result<()> {
504        sqlx::query(
505            "UPDATE workflows
506             SET dispatch_claimed_by = NULL, dispatch_last_heartbeat = NULL
507             WHERE id = $1 AND dispatch_claimed_by = $2",
508        )
509        .bind(workflow_id)
510        .bind(worker_id)
511        .execute(&self.pool)
512        .await?;
513        Ok(())
514    }
515
516    async fn release_stale_dispatch_leases(
517        &self,
518        now: f64,
519        timeout_secs: f64,
520    ) -> Result<u64> {
521        let res = sqlx::query(
522            "UPDATE workflows
523             SET dispatch_claimed_by = NULL,
524                 dispatch_last_heartbeat = NULL,
525                 needs_dispatch = TRUE
526             WHERE dispatch_claimed_by IS NOT NULL
527               AND ($1 - dispatch_last_heartbeat) > $2
528               AND status NOT IN ('COMPLETED', 'FAILED', 'CANCELLED', 'TIMED_OUT')",
529        )
530        .bind(now)
531        .bind(timeout_secs)
532        .execute(&self.pool)
533        .await?;
534        Ok(res.rows_affected())
535    }
536
537    // ── Events ─────────────────────────────────────────────
538
539    async fn append_event(&self, ev: &WorkflowEvent) -> Result<i64> {
540        let row: (i64,) = sqlx::query_as(
541            "INSERT INTO workflow_events (workflow_id, seq, event_type, payload, timestamp) VALUES ($1, $2, $3, $4, $5) RETURNING id",
542        )
543        .bind(&ev.workflow_id)
544        .bind(ev.seq)
545        .bind(&ev.event_type)
546        .bind(&ev.payload)
547        .bind(ev.timestamp)
548        .fetch_one(&self.pool)
549        .await?;
550        Ok(row.0)
551    }
552
553    async fn list_events(&self, workflow_id: &str) -> Result<Vec<WorkflowEvent>> {
554        let rows = sqlx::query_as::<_, PgEventRow>(
555            "SELECT id, workflow_id, seq, event_type, payload, timestamp FROM workflow_events WHERE workflow_id = $1 ORDER BY seq ASC",
556        )
557        .bind(workflow_id)
558        .fetch_all(&self.pool)
559        .await?;
560        Ok(rows.into_iter().map(Into::into).collect())
561    }
562
563    async fn get_event_count(&self, workflow_id: &str) -> Result<i64> {
564        let row: (i64,) =
565            sqlx::query_as("SELECT COUNT(*) FROM workflow_events WHERE workflow_id = $1")
566                .bind(workflow_id)
567                .fetch_one(&self.pool)
568                .await?;
569        Ok(row.0)
570    }
571
572    // ── Activities ──────────────────────────────────────────
573
574    async fn create_activity(&self, act: &WorkflowActivity) -> Result<i64> {
575        let row: (i64,) = sqlx::query_as(
576            "INSERT INTO workflow_activities (workflow_id, seq, name, task_queue, input, status, attempt, max_attempts, initial_interval_secs, backoff_coefficient, start_to_close_secs, heartbeat_timeout_secs, scheduled_at)
577             VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) RETURNING id",
578        )
579        .bind(&act.workflow_id)
580        .bind(act.seq)
581        .bind(&act.name)
582        .bind(&act.task_queue)
583        .bind(&act.input)
584        .bind(&act.status)
585        .bind(act.attempt)
586        .bind(act.max_attempts)
587        .bind(act.initial_interval_secs)
588        .bind(act.backoff_coefficient)
589        .bind(act.start_to_close_secs)
590        .bind(act.heartbeat_timeout_secs)
591        .bind(act.scheduled_at)
592        .fetch_one(&self.pool)
593        .await?;
594        Ok(row.0)
595    }
596
597    async fn get_activity(&self, id: i64) -> Result<Option<WorkflowActivity>> {
598        let row = sqlx::query_as::<_, PgActivityRow>(
599            "SELECT id, workflow_id, seq, name, task_queue, input, status, result, error, attempt, max_attempts, initial_interval_secs, backoff_coefficient, start_to_close_secs, heartbeat_timeout_secs, claimed_by, scheduled_at, started_at, completed_at, last_heartbeat
600             FROM workflow_activities WHERE id = $1",
601        )
602        .bind(id)
603        .fetch_optional(&self.pool)
604        .await?;
605        Ok(row.map(Into::into))
606    }
607
608    async fn get_activity_by_workflow_seq(
609        &self,
610        workflow_id: &str,
611        seq: i32,
612    ) -> Result<Option<WorkflowActivity>> {
613        let row = sqlx::query_as::<_, PgActivityRow>(
614            "SELECT id, workflow_id, seq, name, task_queue, input, status, result, error, attempt, max_attempts, initial_interval_secs, backoff_coefficient, start_to_close_secs, heartbeat_timeout_secs, claimed_by, scheduled_at, started_at, completed_at, last_heartbeat
615             FROM workflow_activities WHERE workflow_id = $1 AND seq = $2",
616        )
617        .bind(workflow_id)
618        .bind(seq)
619        .fetch_optional(&self.pool)
620        .await?;
621        Ok(row.map(Into::into))
622    }
623
624    async fn claim_activity(
625        &self,
626        task_queue: &str,
627        worker_id: &str,
628    ) -> Result<Option<WorkflowActivity>> {
629        let now = timestamp_now();
630        // Atomic claim using FOR UPDATE SKIP LOCKED — prevents contention
631        // between multiple assay serve instances claiming the same activity
632        let row = sqlx::query_as::<_, PgActivityRow>(
633            "UPDATE workflow_activities SET status = 'RUNNING', claimed_by = $1, started_at = $2
634             WHERE id = (
635                SELECT id FROM workflow_activities
636                WHERE task_queue = $3 AND status = 'PENDING'
637                ORDER BY scheduled_at ASC
638                FOR UPDATE SKIP LOCKED
639                LIMIT 1
640             )
641             RETURNING id, workflow_id, seq, name, task_queue, input, status, result, error, attempt, max_attempts, initial_interval_secs, backoff_coefficient, start_to_close_secs, heartbeat_timeout_secs, claimed_by, scheduled_at, started_at, completed_at, last_heartbeat",
642        )
643        .bind(worker_id)
644        .bind(now)
645        .bind(task_queue)
646        .fetch_optional(&self.pool)
647        .await?;
648        Ok(row.map(Into::into))
649    }
650
651    async fn requeue_activity_for_retry(
652        &self,
653        id: i64,
654        next_attempt: i32,
655        next_scheduled_at: f64,
656    ) -> Result<()> {
657        sqlx::query(
658            "UPDATE workflow_activities
659             SET status = 'PENDING', attempt = $1, scheduled_at = $2,
660                 claimed_by = NULL, started_at = NULL, last_heartbeat = NULL,
661                 error = NULL
662             WHERE id = $3",
663        )
664        .bind(next_attempt)
665        .bind(next_scheduled_at)
666        .bind(id)
667        .execute(&self.pool)
668        .await?;
669        Ok(())
670    }
671
672    async fn complete_activity(
673        &self,
674        id: i64,
675        result: Option<&str>,
676        error: Option<&str>,
677        failed: bool,
678    ) -> Result<()> {
679        let status = if failed { "FAILED" } else { "COMPLETED" };
680        sqlx::query(
681            "UPDATE workflow_activities SET status = $1, result = $2, error = $3, completed_at = $4 WHERE id = $5",
682        )
683        .bind(status)
684        .bind(result)
685        .bind(error)
686        .bind(timestamp_now())
687        .bind(id)
688        .execute(&self.pool)
689        .await?;
690        Ok(())
691    }
692
693    async fn heartbeat_activity(&self, id: i64, _details: Option<&str>) -> Result<()> {
694        sqlx::query("UPDATE workflow_activities SET last_heartbeat = $1 WHERE id = $2")
695            .bind(timestamp_now())
696            .bind(id)
697            .execute(&self.pool)
698            .await?;
699        Ok(())
700    }
701
702    async fn get_timed_out_activities(&self, now: f64) -> Result<Vec<WorkflowActivity>> {
703        let rows = sqlx::query_as::<_, PgActivityRow>(
704            "SELECT id, workflow_id, seq, name, task_queue, input, status, result, error, attempt, max_attempts, initial_interval_secs, backoff_coefficient, start_to_close_secs, heartbeat_timeout_secs, claimed_by, scheduled_at, started_at, completed_at, last_heartbeat
705             FROM workflow_activities
706             WHERE status = 'RUNNING'
707               AND heartbeat_timeout_secs IS NOT NULL
708               AND last_heartbeat IS NOT NULL
709               AND ($1 - last_heartbeat) > heartbeat_timeout_secs",
710        )
711        .bind(now)
712        .fetch_all(&self.pool)
713        .await?;
714        Ok(rows.into_iter().map(Into::into).collect())
715    }
716
717    // ── Timers ──────────────────────────────────────────────
718
719    async fn create_timer(&self, timer: &WorkflowTimer) -> Result<i64> {
720        // Idempotent: ON CONFLICT (workflow_id, seq) DO NOTHING.
721        // If a row already exists, RETURNING produces no rows — fall back to SELECT.
722        let inserted: Option<(i64,)> = sqlx::query_as(
723            "INSERT INTO workflow_timers (workflow_id, seq, fire_at, fired)
724             VALUES ($1, $2, $3, FALSE)
725             ON CONFLICT (workflow_id, seq) DO NOTHING
726             RETURNING id",
727        )
728        .bind(&timer.workflow_id)
729        .bind(timer.seq)
730        .bind(timer.fire_at)
731        .fetch_optional(&self.pool)
732        .await?;
733
734        if let Some((id,)) = inserted {
735            return Ok(id);
736        }
737
738        // Row already existed — return its id.
739        let (id,): (i64,) = sqlx::query_as(
740            "SELECT id FROM workflow_timers WHERE workflow_id = $1 AND seq = $2",
741        )
742        .bind(&timer.workflow_id)
743        .bind(timer.seq)
744        .fetch_one(&self.pool)
745        .await?;
746        Ok(id)
747    }
748
749    async fn cancel_pending_activities(&self, workflow_id: &str) -> Result<u64> {
750        let res = sqlx::query(
751            "UPDATE workflow_activities SET status = 'CANCELLED', completed_at = $1
752             WHERE workflow_id = $2 AND status = 'PENDING'",
753        )
754        .bind(timestamp_now())
755        .bind(workflow_id)
756        .execute(&self.pool)
757        .await?;
758        Ok(res.rows_affected())
759    }
760
761    async fn cancel_pending_timers(&self, workflow_id: &str) -> Result<u64> {
762        let res = sqlx::query(
763            "UPDATE workflow_timers SET fired = TRUE
764             WHERE workflow_id = $1 AND fired = FALSE",
765        )
766        .bind(workflow_id)
767        .execute(&self.pool)
768        .await?;
769        Ok(res.rows_affected())
770    }
771
772    async fn get_timer_by_workflow_seq(
773        &self,
774        workflow_id: &str,
775        seq: i32,
776    ) -> Result<Option<WorkflowTimer>> {
777        let row = sqlx::query_as::<_, PgTimerRow>(
778            "SELECT id, workflow_id, seq, fire_at, fired
779             FROM workflow_timers WHERE workflow_id = $1 AND seq = $2",
780        )
781        .bind(workflow_id)
782        .bind(seq)
783        .fetch_optional(&self.pool)
784        .await?;
785        Ok(row.map(Into::into))
786    }
787
788    async fn fire_due_timers(&self, now: f64) -> Result<Vec<WorkflowTimer>> {
789        let rows = sqlx::query_as::<_, PgTimerRow>(
790            "UPDATE workflow_timers SET fired = TRUE
791             WHERE fired = FALSE AND fire_at <= $1
792             RETURNING id, workflow_id, seq, fire_at, fired",
793        )
794        .bind(now)
795        .fetch_all(&self.pool)
796        .await?;
797        Ok(rows.into_iter().map(Into::into).collect())
798    }
799
800    // ── Signals ─────────────────────────────────────────────
801
802    async fn send_signal(&self, sig: &WorkflowSignal) -> Result<i64> {
803        let row: (i64,) = sqlx::query_as(
804            "INSERT INTO workflow_signals (workflow_id, name, payload, consumed, received_at) VALUES ($1, $2, $3, FALSE, $4) RETURNING id",
805        )
806        .bind(&sig.workflow_id)
807        .bind(&sig.name)
808        .bind(&sig.payload)
809        .bind(sig.received_at)
810        .fetch_one(&self.pool)
811        .await?;
812        Ok(row.0)
813    }
814
815    async fn consume_signals(
816        &self,
817        workflow_id: &str,
818        name: &str,
819    ) -> Result<Vec<WorkflowSignal>> {
820        let rows = sqlx::query_as::<_, PgSignalRow>(
821            "UPDATE workflow_signals SET consumed = TRUE
822             WHERE workflow_id = $1 AND name = $2 AND consumed = FALSE
823             RETURNING id, workflow_id, name, payload, consumed, received_at",
824        )
825        .bind(workflow_id)
826        .bind(name)
827        .fetch_all(&self.pool)
828        .await?;
829        Ok(rows.into_iter().map(Into::into).collect())
830    }
831
832    // ── Schedules ───────────────────────────────────────────
833
834    async fn create_schedule(&self, sched: &WorkflowSchedule) -> Result<()> {
835        sqlx::query(
836            "INSERT INTO workflow_schedules (namespace, name, workflow_type, cron_expr, timezone, input, task_queue, overlap_policy, paused, last_run_at, next_run_at, last_workflow_id, created_at)
837             VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)",
838        )
839        .bind(&sched.namespace)
840        .bind(&sched.name)
841        .bind(&sched.workflow_type)
842        .bind(&sched.cron_expr)
843        .bind(&sched.timezone)
844        .bind(&sched.input)
845        .bind(&sched.task_queue)
846        .bind(&sched.overlap_policy)
847        .bind(sched.paused)
848        .bind(sched.last_run_at)
849        .bind(sched.next_run_at)
850        .bind(&sched.last_workflow_id)
851        .bind(sched.created_at)
852        .execute(&self.pool)
853        .await?;
854        Ok(())
855    }
856
857    async fn get_schedule(&self, namespace: &str, name: &str) -> Result<Option<WorkflowSchedule>> {
858        let row = sqlx::query_as::<_, PgScheduleRow>(
859            "SELECT namespace, name, workflow_type, cron_expr, timezone, input, task_queue, overlap_policy, paused, last_run_at, next_run_at, last_workflow_id, created_at FROM workflow_schedules WHERE namespace = $1 AND name = $2",
860        )
861        .bind(namespace)
862        .bind(name)
863        .fetch_optional(&self.pool)
864        .await?;
865        Ok(row.map(Into::into))
866    }
867
868    async fn list_schedules(&self, namespace: &str) -> Result<Vec<WorkflowSchedule>> {
869        let rows = sqlx::query_as::<_, PgScheduleRow>(
870            "SELECT namespace, name, workflow_type, cron_expr, timezone, input, task_queue, overlap_policy, paused, last_run_at, next_run_at, last_workflow_id, created_at FROM workflow_schedules WHERE namespace = $1 ORDER BY name",
871        )
872        .bind(namespace)
873        .fetch_all(&self.pool)
874        .await?;
875        Ok(rows.into_iter().map(Into::into).collect())
876    }
877
878    async fn update_schedule_last_run(
879        &self,
880        namespace: &str,
881        name: &str,
882        last_run_at: f64,
883        next_run_at: f64,
884        workflow_id: &str,
885    ) -> Result<()> {
886        sqlx::query(
887            "UPDATE workflow_schedules SET last_run_at = $1, next_run_at = $2, last_workflow_id = $3 WHERE namespace = $4 AND name = $5",
888        )
889        .bind(last_run_at)
890        .bind(next_run_at)
891        .bind(workflow_id)
892        .bind(namespace)
893        .bind(name)
894        .execute(&self.pool)
895        .await?;
896        Ok(())
897    }
898
899    async fn delete_schedule(&self, namespace: &str, name: &str) -> Result<bool> {
900        let res = sqlx::query("DELETE FROM workflow_schedules WHERE namespace = $1 AND name = $2")
901            .bind(namespace)
902            .bind(name)
903            .execute(&self.pool)
904            .await?;
905        Ok(res.rows_affected() > 0)
906    }
907
908    async fn list_archivable_workflows(
909        &self,
910        cutoff: f64,
911        limit: i64,
912    ) -> Result<Vec<WorkflowRecord>> {
913        let rows = sqlx::query_as::<_, PgWorkflowRow>(
914            "SELECT id, namespace, run_id, workflow_type, task_queue, status, input, result, error, parent_id, claimed_by, search_attributes, archived_at, archive_uri, created_at, updated_at, completed_at
915             FROM workflows
916             WHERE status IN ('COMPLETED', 'FAILED', 'CANCELLED', 'TIMED_OUT')
917               AND completed_at IS NOT NULL
918               AND completed_at < $1
919               AND archived_at IS NULL
920             ORDER BY completed_at ASC
921             LIMIT $2",
922        )
923        .bind(cutoff)
924        .bind(limit)
925        .fetch_all(&self.pool)
926        .await?;
927        Ok(rows.into_iter().map(Into::into).collect())
928    }
929
930    async fn mark_archived_and_purge(
931        &self,
932        workflow_id: &str,
933        archive_uri: &str,
934        archived_at: f64,
935    ) -> Result<()> {
936        let mut tx = self.pool.begin().await?;
937        sqlx::query("DELETE FROM workflow_events WHERE workflow_id = $1")
938            .bind(workflow_id)
939            .execute(&mut *tx)
940            .await?;
941        sqlx::query("DELETE FROM workflow_activities WHERE workflow_id = $1")
942            .bind(workflow_id)
943            .execute(&mut *tx)
944            .await?;
945        sqlx::query("DELETE FROM workflow_timers WHERE workflow_id = $1")
946            .bind(workflow_id)
947            .execute(&mut *tx)
948            .await?;
949        sqlx::query("DELETE FROM workflow_signals WHERE workflow_id = $1")
950            .bind(workflow_id)
951            .execute(&mut *tx)
952            .await?;
953        sqlx::query("DELETE FROM workflow_snapshots WHERE workflow_id = $1")
954            .bind(workflow_id)
955            .execute(&mut *tx)
956            .await?;
957        sqlx::query(
958            "UPDATE workflows SET archived_at = $1, archive_uri = $2 WHERE id = $3",
959        )
960        .bind(archived_at)
961        .bind(archive_uri)
962        .bind(workflow_id)
963        .execute(&mut *tx)
964        .await?;
965        tx.commit().await?;
966        Ok(())
967    }
968
969    async fn upsert_search_attributes(
970        &self,
971        workflow_id: &str,
972        patch_json: &str,
973    ) -> Result<()> {
974        let current: Option<(Option<String>,)> =
975            sqlx::query_as("SELECT search_attributes FROM workflows WHERE id = $1")
976                .bind(workflow_id)
977                .fetch_optional(&self.pool)
978                .await?;
979        let merged = crate::store::sqlite::merge_search_attrs(
980            current.and_then(|(s,)| s).as_deref(),
981            patch_json,
982        )?;
983        sqlx::query("UPDATE workflows SET search_attributes = $1 WHERE id = $2")
984            .bind(merged)
985            .bind(workflow_id)
986            .execute(&self.pool)
987            .await?;
988        Ok(())
989    }
990
991    async fn update_schedule(
992        &self,
993        namespace: &str,
994        name: &str,
995        patch: &SchedulePatch,
996    ) -> Result<Option<WorkflowSchedule>> {
997        let mut sets: Vec<String> = Vec::new();
998        let mut idx = 1usize;
999        if patch.cron_expr.is_some() {
1000            sets.push(format!("cron_expr = ${idx}"));
1001            idx += 1;
1002        }
1003        if patch.timezone.is_some() {
1004            sets.push(format!("timezone = ${idx}"));
1005            idx += 1;
1006        }
1007        if patch.input.is_some() {
1008            sets.push(format!("input = ${idx}"));
1009            idx += 1;
1010        }
1011        if patch.task_queue.is_some() {
1012            sets.push(format!("task_queue = ${idx}"));
1013            idx += 1;
1014        }
1015        if patch.overlap_policy.is_some() {
1016            sets.push(format!("overlap_policy = ${idx}"));
1017            idx += 1;
1018        }
1019        if sets.is_empty() {
1020            return self.get_schedule(namespace, name).await;
1021        }
1022        let sql = format!(
1023            "UPDATE workflow_schedules SET {} WHERE namespace = ${} AND name = ${}",
1024            sets.join(", "),
1025            idx,
1026            idx + 1
1027        );
1028        let mut q = sqlx::query(&sql);
1029        if let Some(ref v) = patch.cron_expr {
1030            q = q.bind(v);
1031        }
1032        if let Some(ref v) = patch.timezone {
1033            q = q.bind(v);
1034        }
1035        if let Some(ref v) = patch.input {
1036            q = q.bind(v.to_string());
1037        }
1038        if let Some(ref v) = patch.task_queue {
1039            q = q.bind(v);
1040        }
1041        if let Some(ref v) = patch.overlap_policy {
1042            q = q.bind(v);
1043        }
1044        let res = q
1045            .bind(namespace)
1046            .bind(name)
1047            .execute(&self.pool)
1048            .await?;
1049        if res.rows_affected() == 0 {
1050            return Ok(None);
1051        }
1052        self.get_schedule(namespace, name).await
1053    }
1054
1055    async fn set_schedule_paused(
1056        &self,
1057        namespace: &str,
1058        name: &str,
1059        paused: bool,
1060    ) -> Result<Option<WorkflowSchedule>> {
1061        let res = sqlx::query(
1062            "UPDATE workflow_schedules SET paused = $1 WHERE namespace = $2 AND name = $3",
1063        )
1064        .bind(paused)
1065        .bind(namespace)
1066        .bind(name)
1067        .execute(&self.pool)
1068        .await?;
1069        if res.rows_affected() == 0 {
1070            return Ok(None);
1071        }
1072        self.get_schedule(namespace, name).await
1073    }
1074
1075    // ── Workers ─────────────────────────────────────────────
1076
1077    async fn register_worker(&self, w: &WorkflowWorker) -> Result<()> {
1078        sqlx::query(
1079            "INSERT INTO workflow_workers (id, namespace, identity, task_queue, workflows, activities, max_concurrent_workflows, max_concurrent_activities, active_tasks, last_heartbeat, registered_at)
1080             VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
1081             ON CONFLICT (id) DO UPDATE SET last_heartbeat = EXCLUDED.last_heartbeat, identity = EXCLUDED.identity",
1082        )
1083        .bind(&w.id)
1084        .bind(&w.namespace)
1085        .bind(&w.identity)
1086        .bind(&w.task_queue)
1087        .bind(&w.workflows)
1088        .bind(&w.activities)
1089        .bind(w.max_concurrent_workflows)
1090        .bind(w.max_concurrent_activities)
1091        .bind(w.active_tasks)
1092        .bind(w.last_heartbeat)
1093        .bind(w.registered_at)
1094        .execute(&self.pool)
1095        .await?;
1096        Ok(())
1097    }
1098
1099    async fn heartbeat_worker(&self, id: &str, now: f64) -> Result<()> {
1100        sqlx::query("UPDATE workflow_workers SET last_heartbeat = $1 WHERE id = $2")
1101            .bind(now)
1102            .bind(id)
1103            .execute(&self.pool)
1104            .await?;
1105        Ok(())
1106    }
1107
1108    async fn list_workers(&self, namespace: &str) -> Result<Vec<WorkflowWorker>> {
1109        let rows = sqlx::query_as::<_, PgWorkerRow>(
1110            "SELECT id, namespace, identity, task_queue, workflows, activities, max_concurrent_workflows, max_concurrent_activities, active_tasks, last_heartbeat, registered_at FROM workflow_workers WHERE namespace = $1 ORDER BY registered_at",
1111        )
1112        .bind(namespace)
1113        .fetch_all(&self.pool)
1114        .await?;
1115        Ok(rows.into_iter().map(Into::into).collect())
1116    }
1117
1118    async fn remove_dead_workers(&self, cutoff: f64) -> Result<Vec<String>> {
1119        let rows: Vec<(String,)> =
1120            sqlx::query_as("SELECT id FROM workflow_workers WHERE last_heartbeat < $1")
1121                .bind(cutoff)
1122                .fetch_all(&self.pool)
1123                .await?;
1124        let ids: Vec<String> = rows.into_iter().map(|r| r.0).collect();
1125        if !ids.is_empty() {
1126            sqlx::query("DELETE FROM workflow_workers WHERE last_heartbeat < $1")
1127                .bind(cutoff)
1128                .execute(&self.pool)
1129                .await?;
1130        }
1131        Ok(ids)
1132    }
1133
1134    // ── API Keys ────────────────────────────────────────────
1135
1136    async fn create_api_key(
1137        &self,
1138        key_hash: &str,
1139        prefix: &str,
1140        label: Option<&str>,
1141        created_at: f64,
1142    ) -> Result<()> {
1143        sqlx::query("INSERT INTO api_keys (key_hash, prefix, label, created_at) VALUES ($1, $2, $3, $4)")
1144            .bind(key_hash)
1145            .bind(prefix)
1146            .bind(label)
1147            .bind(created_at)
1148            .execute(&self.pool)
1149            .await?;
1150        Ok(())
1151    }
1152
1153    async fn validate_api_key(&self, key_hash: &str) -> Result<bool> {
1154        let row: Option<(i64,)> =
1155            sqlx::query_as("SELECT 1::BIGINT FROM api_keys WHERE key_hash = $1")
1156                .bind(key_hash)
1157                .fetch_optional(&self.pool)
1158                .await?;
1159        Ok(row.is_some())
1160    }
1161
1162    async fn list_api_keys(&self) -> Result<Vec<crate::store::ApiKeyRecord>> {
1163        let rows = sqlx::query_as::<_, (String, Option<String>, f64)>(
1164            "SELECT prefix, label, created_at FROM api_keys ORDER BY created_at DESC",
1165        )
1166        .fetch_all(&self.pool)
1167        .await?;
1168        Ok(rows
1169            .into_iter()
1170            .map(|(prefix, label, created_at)| crate::store::ApiKeyRecord {
1171                prefix,
1172                label,
1173                created_at,
1174            })
1175            .collect())
1176    }
1177
1178    async fn revoke_api_key(&self, prefix: &str) -> Result<bool> {
1179        let res = sqlx::query("DELETE FROM api_keys WHERE prefix = $1")
1180            .bind(prefix)
1181            .execute(&self.pool)
1182            .await?;
1183        Ok(res.rows_affected() > 0)
1184    }
1185
1186    async fn api_keys_empty(&self) -> Result<bool> {
1187        let row: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM api_keys")
1188            .fetch_one(&self.pool)
1189            .await?;
1190        Ok(row.0 == 0)
1191    }
1192
1193    async fn get_api_key_by_label(
1194        &self,
1195        label: &str,
1196    ) -> Result<Option<crate::store::ApiKeyRecord>> {
1197        let row: Option<(String, Option<String>, f64)> = sqlx::query_as(
1198            "SELECT prefix, label, created_at FROM api_keys WHERE label = $1 LIMIT 1",
1199        )
1200        .bind(label)
1201        .fetch_optional(&self.pool)
1202        .await?;
1203        Ok(row.map(|(prefix, label, created_at)| crate::store::ApiKeyRecord {
1204            prefix,
1205            label,
1206            created_at,
1207        }))
1208    }
1209
1210    // ── Child Workflows ─────────────────────────────────────
1211
1212    async fn list_child_workflows(&self, parent_id: &str) -> Result<Vec<WorkflowRecord>> {
1213        let rows = sqlx::query_as::<_, PgWorkflowRow>(
1214            "SELECT id, namespace, run_id, workflow_type, task_queue, status, input, result, error, parent_id, claimed_by, search_attributes, archived_at, archive_uri, created_at, updated_at, completed_at
1215             FROM workflows WHERE parent_id = $1 ORDER BY created_at ASC",
1216        )
1217        .bind(parent_id)
1218        .fetch_all(&self.pool)
1219        .await?;
1220        Ok(rows.into_iter().map(Into::into).collect())
1221    }
1222
1223    // ── Snapshots ───────────────────────────────────────────
1224
1225    async fn create_snapshot(
1226        &self,
1227        workflow_id: &str,
1228        event_seq: i32,
1229        state_json: &str,
1230    ) -> Result<()> {
1231        sqlx::query(
1232            "INSERT INTO workflow_snapshots (workflow_id, event_seq, state_json, created_at)
1233             VALUES ($1, $2, $3, $4)
1234             ON CONFLICT (workflow_id, event_seq) DO UPDATE SET state_json = EXCLUDED.state_json, created_at = EXCLUDED.created_at",
1235        )
1236        .bind(workflow_id)
1237        .bind(event_seq)
1238        .bind(state_json)
1239        .bind(timestamp_now())
1240        .execute(&self.pool)
1241        .await?;
1242        Ok(())
1243    }
1244
1245    async fn get_latest_snapshot(
1246        &self,
1247        workflow_id: &str,
1248    ) -> Result<Option<WorkflowSnapshot>> {
1249        let row = sqlx::query_as::<_, (String, i32, String, f64)>(
1250            "SELECT workflow_id, event_seq, state_json, created_at
1251             FROM workflow_snapshots WHERE workflow_id = $1
1252             ORDER BY event_seq DESC LIMIT 1",
1253        )
1254        .bind(workflow_id)
1255        .fetch_optional(&self.pool)
1256        .await?;
1257
1258        Ok(row.map(|(workflow_id, event_seq, state_json, created_at)| WorkflowSnapshot {
1259            workflow_id,
1260            event_seq,
1261            state_json,
1262            created_at,
1263        }))
1264    }
1265
1266    // ── Queue Stats ─────────────────────────────────────────
1267
1268    async fn get_queue_stats(&self, namespace: &str) -> Result<Vec<crate::store::QueueStats>> {
1269        let rows = sqlx::query_as::<_, (String, i64, i64, i64)>(
1270            "SELECT
1271                a.task_queue AS queue,
1272                SUM(CASE WHEN a.status = 'PENDING' THEN 1 ELSE 0 END) AS pending,
1273                SUM(CASE WHEN a.status = 'RUNNING' THEN 1 ELSE 0 END) AS running,
1274                (SELECT COUNT(*) FROM workflow_workers w WHERE w.task_queue = a.task_queue AND w.namespace = $1) AS workers
1275             FROM workflow_activities a
1276             JOIN workflows wf ON a.workflow_id = wf.id AND wf.namespace = $1
1277             GROUP BY a.task_queue",
1278        )
1279        .bind(namespace)
1280        .fetch_all(&self.pool)
1281        .await?;
1282
1283        Ok(rows
1284            .into_iter()
1285            .map(|(queue, pending, running, workers)| crate::store::QueueStats {
1286                queue,
1287                pending_activities: pending,
1288                running_activities: running,
1289                workers,
1290            })
1291            .collect())
1292    }
1293
1294    // ── Leader Election ─────────────────────────────────────
1295
1296    async fn try_acquire_scheduler_lock(&self) -> Result<bool> {
1297        // pg_try_advisory_lock is session-scoped — only one connection
1298        // in the pool will hold the lock. In a multi-replica Kubernetes
1299        // deployment, only one pod's connection wins.
1300        let row: (bool,) =
1301            sqlx::query_as("SELECT pg_try_advisory_lock(42)")
1302                .fetch_one(&self.pool)
1303                .await?;
1304        Ok(row.0)
1305    }
1306
1307    fn subscribe_runnable<'a>(
1308        &'a self,
1309        namespace: &'a str,
1310    ) -> impl std::future::Future<Output = futures_util::stream::BoxStream<'a, String>>
1311           + Send
1312           + 'a {
1313        use futures_util::StreamExt;
1314        let pool = self.pool.clone();
1315        let ns = namespace.to_string();
1316        async move {
1317            let channel = format!("assay_runnable_{ns}");
1318            let mut listener = match sqlx::postgres::PgListener::connect_with(&pool).await {
1319                Ok(l) => l,
1320                Err(e) => {
1321                    tracing::error!(?e, "subscribe_runnable: pg listener connect failed");
1322                    return futures_util::stream::empty().boxed();
1323                }
1324            };
1325            if let Err(e) = listener.listen(&channel).await {
1326                tracing::error!(?e, %channel, "subscribe_runnable: LISTEN failed");
1327                return futures_util::stream::empty().boxed();
1328            }
1329            // LISTEN is now registered. From here any `pg_notify` on this
1330            // channel reaches us — hand the caller a stream that just reads.
1331            async_stream::stream! {
1332                loop {
1333                    match listener.recv().await {
1334                        Ok(n) => yield n.payload().to_string(),
1335                        Err(e) => {
1336                            tracing::warn!(?e, "subscribe_runnable: recv error, reconnecting");
1337                            // PgListener handles reconnection internally; a
1338                            // recv error means the connection was dropped.
1339                            // End the stream so the scheduler falls back to
1340                            // its timer-heap wake-up.
1341                            break;
1342                        }
1343                    }
1344                }
1345            }
1346            .boxed()
1347        }
1348    }
1349
1350    fn subscribe_tasks<'a>(
1351        &'a self,
1352        queue_names: &'a [&'a str],
1353    ) -> impl std::future::Future<Output = futures_util::stream::BoxStream<'a, String>>
1354           + Send
1355           + 'a {
1356        use futures_util::StreamExt;
1357        let pool = self.pool.clone();
1358        let channels: Vec<String> = queue_names
1359            .iter()
1360            .map(|q| format!("assay_task_{q}"))
1361            .collect();
1362        async move {
1363            if channels.is_empty() {
1364                return futures_util::stream::empty().boxed();
1365            }
1366            let mut listener = match sqlx::postgres::PgListener::connect_with(&pool).await {
1367                Ok(l) => l,
1368                Err(e) => {
1369                    tracing::error!(?e, "subscribe_tasks: pg listener connect failed");
1370                    return futures_util::stream::empty().boxed();
1371                }
1372            };
1373            let channel_refs: Vec<&str> = channels.iter().map(|s| s.as_str()).collect();
1374            if let Err(e) = listener.listen_all(channel_refs).await {
1375                tracing::error!(?e, "subscribe_tasks: LISTEN failed");
1376                return futures_util::stream::empty().boxed();
1377            }
1378            async_stream::stream! {
1379                loop {
1380                    match listener.recv().await {
1381                        Ok(n) => yield n.payload().to_string(),
1382                        Err(e) => {
1383                            tracing::warn!(?e, "subscribe_tasks: recv error");
1384                            break;
1385                        }
1386                    }
1387                }
1388            }
1389            .boxed()
1390        }
1391    }
1392}
1393
1394fn timestamp_now() -> f64 {
1395    std::time::SystemTime::now()
1396        .duration_since(std::time::UNIX_EPOCH)
1397        .unwrap()
1398        .as_secs_f64()
1399}
1400
1401// ── Postgres row types (sqlx::FromRow) ──────────────────────
1402
1403#[derive(sqlx::FromRow)]
1404struct PgWorkflowRow {
1405    id: String,
1406    namespace: String,
1407    run_id: String,
1408    workflow_type: String,
1409    task_queue: String,
1410    status: String,
1411    input: Option<String>,
1412    result: Option<String>,
1413    error: Option<String>,
1414    parent_id: Option<String>,
1415    claimed_by: Option<String>,
1416    search_attributes: Option<String>,
1417    archived_at: Option<f64>,
1418    archive_uri: Option<String>,
1419    created_at: f64,
1420    updated_at: f64,
1421    completed_at: Option<f64>,
1422}
1423
1424impl From<PgWorkflowRow> for WorkflowRecord {
1425    fn from(r: PgWorkflowRow) -> Self {
1426        Self {
1427            id: r.id,
1428            namespace: r.namespace,
1429            run_id: r.run_id,
1430            workflow_type: r.workflow_type,
1431            task_queue: r.task_queue,
1432            status: r.status,
1433            input: r.input,
1434            result: r.result,
1435            error: r.error,
1436            parent_id: r.parent_id,
1437            claimed_by: r.claimed_by,
1438            search_attributes: r.search_attributes,
1439            archived_at: r.archived_at,
1440            archive_uri: r.archive_uri,
1441            created_at: r.created_at,
1442            updated_at: r.updated_at,
1443            completed_at: r.completed_at,
1444        }
1445    }
1446}
1447
1448#[derive(sqlx::FromRow)]
1449struct PgEventRow {
1450    id: i64,
1451    workflow_id: String,
1452    seq: i32,
1453    event_type: String,
1454    payload: Option<String>,
1455    timestamp: f64,
1456}
1457
1458impl From<PgEventRow> for WorkflowEvent {
1459    fn from(r: PgEventRow) -> Self {
1460        Self {
1461            id: Some(r.id),
1462            workflow_id: r.workflow_id,
1463            seq: r.seq,
1464            event_type: r.event_type,
1465            payload: r.payload,
1466            timestamp: r.timestamp,
1467        }
1468    }
1469}
1470
1471#[derive(sqlx::FromRow)]
1472struct PgActivityRow {
1473    id: i64,
1474    workflow_id: String,
1475    seq: i32,
1476    name: String,
1477    task_queue: String,
1478    input: Option<String>,
1479    status: String,
1480    result: Option<String>,
1481    error: Option<String>,
1482    attempt: i32,
1483    max_attempts: i32,
1484    initial_interval_secs: f64,
1485    backoff_coefficient: f64,
1486    start_to_close_secs: f64,
1487    heartbeat_timeout_secs: Option<f64>,
1488    claimed_by: Option<String>,
1489    scheduled_at: f64,
1490    started_at: Option<f64>,
1491    completed_at: Option<f64>,
1492    last_heartbeat: Option<f64>,
1493}
1494
1495impl From<PgActivityRow> for WorkflowActivity {
1496    fn from(r: PgActivityRow) -> Self {
1497        Self {
1498            id: Some(r.id),
1499            workflow_id: r.workflow_id,
1500            seq: r.seq,
1501            name: r.name,
1502            task_queue: r.task_queue,
1503            input: r.input,
1504            status: r.status,
1505            result: r.result,
1506            error: r.error,
1507            attempt: r.attempt,
1508            max_attempts: r.max_attempts,
1509            initial_interval_secs: r.initial_interval_secs,
1510            backoff_coefficient: r.backoff_coefficient,
1511            start_to_close_secs: r.start_to_close_secs,
1512            heartbeat_timeout_secs: r.heartbeat_timeout_secs,
1513            claimed_by: r.claimed_by,
1514            scheduled_at: r.scheduled_at,
1515            started_at: r.started_at,
1516            completed_at: r.completed_at,
1517            last_heartbeat: r.last_heartbeat,
1518        }
1519    }
1520}
1521
1522#[derive(sqlx::FromRow)]
1523struct PgTimerRow {
1524    id: i64,
1525    workflow_id: String,
1526    seq: i32,
1527    fire_at: f64,
1528    fired: bool,
1529}
1530
1531impl From<PgTimerRow> for WorkflowTimer {
1532    fn from(r: PgTimerRow) -> Self {
1533        Self {
1534            id: Some(r.id),
1535            workflow_id: r.workflow_id,
1536            seq: r.seq,
1537            fire_at: r.fire_at,
1538            fired: r.fired,
1539        }
1540    }
1541}
1542
1543#[derive(sqlx::FromRow)]
1544struct PgSignalRow {
1545    id: i64,
1546    workflow_id: String,
1547    name: String,
1548    payload: Option<String>,
1549    consumed: bool,
1550    received_at: f64,
1551}
1552
1553impl From<PgSignalRow> for WorkflowSignal {
1554    fn from(r: PgSignalRow) -> Self {
1555        Self {
1556            id: Some(r.id),
1557            workflow_id: r.workflow_id,
1558            name: r.name,
1559            payload: r.payload,
1560            consumed: r.consumed,
1561            received_at: r.received_at,
1562        }
1563    }
1564}
1565
1566#[derive(sqlx::FromRow)]
1567struct PgScheduleRow {
1568    namespace: String,
1569    name: String,
1570    workflow_type: String,
1571    cron_expr: String,
1572    timezone: String,
1573    input: Option<String>,
1574    task_queue: String,
1575    overlap_policy: String,
1576    paused: bool,
1577    last_run_at: Option<f64>,
1578    next_run_at: Option<f64>,
1579    last_workflow_id: Option<String>,
1580    created_at: f64,
1581}
1582
1583impl From<PgScheduleRow> for WorkflowSchedule {
1584    fn from(r: PgScheduleRow) -> Self {
1585        Self {
1586            namespace: r.namespace,
1587            name: r.name,
1588            workflow_type: r.workflow_type,
1589            cron_expr: r.cron_expr,
1590            timezone: r.timezone,
1591            input: r.input,
1592            task_queue: r.task_queue,
1593            overlap_policy: r.overlap_policy,
1594            paused: r.paused,
1595            last_run_at: r.last_run_at,
1596            next_run_at: r.next_run_at,
1597            last_workflow_id: r.last_workflow_id,
1598            created_at: r.created_at,
1599        }
1600    }
1601}
1602
1603#[derive(sqlx::FromRow)]
1604struct PgWorkerRow {
1605    id: String,
1606    namespace: String,
1607    identity: String,
1608    task_queue: String,
1609    workflows: Option<String>,
1610    activities: Option<String>,
1611    max_concurrent_workflows: i32,
1612    max_concurrent_activities: i32,
1613    active_tasks: i32,
1614    last_heartbeat: f64,
1615    registered_at: f64,
1616}
1617
1618impl From<PgWorkerRow> for WorkflowWorker {
1619    fn from(r: PgWorkerRow) -> Self {
1620        Self {
1621            id: r.id,
1622            namespace: r.namespace,
1623            identity: r.identity,
1624            task_queue: r.task_queue,
1625            workflows: r.workflows,
1626            activities: r.activities,
1627            max_concurrent_workflows: r.max_concurrent_workflows,
1628            max_concurrent_activities: r.max_concurrent_activities,
1629            active_tasks: r.active_tasks,
1630            last_heartbeat: r.last_heartbeat,
1631            registered_at: r.registered_at,
1632        }
1633    }
1634}
1635
1636#[cfg(test)]
1637mod tests {
1638    use super::*;
1639
1640    #[test]
1641    fn sanitise_schema_keeps_statements_intact() {
1642        let input = "CREATE TABLE foo (x INT);\nCREATE INDEX idx_foo ON foo(x);\n";
1643        let out = sanitise_schema(input);
1644        assert_eq!(out.len(), 2);
1645        assert!(out[0].starts_with("CREATE TABLE foo"));
1646        assert!(out[1].starts_with("CREATE INDEX idx_foo"));
1647    }
1648
1649    #[test]
1650    fn sanitise_schema_drops_pure_comment_lines() {
1651        let input = "-- header comment\nCREATE TABLE foo (x INT);\n-- trailing comment\n";
1652        let out = sanitise_schema(input);
1653        assert_eq!(out.len(), 1);
1654        assert!(out[0].starts_with("CREATE TABLE foo"));
1655    }
1656
1657    #[test]
1658    fn sanitise_schema_ignores_semicolons_inside_comment_prose() {
1659        // Regression: the exact shape that broke v0.11.3–v0.11.5 in production.
1660        // `-- foo; bar` used to split into "foo" and " bar" fragments, the second
1661        // of which was executed as SQL and rejected with `syntax error at or near "bar"`.
1662        let input = "\
1663CREATE TABLE foo (x INT);
1664-- Idempotent across startups; fresh installs pick the column up from the
1665-- CREATE TABLE above so the ADD is a no-op.
1666";
1667        let out = sanitise_schema(input);
1668        assert_eq!(
1669            out.len(),
1670            1,
1671            "expected 1 real statement, got {}: {:?}",
1672            out.len(),
1673            out
1674        );
1675        assert!(out[0].starts_with("CREATE TABLE foo"));
1676    }
1677
1678    #[test]
1679    fn sanitise_schema_drops_indented_comment_lines() {
1680        let input = "  -- indented comment\n\tCREATE TABLE foo (x INT);\n";
1681        let out = sanitise_schema(input);
1682        assert_eq!(out.len(), 1);
1683        assert!(out[0].contains("CREATE TABLE foo"));
1684    }
1685
1686    #[test]
1687    fn sanitise_schema_real_constant_produces_only_ddl() {
1688        // The real SCHEMA constant must not produce any statement whose first
1689        // token isn't a recognised SQL keyword. A prose fragment leaking in
1690        // (e.g. "fresh installs...") means the filter regressed.
1691        for stmt in sanitise_schema(SCHEMA) {
1692            let first_word = stmt
1693                .split_whitespace()
1694                .next()
1695                .expect("non-empty statement")
1696                .to_uppercase();
1697            assert!(
1698                matches!(
1699                    first_word.as_str(),
1700                    "CREATE" | "INSERT" | "UPDATE" | "DROP" | "ALTER" | "WITH"
1701                ),
1702                "SCHEMA produced non-DDL statement starting with {first_word:?}: {stmt:?}"
1703            );
1704        }
1705    }
1706}