1use anyhow::Result;
2use sqlx::SqlitePool;
3use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};
4
5use crate::store::{
6 NOT_A_SETTLEMENT, NamespaceRecord, NamespaceStats, QueueStats, RetryEvent, WorkflowStore,
7 payload_activity_id, retry_denial, settle_outcome,
8};
9use crate::types::*;
10
11const RETRY_ACTIVITY_SELECT: &str = "SELECT id, workflow_id, seq, name, task_queue, input, status, result, error, attempt, max_attempts, initial_interval_secs, backoff_coefficient, start_to_close_secs, heartbeat_timeout_secs, claimed_by, scheduled_at, started_at, completed_at, last_heartbeat FROM workflow.activities WHERE workflow_id = ? AND status = 'FAILED' ORDER BY seq DESC LIMIT 1";
12const UNSETTLED_ACTIVITY_SELECT: &str = "SELECT a.id, a.workflow_id, a.seq, a.name, a.task_queue, a.input, a.status, a.result, a.error, a.attempt, a.max_attempts, a.initial_interval_secs, a.backoff_coefficient, a.start_to_close_secs, a.heartbeat_timeout_secs, a.claimed_by, a.scheduled_at, a.started_at, a.completed_at, a.last_heartbeat FROM workflow.activities a JOIN workflow.workflows w ON w.id = a.workflow_id WHERE a.status IN ('COMPLETED', 'FAILED') AND w.status NOT IN ('COMPLETED', 'FAILED', 'CANCELLED', 'TIMED_OUT') AND w.archived_at IS NULL AND NOT EXISTS (SELECT 1 FROM workflow.events e WHERE e.workflow_id = a.workflow_id AND e.activity_id = a.id AND e.event_type IN ('ActivityCompleted', 'ActivityFailed')) ORDER BY a.completed_at ASC LIMIT ?";
16const RETRY_ACTIVITY_UPDATE: &str = "UPDATE workflow.activities SET status = 'PENDING', result = NULL, error = NULL, attempt = 1, claimed_by = NULL, scheduled_at = ?, started_at = NULL, completed_at = NULL, last_heartbeat = NULL WHERE id = ? RETURNING id, workflow_id, seq, name, task_queue, input, status, result, error, attempt, max_attempts, initial_interval_secs, backoff_coefficient, start_to_close_secs, heartbeat_timeout_secs, claimed_by, scheduled_at, started_at, completed_at, last_heartbeat";
17
18const SCHEMA: &str = r#"
31CREATE TABLE IF NOT EXISTS workflow.namespaces (
32 name TEXT PRIMARY KEY,
33 created_at REAL NOT NULL
34);
35
36INSERT OR IGNORE INTO workflow.namespaces (name, created_at)
37 VALUES ('main', strftime('%s', 'now'));
38
39CREATE TABLE IF NOT EXISTS workflow.workflows (
40 id TEXT PRIMARY KEY,
41 namespace TEXT NOT NULL DEFAULT 'main',
42 run_id TEXT NOT NULL,
43 workflow_type TEXT NOT NULL,
44 task_queue TEXT NOT NULL DEFAULT 'main',
45 status TEXT NOT NULL DEFAULT 'PENDING',
46 input TEXT,
47 result TEXT,
48 error TEXT,
49 parent_id TEXT,
50 claimed_by TEXT,
51 search_attributes TEXT,
52 archived_at REAL,
53 archive_uri TEXT,
54 -- Workflow-task dispatch (Phase 9): a workflow is "dispatchable" when
55 -- it has new events a worker needs to replay against. Set true on
56 -- start, on activity completion, on timer fire, on signal arrival.
57 -- Cleared when a worker claims the dispatch lease.
58 needs_dispatch INTEGER NOT NULL DEFAULT 0,
59 dispatch_claimed_by TEXT,
60 dispatch_last_heartbeat REAL,
61 created_at REAL NOT NULL,
62 updated_at REAL NOT NULL,
63 completed_at REAL
64);
65CREATE INDEX IF NOT EXISTS workflow.idx_wf_status_queue ON workflows(status, task_queue);
66CREATE INDEX IF NOT EXISTS workflow.idx_wf_namespace ON workflows(namespace);
67CREATE INDEX IF NOT EXISTS workflow.idx_wf_dispatch ON workflows(task_queue, needs_dispatch, dispatch_claimed_by);
68
69CREATE TABLE IF NOT EXISTS workflow.events (
70 id INTEGER PRIMARY KEY AUTOINCREMENT,
71 workflow_id TEXT NOT NULL REFERENCES workflows(id),
72 seq INTEGER NOT NULL,
73 event_type TEXT NOT NULL,
74 payload TEXT,
75 -- Set on ActivityCompleted / ActivityFailed only. Answers "did this
76 -- activity's terminal event land" without parsing payload JSON, which
77 -- is what the settle transaction and the reconciler both need.
78 activity_id INTEGER,
79 timestamp REAL NOT NULL
80);
81CREATE INDEX IF NOT EXISTS workflow.idx_wf_events_lookup ON events(workflow_id, seq);
82CREATE INDEX IF NOT EXISTS workflow.idx_wf_events_activity ON events(activity_id);
83
84CREATE TABLE IF NOT EXISTS workflow.activities (
85 id INTEGER PRIMARY KEY AUTOINCREMENT,
86 workflow_id TEXT NOT NULL REFERENCES workflows(id),
87 seq INTEGER NOT NULL,
88 name TEXT NOT NULL,
89 task_queue TEXT NOT NULL DEFAULT 'main',
90 input TEXT,
91 status TEXT NOT NULL DEFAULT 'PENDING',
92 result TEXT,
93 error TEXT,
94 attempt INTEGER NOT NULL DEFAULT 1,
95 max_attempts INTEGER NOT NULL DEFAULT 3,
96 initial_interval_secs REAL NOT NULL DEFAULT 1,
97 backoff_coefficient REAL NOT NULL DEFAULT 2,
98 start_to_close_secs REAL NOT NULL DEFAULT 300,
99 heartbeat_timeout_secs REAL,
100 claimed_by TEXT,
101 scheduled_at REAL NOT NULL,
102 started_at REAL,
103 completed_at REAL,
104 last_heartbeat REAL,
105 UNIQUE (workflow_id, seq)
106);
107CREATE INDEX IF NOT EXISTS workflow.idx_wf_act_pending ON activities(task_queue, status, scheduled_at);
108
109CREATE TABLE IF NOT EXISTS workflow.timers (
110 id INTEGER PRIMARY KEY AUTOINCREMENT,
111 workflow_id TEXT NOT NULL REFERENCES workflows(id),
112 seq INTEGER NOT NULL,
113 fire_at REAL NOT NULL,
114 fired INTEGER NOT NULL DEFAULT 0,
115 UNIQUE (workflow_id, seq)
116);
117CREATE INDEX IF NOT EXISTS workflow.idx_wf_timers_due ON timers(fire_at);
118
119CREATE TABLE IF NOT EXISTS workflow.signals (
120 id INTEGER PRIMARY KEY AUTOINCREMENT,
121 workflow_id TEXT NOT NULL REFERENCES workflows(id),
122 name TEXT NOT NULL,
123 payload TEXT,
124 consumed INTEGER NOT NULL DEFAULT 0,
125 received_at REAL NOT NULL
126);
127CREATE INDEX IF NOT EXISTS workflow.idx_wf_signals_lookup ON signals(workflow_id, name, consumed);
128
129CREATE TABLE IF NOT EXISTS workflow.schedules (
130 name TEXT NOT NULL,
131 namespace TEXT NOT NULL DEFAULT 'main',
132 workflow_type TEXT NOT NULL,
133 cron_expr TEXT NOT NULL,
134 timezone TEXT NOT NULL DEFAULT 'UTC',
135 input TEXT,
136 task_queue TEXT NOT NULL DEFAULT 'main',
137 overlap_policy TEXT NOT NULL DEFAULT 'skip',
138 paused INTEGER NOT NULL DEFAULT 0,
139 last_run_at REAL,
140 next_run_at REAL,
141 last_workflow_id TEXT,
142 created_at REAL NOT NULL,
143 PRIMARY KEY (namespace, name)
144);
145
146CREATE TABLE IF NOT EXISTS workflow.workers (
147 id TEXT PRIMARY KEY,
148 namespace TEXT NOT NULL DEFAULT 'main',
149 identity TEXT NOT NULL,
150 task_queue TEXT NOT NULL,
151 workflows TEXT,
152 activities TEXT,
153 max_concurrent_workflows INTEGER NOT NULL DEFAULT 10,
154 max_concurrent_activities INTEGER NOT NULL DEFAULT 10,
155 active_tasks INTEGER NOT NULL DEFAULT 0,
156 last_heartbeat REAL NOT NULL,
157 registered_at REAL NOT NULL
158);
159
160CREATE TABLE IF NOT EXISTS workflow.snapshots (
161 workflow_id TEXT NOT NULL REFERENCES workflows(id),
162 event_seq INTEGER NOT NULL,
163 state_json TEXT NOT NULL,
164 created_at REAL NOT NULL,
165 PRIMARY KEY (workflow_id, event_seq)
166);
167
168-- workflow.api_keys retired in plan-15 slice 3 (auth tokens come from
169-- the auth module).
170DROP TABLE IF EXISTS workflow.api_keys;
171
172CREATE TABLE IF NOT EXISTS engine.lock (
173 id INTEGER PRIMARY KEY CHECK (id = 1),
174 instance_id TEXT NOT NULL,
175 started_at REAL NOT NULL,
176 last_heartbeat REAL NOT NULL
177);
178
179CREATE TABLE IF NOT EXISTS engine.events (
180 id INTEGER PRIMARY KEY AUTOINCREMENT,
181 ts REAL NOT NULL DEFAULT (CAST(strftime('%s','now') AS REAL)),
182 namespace TEXT NOT NULL,
183 subsystem TEXT NOT NULL,
184 kind TEXT NOT NULL,
185 payload TEXT NOT NULL DEFAULT '{}'
186);
187CREATE INDEX IF NOT EXISTS engine.idx_engine_events_ns_id ON events(namespace, id);
188CREATE INDEX IF NOT EXISTS engine.idx_engine_events_ts_prune ON events(ts);
189"#;
190
191const LOCK_STALE_SECS: f64 = 60.0;
194const LOCK_HEARTBEAT_SECS: u64 = 15;
196
197#[derive(Clone)]
204pub struct SqliteStore {
205 pool: SqlitePool,
206 instance_id: String,
207}
208
209async fn build_default_pool(url: &str) -> Result<SqlitePool> {
222 use std::str::FromStr;
223 use std::sync::atomic::{AtomicU64, Ordering};
224
225 static SEQ: AtomicU64 = AtomicU64::new(0);
226 let suffix = format!(
227 "{}_{}",
228 std::process::id(),
229 SEQ.fetch_add(1, Ordering::Relaxed)
230 );
231 let engine_alias = format!("file:assay_engine_{suffix}?mode=memory&cache=shared");
232 let workflow_alias = format!("file:assay_workflow_{suffix}?mode=memory&cache=shared");
233
234 let opts = SqliteConnectOptions::from_str(url)?.create_if_missing(true);
235
236 let pool = SqlitePoolOptions::new()
237 .max_connections(1)
238 .after_connect(move |conn, _meta| {
239 let engine_alias = engine_alias.clone();
240 let workflow_alias = workflow_alias.clone();
241 Box::pin(async move {
242 use sqlx::Executor;
243 conn.execute(format!("ATTACH DATABASE '{engine_alias}' AS engine").as_str())
244 .await?;
245 conn.execute(format!("ATTACH DATABASE '{workflow_alias}' AS workflow").as_str())
246 .await?;
247 Ok(())
248 })
249 })
250 .connect_with(opts)
251 .await?;
252 Ok(pool)
253}
254
255impl SqliteStore {
256 pub async fn new(url: &str) -> Result<Self> {
262 let pool = build_default_pool(url).await?;
263 Self::from_attached_pool(pool).await
264 }
265
266 pub async fn from_attached_pool(pool: SqlitePool) -> Result<Self> {
271 let instance_id = format!("assay-{:016x}", {
272 use std::collections::hash_map::DefaultHasher;
273 use std::hash::{Hash, Hasher};
274 let mut h = DefaultHasher::new();
275 std::time::SystemTime::now().hash(&mut h);
276 std::process::id().hash(&mut h);
277 h.finish()
278 });
279 let store = Self { pool, instance_id };
280 store.migrate().await?;
281 Ok(store)
282 }
283
284 pub async fn from_pool(pool: SqlitePool) -> Result<Self> {
290 Self::from_attached_pool(pool).await
291 }
292
293 pub fn pool(&self) -> &SqlitePool {
296 &self.pool
297 }
298
299 pub async fn acquire_engine_lock(&self) -> Result<()> {
302 let now = timestamp_now();
303
304 let result = sqlx::query(
306 "INSERT INTO engine.lock (id, instance_id, started_at, last_heartbeat) VALUES (1, ?, ?, ?)",
307 )
308 .bind(&self.instance_id)
309 .bind(now)
310 .bind(now)
311 .execute(&self.pool)
312 .await;
313
314 match result {
315 Ok(_) => Ok(()),
316 Err(_) => {
317 let row: Option<(String, f64)> = sqlx::query_as(
319 "SELECT instance_id, last_heartbeat FROM engine.lock WHERE id = 1",
320 )
321 .fetch_optional(&self.pool)
322 .await?;
323
324 if let Some((existing_id, last_hb)) = row {
325 if now - last_hb > LOCK_STALE_SECS {
326 sqlx::query(
328 "UPDATE engine.lock SET instance_id = ?, started_at = ?, last_heartbeat = ? WHERE id = 1",
329 )
330 .bind(&self.instance_id)
331 .bind(now)
332 .bind(now)
333 .execute(&self.pool)
334 .await?;
335 tracing::warn!(
336 "Took over stale engine lock from {existing_id} (last heartbeat {:.0}s ago)",
337 now - last_hb
338 );
339 Ok(())
340 } else {
341 let age = now - last_hb;
342 anyhow::bail!(
343 "Another assay engine instance is already running (id: {existing_id}, \
344 last heartbeat {age:.0}s ago).\n\n\
345 SQLite only supports a single engine instance. For multi-instance \
346 deployment (Kubernetes, Docker Swarm), use PostgreSQL:\n\n\
347 \x20 assay serve --backend postgres://user:pass@host:5432/dbname"
348 );
349 }
350 } else {
351 anyhow::bail!("Unexpected engine lock state");
352 }
353 }
354 }
355 }
356
357 pub async fn refresh_engine_lock(&self) -> Result<()> {
359 sqlx::query("UPDATE engine.lock SET last_heartbeat = ? WHERE id = 1 AND instance_id = ?")
360 .bind(timestamp_now())
361 .bind(&self.instance_id)
362 .execute(&self.pool)
363 .await?;
364 Ok(())
365 }
366
367 pub async fn release_engine_lock(&self) -> Result<()> {
369 sqlx::query("DELETE FROM engine.lock WHERE id = 1 AND instance_id = ?")
370 .bind(&self.instance_id)
371 .execute(&self.pool)
372 .await?;
373 Ok(())
374 }
375
376 pub fn spawn_lock_heartbeat(self: &std::sync::Arc<Self>) {
378 let store = std::sync::Arc::clone(self);
379 tokio::spawn(async move {
380 let mut tick =
381 tokio::time::interval(std::time::Duration::from_secs(LOCK_HEARTBEAT_SECS));
382 loop {
383 tick.tick().await;
384 if let Err(e) = store.refresh_engine_lock().await {
385 tracing::error!("Engine lock heartbeat failed: {e}");
386 }
387 }
388 });
389 }
390
391 async fn migrate(&self) -> Result<()> {
397 for statement in SCHEMA.split(';') {
398 let trimmed = statement.trim();
399 if !trimmed.is_empty() {
400 sqlx::query(trimmed).execute(&self.pool).await?;
401 }
402 }
403 Self::add_column_if_missing(&self.pool, "workflow.events", "activity_id", "INTEGER")
404 .await?;
405 self.backfill_event_activity_ids().await?;
406 Ok(())
407 }
408
409 async fn backfill_event_activity_ids(&self) -> Result<()> {
415 const BATCH: i64 = 500;
416 loop {
417 let rows: Vec<(i64, Option<String>)> = sqlx::query_as(
418 "SELECT id, payload FROM workflow.events
419 WHERE activity_id IS NULL
420 AND event_type IN ('ActivityCompleted', 'ActivityFailed')
421 LIMIT ?",
422 )
423 .bind(BATCH)
424 .fetch_all(&self.pool)
425 .await?;
426 if rows.is_empty() {
427 return Ok(());
428 }
429 let batch_len = rows.len() as i64;
430 for (id, payload) in rows {
431 sqlx::query("UPDATE workflow.events SET activity_id = ? WHERE id = ?")
432 .bind(payload_activity_id(payload.as_deref()))
433 .bind(id)
434 .execute(&self.pool)
435 .await?;
436 }
437 if batch_len < BATCH {
438 return Ok(());
439 }
440 }
441 }
442
443 async fn add_column_if_missing(
452 pool: &SqlitePool,
453 table: &str,
454 column: &str,
455 type_def: &str,
456 ) -> Result<()> {
457 let (schema, bare) = match table.split_once('.') {
458 Some((schema, bare)) => (schema, bare),
459 None => ("main", table),
460 };
461 let exists: Option<(String,)> =
462 sqlx::query_as("SELECT name FROM pragma_table_info(?, ?) WHERE name = ?")
463 .bind(bare)
464 .bind(schema)
465 .bind(column)
466 .fetch_optional(pool)
467 .await?;
468 if exists.is_none() {
469 let sql = format!("ALTER TABLE {table} ADD COLUMN {column} {type_def}");
470 sqlx::query(&sql).execute(pool).await?;
471 }
472 Ok(())
473 }
474}
475
476impl WorkflowStore for SqliteStore {
477 async fn create_namespace(&self, name: &str) -> Result<()> {
480 sqlx::query("INSERT INTO workflow.namespaces (name, created_at) VALUES (?, ?)")
481 .bind(name)
482 .bind(timestamp_now())
483 .execute(&self.pool)
484 .await?;
485 Ok(())
486 }
487
488 async fn list_namespaces(&self) -> Result<Vec<NamespaceRecord>> {
489 let rows = sqlx::query_as::<_, (String, f64)>(
490 "SELECT name, created_at FROM workflow.namespaces ORDER BY name",
491 )
492 .fetch_all(&self.pool)
493 .await?;
494 Ok(rows
495 .into_iter()
496 .map(|(name, created_at)| NamespaceRecord { name, created_at })
497 .collect())
498 }
499
500 async fn delete_namespace(&self, name: &str) -> Result<bool> {
501 let res = sqlx::query("DELETE FROM workflow.namespaces WHERE name = ? AND name != 'main'")
503 .bind(name)
504 .execute(&self.pool)
505 .await?;
506 Ok(res.rows_affected() > 0)
507 }
508
509 async fn get_namespace_stats(&self, namespace: &str) -> Result<NamespaceStats> {
510 let total: (i64,) =
511 sqlx::query_as("SELECT COUNT(*) FROM workflow.workflows WHERE namespace = ?")
512 .bind(namespace)
513 .fetch_one(&self.pool)
514 .await?;
515 let running: (i64,) = sqlx::query_as(
516 "SELECT COUNT(*) FROM workflow.workflows WHERE namespace = ? AND status = 'RUNNING'",
517 )
518 .bind(namespace)
519 .fetch_one(&self.pool)
520 .await?;
521 let pending: (i64,) = sqlx::query_as(
522 "SELECT COUNT(*) FROM workflow.workflows WHERE namespace = ? AND status = 'PENDING'",
523 )
524 .bind(namespace)
525 .fetch_one(&self.pool)
526 .await?;
527 let completed: (i64,) = sqlx::query_as(
528 "SELECT COUNT(*) FROM workflow.workflows WHERE namespace = ? AND status = 'COMPLETED'",
529 )
530 .bind(namespace)
531 .fetch_one(&self.pool)
532 .await?;
533 let failed: (i64,) = sqlx::query_as(
534 "SELECT COUNT(*) FROM workflow.workflows WHERE namespace = ? AND status = 'FAILED'",
535 )
536 .bind(namespace)
537 .fetch_one(&self.pool)
538 .await?;
539 let schedules: (i64,) =
540 sqlx::query_as("SELECT COUNT(*) FROM workflow.schedules WHERE namespace = ?")
541 .bind(namespace)
542 .fetch_one(&self.pool)
543 .await?;
544 let workers: (i64,) =
545 sqlx::query_as("SELECT COUNT(*) FROM workflow.workers WHERE namespace = ?")
546 .bind(namespace)
547 .fetch_one(&self.pool)
548 .await?;
549
550 Ok(NamespaceStats {
551 namespace: namespace.to_string(),
552 total_workflows: total.0,
553 running: running.0,
554 pending: pending.0,
555 completed: completed.0,
556 failed: failed.0,
557 schedules: schedules.0,
558 workers: workers.0,
559 })
560 }
561
562 async fn create_workflow(&self, wf: &WorkflowRecord) -> Result<()> {
565 sqlx::query(
566 "INSERT INTO workflow.workflows (id, namespace, run_id, workflow_type, task_queue, status, input, result, error, parent_id, claimed_by, search_attributes, archived_at, archive_uri, created_at, updated_at, completed_at)
567 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
568 )
569 .bind(&wf.id)
570 .bind(&wf.namespace)
571 .bind(&wf.run_id)
572 .bind(&wf.workflow_type)
573 .bind(&wf.task_queue)
574 .bind(&wf.status)
575 .bind(&wf.input)
576 .bind(&wf.result)
577 .bind(&wf.error)
578 .bind(&wf.parent_id)
579 .bind(&wf.claimed_by)
580 .bind(&wf.search_attributes)
581 .bind(wf.archived_at)
582 .bind(&wf.archive_uri)
583 .bind(wf.created_at)
584 .bind(wf.updated_at)
585 .bind(wf.completed_at)
586 .execute(&self.pool)
587 .await?;
588 Ok(())
589 }
590
591 async fn get_workflow(&self, id: &str) -> Result<Option<WorkflowRecord>> {
592 let row = sqlx::query_as::<_, SqliteWorkflowRow>(
593 "SELECT id, namespace, run_id, workflow_type, task_queue, status, input, result, error, parent_id, claimed_by, search_attributes, archived_at, archive_uri, created_at, updated_at, completed_at FROM workflow.workflows WHERE id = ?",
594 )
595 .bind(id)
596 .fetch_optional(&self.pool)
597 .await?;
598 Ok(row.map(Into::into))
599 }
600
601 async fn list_workflows(
602 &self,
603 namespace: &str,
604 status: Option<WorkflowStatus>,
605 workflow_type: Option<&str>,
606 search_attrs_filter: Option<&str>,
607 limit: i64,
608 offset: i64,
609 ) -> Result<Vec<WorkflowRecord>> {
610 let status_str = status.map(|s| s.to_string());
611
612 let filter_pairs: Vec<(String, serde_json::Value)> = search_attrs_filter
617 .and_then(|s| serde_json::from_str::<serde_json::Value>(s).ok())
618 .and_then(|v| v.as_object().cloned())
619 .map(|m| m.into_iter().collect())
620 .unwrap_or_default();
621
622 let mut sql = String::from(
623 "SELECT id, namespace, run_id, workflow_type, task_queue, status, input, result, error, parent_id, claimed_by, search_attributes, archived_at, archive_uri, created_at, updated_at, completed_at
624 FROM workflow.workflows
625 WHERE namespace = ?
626 AND (? IS NULL OR status = ?)
627 AND (? IS NULL OR workflow_type = ?)",
628 );
629 for _ in &filter_pairs {
630 sql.push_str(" AND json_extract(search_attributes, '$.' || ?) = ?");
631 }
632 sql.push_str(" ORDER BY created_at DESC LIMIT ? OFFSET ?");
633
634 let mut q = sqlx::query_as::<_, SqliteWorkflowRow>(&sql)
635 .bind(namespace)
636 .bind(&status_str)
637 .bind(&status_str)
638 .bind(workflow_type)
639 .bind(workflow_type);
640 for (key, value) in &filter_pairs {
641 q = q.bind(key.clone());
642 match value {
647 serde_json::Value::String(s) => q = q.bind(s.clone()),
648 serde_json::Value::Number(n) => {
649 if let Some(i) = n.as_i64() {
650 q = q.bind(i);
651 } else if let Some(f) = n.as_f64() {
652 q = q.bind(f);
653 } else {
654 q = q.bind(n.to_string());
655 }
656 }
657 serde_json::Value::Bool(b) => q = q.bind(*b as i64),
658 _ => q = q.bind(value.to_string()),
659 }
660 }
661 let rows = q.bind(limit).bind(offset).fetch_all(&self.pool).await?;
662 Ok(rows.into_iter().map(Into::into).collect())
663 }
664
665 async fn update_workflow_status(
666 &self,
667 id: &str,
668 status: WorkflowStatus,
669 result: Option<&str>,
670 error: Option<&str>,
671 ) -> Result<()> {
672 let now = timestamp_now();
673 let completed_at = if status.is_terminal() {
674 Some(now)
675 } else {
676 None
677 };
678 sqlx::query(
679 "UPDATE workflow.workflows SET status = ?, result = COALESCE(?, result), error = COALESCE(?, error), updated_at = ?, completed_at = COALESCE(?, completed_at) WHERE id = ?",
680 )
681 .bind(status.to_string())
682 .bind(result)
683 .bind(error)
684 .bind(now)
685 .bind(completed_at)
686 .bind(id)
687 .execute(&self.pool)
688 .await?;
689 Ok(())
690 }
691
692 async fn claim_workflow(&self, id: &str, worker_id: &str) -> Result<bool> {
693 let res = sqlx::query(
694 "UPDATE workflow.workflows SET claimed_by = ?, status = 'RUNNING', updated_at = ? WHERE id = ? AND claimed_by IS NULL",
695 )
696 .bind(worker_id)
697 .bind(timestamp_now())
698 .bind(id)
699 .execute(&self.pool)
700 .await?;
701 Ok(res.rows_affected() > 0)
702 }
703
704 async fn mark_workflow_dispatchable(&self, workflow_id: &str) -> Result<()> {
705 sqlx::query("UPDATE workflow.workflows SET needs_dispatch = 1 WHERE id = ?")
706 .bind(workflow_id)
707 .execute(&self.pool)
708 .await?;
709 Ok(())
710 }
711
712 async fn claim_workflow_task(
713 &self,
714 task_queue: &str,
715 worker_id: &str,
716 ) -> Result<Option<WorkflowRecord>> {
717 let now = timestamp_now();
718 let row = sqlx::query_as::<_, SqliteWorkflowRow>(
720 "UPDATE workflow.workflows
721 SET dispatch_claimed_by = ?, dispatch_last_heartbeat = ?, needs_dispatch = 0
722 WHERE id = (
723 SELECT id FROM workflow.workflows
724 WHERE task_queue = ?
725 AND needs_dispatch = 1
726 AND dispatch_claimed_by IS NULL
727 AND status NOT IN ('COMPLETED', 'FAILED', 'CANCELLED', 'TIMED_OUT')
728 ORDER BY updated_at ASC
729 LIMIT 1
730 )
731 RETURNING id, namespace, run_id, workflow_type, task_queue, status, input, result, error, parent_id, claimed_by, search_attributes, archived_at, archive_uri, created_at, updated_at, completed_at",
732 )
733 .bind(worker_id)
734 .bind(now)
735 .bind(task_queue)
736 .fetch_optional(&self.pool)
737 .await?;
738 Ok(row.map(Into::into))
739 }
740
741 async fn release_workflow_task(&self, workflow_id: &str, worker_id: &str) -> Result<()> {
742 sqlx::query(
743 "UPDATE workflow.workflows
744 SET dispatch_claimed_by = NULL, dispatch_last_heartbeat = NULL
745 WHERE id = ? AND dispatch_claimed_by = ?",
746 )
747 .bind(workflow_id)
748 .bind(worker_id)
749 .execute(&self.pool)
750 .await?;
751 Ok(())
752 }
753
754 async fn release_stale_dispatch_leases(&self, now: f64, timeout_secs: f64) -> Result<u64> {
755 let res = sqlx::query(
759 "UPDATE workflow.workflows
760 SET dispatch_claimed_by = NULL,
761 dispatch_last_heartbeat = NULL,
762 needs_dispatch = 1
763 WHERE dispatch_claimed_by IS NOT NULL
764 AND (? - dispatch_last_heartbeat) > ?
765 AND status NOT IN ('COMPLETED', 'FAILED', 'CANCELLED', 'TIMED_OUT')",
766 )
767 .bind(now)
768 .bind(timeout_secs)
769 .execute(&self.pool)
770 .await?;
771 Ok(res.rows_affected())
772 }
773
774 async fn append_event(&self, ev: &WorkflowEvent) -> Result<i64> {
777 let res = sqlx::query(
778 "INSERT INTO workflow.events (workflow_id, seq, event_type, payload, timestamp) VALUES (?, ?, ?, ?, ?)",
779 )
780 .bind(&ev.workflow_id)
781 .bind(ev.seq)
782 .bind(&ev.event_type)
783 .bind(&ev.payload)
784 .bind(ev.timestamp)
785 .execute(&self.pool)
786 .await?;
787 Ok(res.last_insert_rowid())
788 }
789
790 async fn list_events(&self, workflow_id: &str) -> Result<Vec<WorkflowEvent>> {
791 let rows = sqlx::query_as::<_, SqliteEventRow>(
792 "SELECT id, workflow_id, seq, event_type, payload, timestamp FROM workflow.events WHERE workflow_id = ? ORDER BY seq ASC",
793 )
794 .bind(workflow_id)
795 .fetch_all(&self.pool)
796 .await?;
797 Ok(rows.into_iter().map(Into::into).collect())
798 }
799
800 async fn list_events_page(
801 &self,
802 workflow_id: &str,
803 cursor: Option<i32>,
804 limit: i64,
805 descending: bool,
806 ) -> Result<Vec<WorkflowEvent>> {
807 let limit = limit.clamp(0, 1_000);
808 if limit == 0 {
809 return Ok(Vec::new());
810 }
811 let rows = if descending {
812 sqlx::query_as::<_, SqliteEventRow>(
813 "SELECT id, workflow_id, seq, event_type, payload, timestamp
814 FROM workflow.events
815 WHERE workflow_id = ? AND (? IS NULL OR seq < ?)
816 ORDER BY seq DESC LIMIT ?",
817 )
818 .bind(workflow_id)
819 .bind(cursor)
820 .bind(cursor)
821 .bind(limit)
822 .fetch_all(&self.pool)
823 .await?
824 } else {
825 sqlx::query_as::<_, SqliteEventRow>(
826 "SELECT id, workflow_id, seq, event_type, payload, timestamp
827 FROM workflow.events
828 WHERE workflow_id = ? AND (? IS NULL OR seq > ?)
829 ORDER BY seq ASC LIMIT ?",
830 )
831 .bind(workflow_id)
832 .bind(cursor)
833 .bind(cursor)
834 .bind(limit)
835 .fetch_all(&self.pool)
836 .await?
837 };
838 Ok(rows.into_iter().map(Into::into).collect())
839 }
840
841 async fn get_event_count(&self, workflow_id: &str) -> Result<i64> {
842 let row: (i64,) =
843 sqlx::query_as("SELECT COUNT(*) FROM workflow.events WHERE workflow_id = ?")
844 .bind(workflow_id)
845 .fetch_one(&self.pool)
846 .await?;
847 Ok(row.0)
848 }
849
850 async fn create_activity(&self, act: &WorkflowActivity) -> Result<i64> {
853 let res = sqlx::query(
854 "INSERT INTO workflow.activities (workflow_id, seq, name, task_queue, input, status, attempt, max_attempts, initial_interval_secs, backoff_coefficient, start_to_close_secs, heartbeat_timeout_secs, scheduled_at)
855 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
856 )
857 .bind(&act.workflow_id)
858 .bind(act.seq)
859 .bind(&act.name)
860 .bind(&act.task_queue)
861 .bind(&act.input)
862 .bind(&act.status)
863 .bind(act.attempt)
864 .bind(act.max_attempts)
865 .bind(act.initial_interval_secs)
866 .bind(act.backoff_coefficient)
867 .bind(act.start_to_close_secs)
868 .bind(act.heartbeat_timeout_secs)
869 .bind(act.scheduled_at)
870 .execute(&self.pool)
871 .await?;
872 Ok(res.last_insert_rowid())
873 }
874
875 async fn get_activity(&self, id: i64) -> Result<Option<WorkflowActivity>> {
876 let row = sqlx::query_as::<_, SqliteActivityRow>(
877 "SELECT id, workflow_id, seq, name, task_queue, input, status, result, error, attempt, max_attempts, initial_interval_secs, backoff_coefficient, start_to_close_secs, heartbeat_timeout_secs, claimed_by, scheduled_at, started_at, completed_at, last_heartbeat
878 FROM workflow.activities WHERE id = ?",
879 )
880 .bind(id)
881 .fetch_optional(&self.pool)
882 .await?;
883 Ok(row.map(Into::into))
884 }
885
886 async fn get_activity_by_workflow_seq(
887 &self,
888 workflow_id: &str,
889 seq: i32,
890 ) -> Result<Option<WorkflowActivity>> {
891 let row = sqlx::query_as::<_, SqliteActivityRow>(
892 "SELECT id, workflow_id, seq, name, task_queue, input, status, result, error, attempt, max_attempts, initial_interval_secs, backoff_coefficient, start_to_close_secs, heartbeat_timeout_secs, claimed_by, scheduled_at, started_at, completed_at, last_heartbeat
893 FROM workflow.activities WHERE workflow_id = ? AND seq = ?",
894 )
895 .bind(workflow_id)
896 .bind(seq)
897 .fetch_optional(&self.pool)
898 .await?;
899 Ok(row.map(Into::into))
900 }
901
902 async fn claim_activity(
903 &self,
904 task_queue: &str,
905 worker_id: &str,
906 ) -> Result<Option<WorkflowActivity>> {
907 let now = timestamp_now();
908 let row = sqlx::query_as::<_, SqliteActivityRow>(
909 "UPDATE workflow.activities SET status = 'RUNNING', claimed_by = ?, started_at = ?
910 WHERE id = (
911 SELECT id FROM workflow.activities
912 WHERE task_queue = ? AND status = 'PENDING'
913 ORDER BY scheduled_at ASC
914 LIMIT 1
915 )
916 RETURNING id, workflow_id, seq, name, task_queue, input, status, result, error, attempt, max_attempts, initial_interval_secs, backoff_coefficient, start_to_close_secs, heartbeat_timeout_secs, claimed_by, scheduled_at, started_at, completed_at, last_heartbeat",
917 )
918 .bind(worker_id)
919 .bind(now)
920 .bind(task_queue)
921 .fetch_optional(&self.pool)
922 .await?;
923 Ok(row.map(Into::into))
924 }
925
926 async fn requeue_activity_for_retry(
927 &self,
928 id: i64,
929 next_attempt: i32,
930 next_scheduled_at: f64,
931 ) -> Result<()> {
932 sqlx::query(
933 "UPDATE workflow.activities
934 SET status = 'PENDING', attempt = ?, scheduled_at = ?,
935 claimed_by = NULL, started_at = NULL, last_heartbeat = NULL,
936 error = NULL
937 WHERE id = ?",
938 )
939 .bind(next_attempt)
940 .bind(next_scheduled_at)
941 .bind(id)
942 .execute(&self.pool)
943 .await?;
944 Ok(())
945 }
946
947 async fn retry_failed_activity(
948 &self,
949 workflow_id: &str,
950 requested_by: &str,
951 reason: &str,
952 requested_at: f64,
953 ) -> Result<RetryFailedActivityResult> {
954 let mut tx = self.pool.begin().await?;
955 let workflow: Option<(String, Option<String>, Option<f64>)> = sqlx::query_as(
956 "SELECT status, parent_id, archived_at FROM workflow.workflows WHERE id = ?",
957 )
958 .bind(workflow_id)
959 .fetch_optional(&mut *tx)
960 .await?;
961 let Some((status, parent_id, archived_at)) = workflow else {
962 return Ok(RetryFailedActivityResult::NotFound);
963 };
964 if let Some(denial) = retry_denial(status, parent_id, archived_at) {
965 return Ok(denial);
966 }
967
968 let failed = sqlx::query_as::<_, SqliteActivityRow>(RETRY_ACTIVITY_SELECT)
969 .bind(workflow_id)
970 .fetch_optional(&mut *tx)
971 .await?;
972 let Some(failed) = failed else {
973 return Ok(RetryFailedActivityResult::NoFailedActivity);
974 };
975 let failed_event_seq: (i32,) = sqlx::query_as(
976 "SELECT seq FROM workflow.events
977 WHERE workflow_id = ? AND event_type = 'ActivityFailed'
978 ORDER BY seq DESC LIMIT 1",
979 )
980 .bind(workflow_id)
981 .fetch_one(&mut *tx)
982 .await?;
983 let invalidated =
984 sqlx::query("DELETE FROM workflow.activities WHERE workflow_id = ? AND seq > ?")
985 .bind(workflow_id)
986 .bind(failed.seq)
987 .execute(&mut *tx)
988 .await?
989 .rows_affected();
990 let activity = sqlx::query_as::<_, SqliteActivityRow>(RETRY_ACTIVITY_UPDATE)
991 .bind(requested_at)
992 .bind(failed.id)
993 .fetch_one(&mut *tx)
994 .await?;
995 sqlx::query("UPDATE workflow.events SET activity_id = ? WHERE activity_id = ?")
998 .bind(NOT_A_SETTLEMENT)
999 .bind(failed.id)
1000 .execute(&mut *tx)
1001 .await?;
1002 sqlx::query(
1003 "UPDATE workflow.workflows
1004 SET status = 'WAITING', result = NULL, error = NULL, completed_at = NULL,
1005 updated_at = ?, needs_dispatch = 0, dispatch_claimed_by = NULL,
1006 dispatch_last_heartbeat = NULL
1007 WHERE id = ?",
1008 )
1009 .bind(requested_at)
1010 .bind(workflow_id)
1011 .execute(&mut *tx)
1012 .await?;
1013 let event_seq: (i32,) = sqlx::query_as(
1014 "SELECT COALESCE(MAX(seq), 0) + 1 FROM workflow.events WHERE workflow_id = ?",
1015 )
1016 .bind(workflow_id)
1017 .fetch_one(&mut *tx)
1018 .await?;
1019 let payload = RetryEvent {
1020 activity_id: failed.id,
1021 activity_seq: failed.seq,
1022 activity_name: &failed.name,
1023 failed_event_seq: failed_event_seq.0,
1024 requested_by,
1025 reason,
1026 invalidated_activities: invalidated,
1027 }
1028 .payload();
1029 sqlx::query(
1030 "INSERT INTO workflow.events (workflow_id, seq, event_type, payload, timestamp)
1031 VALUES (?, ?, 'ActivityRetryRequested', ?, ?)",
1032 )
1033 .bind(workflow_id)
1034 .bind(event_seq.0)
1035 .bind(payload.to_string())
1036 .bind(requested_at)
1037 .execute(&mut *tx)
1038 .await?;
1039 tx.commit().await?;
1040 Ok(RetryFailedActivityResult::Retried(Box::new(
1041 RetriedActivity {
1042 activity: activity.into(),
1043 invalidated_activities: invalidated,
1044 },
1045 )))
1046 }
1047
1048 async fn complete_activity(
1049 &self,
1050 id: i64,
1051 result: Option<&str>,
1052 error: Option<&str>,
1053 failed: bool,
1054 ) -> Result<()> {
1055 let status = if failed { "FAILED" } else { "COMPLETED" };
1056 sqlx::query(
1057 "UPDATE workflow.activities SET status = ?, result = ?, error = ?, completed_at = ? WHERE id = ?",
1058 )
1059 .bind(status)
1060 .bind(result)
1061 .bind(error)
1062 .bind(timestamp_now())
1063 .bind(id)
1064 .execute(&self.pool)
1065 .await?;
1066 Ok(())
1067 }
1068
1069 async fn settle_activity(&self, settlement: &ActivitySettlement<'_>) -> Result<SettleOutcome> {
1070 let mut tx = self.pool.begin().await?;
1071 let current: Option<(String,)> =
1072 sqlx::query_as("SELECT status FROM workflow.activities WHERE id = ?")
1073 .bind(settlement.activity_id)
1074 .fetch_optional(&mut *tx)
1075 .await?;
1076 let Some((status,)) = current else {
1077 return Ok(SettleOutcome::Unknown);
1078 };
1079 let settled = matches!(status.as_str(), "COMPLETED" | "FAILED");
1080 let event_id: Option<(i64,)> = sqlx::query_as(
1081 "SELECT id FROM workflow.events
1082 WHERE workflow_id = ? AND activity_id = ?
1083 AND event_type IN ('ActivityCompleted', 'ActivityFailed')
1084 LIMIT 1",
1085 )
1086 .bind(settlement.workflow_id)
1087 .bind(settlement.activity_id)
1088 .fetch_optional(&mut *tx)
1089 .await?;
1090
1091 if !settled {
1092 sqlx::query(
1093 "UPDATE workflow.activities
1094 SET status = ?, result = ?, error = ?, completed_at = ?
1095 WHERE id = ?",
1096 )
1097 .bind(if settlement.failed {
1098 "FAILED"
1099 } else {
1100 "COMPLETED"
1101 })
1102 .bind(settlement.result)
1103 .bind(settlement.error)
1104 .bind(settlement.now)
1105 .bind(settlement.activity_id)
1106 .execute(&mut *tx)
1107 .await?;
1108 }
1109 if !settled || event_id.is_none() {
1113 let seq: (i32,) = sqlx::query_as(
1114 "SELECT COALESCE(MAX(seq), 0) + 1 FROM workflow.events WHERE workflow_id = ?",
1115 )
1116 .bind(settlement.workflow_id)
1117 .fetch_one(&mut *tx)
1118 .await?;
1119 sqlx::query(
1120 "INSERT INTO workflow.events (workflow_id, seq, event_type, payload, activity_id, timestamp)
1121 VALUES (?, ?, ?, ?, ?, ?)",
1122 )
1123 .bind(settlement.workflow_id)
1124 .bind(seq.0)
1125 .bind(settlement.event_type)
1126 .bind(settlement.payload)
1127 .bind(settlement.activity_id)
1128 .bind(settlement.now)
1129 .execute(&mut *tx)
1130 .await?;
1131 }
1132 sqlx::query("UPDATE workflow.workflows SET needs_dispatch = 1 WHERE id = ?")
1133 .bind(settlement.workflow_id)
1134 .execute(&mut *tx)
1135 .await?;
1136 tx.commit().await?;
1137 Ok(settle_outcome(settled, event_id.is_some()))
1138 }
1139
1140 async fn list_unsettled_activities(&self, limit: i64) -> Result<Vec<WorkflowActivity>> {
1141 let rows = sqlx::query_as::<_, SqliteActivityRow>(UNSETTLED_ACTIVITY_SELECT)
1142 .bind(limit)
1143 .fetch_all(&self.pool)
1144 .await?;
1145 Ok(rows.into_iter().map(Into::into).collect())
1146 }
1147
1148 async fn heartbeat_activity(&self, id: i64, _details: Option<&str>) -> Result<()> {
1149 sqlx::query("UPDATE workflow.activities SET last_heartbeat = ? WHERE id = ?")
1150 .bind(timestamp_now())
1151 .bind(id)
1152 .execute(&self.pool)
1153 .await?;
1154 Ok(())
1155 }
1156
1157 async fn get_timed_out_activities(&self, now: f64) -> Result<Vec<WorkflowActivity>> {
1158 let rows = sqlx::query_as::<_, SqliteActivityRow>(
1159 "SELECT id, workflow_id, seq, name, task_queue, input, status, result, error, attempt, max_attempts, initial_interval_secs, backoff_coefficient, start_to_close_secs, heartbeat_timeout_secs, claimed_by, scheduled_at, started_at, completed_at, last_heartbeat
1160 FROM workflow.activities
1161 WHERE status = 'RUNNING'
1162 AND heartbeat_timeout_secs IS NOT NULL
1163 AND (? - COALESCE(last_heartbeat, started_at)) > heartbeat_timeout_secs",
1164 )
1165 .bind(now)
1166 .fetch_all(&self.pool)
1167 .await?;
1168 Ok(rows.into_iter().map(Into::into).collect())
1169 }
1170
1171 async fn create_timer(&self, timer: &WorkflowTimer) -> Result<i64> {
1174 let res = sqlx::query(
1177 "INSERT OR IGNORE INTO workflow.timers (workflow_id, seq, fire_at, fired) VALUES (?, ?, ?, 0)",
1178 )
1179 .bind(&timer.workflow_id)
1180 .bind(timer.seq)
1181 .bind(timer.fire_at)
1182 .execute(&self.pool)
1183 .await?;
1184
1185 let id = res.last_insert_rowid();
1186 if id != 0 {
1187 return Ok(id);
1188 }
1189
1190 let (existing_id,): (i64,) =
1192 sqlx::query_as("SELECT id FROM workflow.timers WHERE workflow_id = ? AND seq = ?")
1193 .bind(&timer.workflow_id)
1194 .bind(timer.seq)
1195 .fetch_one(&self.pool)
1196 .await?;
1197 Ok(existing_id)
1198 }
1199
1200 async fn cancel_pending_activities(&self, workflow_id: &str) -> Result<u64> {
1201 let res = sqlx::query(
1202 "UPDATE workflow.activities SET status = 'CANCELLED', completed_at = ?
1203 WHERE workflow_id = ? AND status = 'PENDING'",
1204 )
1205 .bind(timestamp_now())
1206 .bind(workflow_id)
1207 .execute(&self.pool)
1208 .await?;
1209 Ok(res.rows_affected())
1210 }
1211
1212 async fn cancel_pending_timers(&self, workflow_id: &str) -> Result<u64> {
1213 let res = sqlx::query(
1214 "UPDATE workflow.timers SET fired = 1
1215 WHERE workflow_id = ? AND fired = 0",
1216 )
1217 .bind(workflow_id)
1218 .execute(&self.pool)
1219 .await?;
1220 Ok(res.rows_affected())
1221 }
1222
1223 async fn get_timer_by_workflow_seq(
1224 &self,
1225 workflow_id: &str,
1226 seq: i32,
1227 ) -> Result<Option<WorkflowTimer>> {
1228 let row = sqlx::query_as::<_, SqliteTimerRow>(
1229 "SELECT id, workflow_id, seq, fire_at, fired
1230 FROM workflow.timers WHERE workflow_id = ? AND seq = ?",
1231 )
1232 .bind(workflow_id)
1233 .bind(seq)
1234 .fetch_optional(&self.pool)
1235 .await?;
1236 Ok(row.map(Into::into))
1237 }
1238
1239 async fn fire_due_timers(&self, now: f64) -> Result<Vec<WorkflowTimer>> {
1240 let rows = sqlx::query_as::<_, SqliteTimerRow>(
1241 "UPDATE workflow.timers SET fired = 1
1242 WHERE fired = 0 AND fire_at <= ?
1243 RETURNING id, workflow_id, seq, fire_at, fired",
1244 )
1245 .bind(now)
1246 .fetch_all(&self.pool)
1247 .await?;
1248 Ok(rows.into_iter().map(Into::into).collect())
1249 }
1250
1251 async fn send_signal(&self, sig: &WorkflowSignal) -> Result<i64> {
1254 let res = sqlx::query(
1255 "INSERT INTO workflow.signals (workflow_id, name, payload, consumed, received_at) VALUES (?, ?, ?, 0, ?)",
1256 )
1257 .bind(&sig.workflow_id)
1258 .bind(&sig.name)
1259 .bind(&sig.payload)
1260 .bind(sig.received_at)
1261 .execute(&self.pool)
1262 .await?;
1263 Ok(res.last_insert_rowid())
1264 }
1265
1266 async fn deliver_signal(&self, sig: &WorkflowSignal, payload_json: &str) -> Result<i64> {
1267 let mut tx = self.pool.begin().await?;
1268 let res = sqlx::query(
1269 "INSERT INTO workflow.signals (workflow_id, name, payload, consumed, received_at) VALUES (?, ?, ?, 0, ?)",
1270 )
1271 .bind(&sig.workflow_id)
1272 .bind(&sig.name)
1273 .bind(&sig.payload)
1274 .bind(sig.received_at)
1275 .execute(&mut *tx)
1276 .await?;
1277 let signal_id = res.last_insert_rowid();
1278 let seq: (i32,) = sqlx::query_as(
1279 "SELECT COALESCE(MAX(seq), 0) + 1 FROM workflow.events WHERE workflow_id = ?",
1280 )
1281 .bind(&sig.workflow_id)
1282 .fetch_one(&mut *tx)
1283 .await?;
1284 sqlx::query(
1285 "INSERT INTO workflow.events (workflow_id, seq, event_type, payload, timestamp)
1286 VALUES (?, ?, 'SignalReceived', ?, ?)",
1287 )
1288 .bind(&sig.workflow_id)
1289 .bind(seq.0)
1290 .bind(payload_json)
1291 .bind(sig.received_at)
1292 .execute(&mut *tx)
1293 .await?;
1294 sqlx::query("UPDATE workflow.workflows SET needs_dispatch = 1 WHERE id = ?")
1295 .bind(&sig.workflow_id)
1296 .execute(&mut *tx)
1297 .await?;
1298 tx.commit().await?;
1299 Ok(signal_id)
1300 }
1301
1302 async fn consume_signals(&self, workflow_id: &str, name: &str) -> Result<Vec<WorkflowSignal>> {
1303 let rows = sqlx::query_as::<_, SqliteSignalRow>(
1304 "UPDATE workflow.signals SET consumed = 1
1305 WHERE workflow_id = ? AND name = ? AND consumed = 0
1306 RETURNING id, workflow_id, name, payload, consumed, received_at",
1307 )
1308 .bind(workflow_id)
1309 .bind(name)
1310 .fetch_all(&self.pool)
1311 .await?;
1312 Ok(rows.into_iter().map(Into::into).collect())
1313 }
1314
1315 async fn create_schedule(&self, sched: &WorkflowSchedule) -> Result<()> {
1318 sqlx::query(
1319 "INSERT INTO workflow.schedules (name, namespace, workflow_type, cron_expr, timezone, input, task_queue, overlap_policy, paused, last_run_at, next_run_at, last_workflow_id, created_at)
1320 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1321 )
1322 .bind(&sched.name)
1323 .bind(&sched.namespace)
1324 .bind(&sched.workflow_type)
1325 .bind(&sched.cron_expr)
1326 .bind(&sched.timezone)
1327 .bind(&sched.input)
1328 .bind(&sched.task_queue)
1329 .bind(&sched.overlap_policy)
1330 .bind(sched.paused)
1331 .bind(sched.last_run_at)
1332 .bind(crate::scheduler::seed_next_run(sched))
1333 .bind(&sched.last_workflow_id)
1334 .bind(sched.created_at)
1335 .execute(&self.pool)
1336 .await?;
1337 Ok(())
1338 }
1339
1340 async fn get_schedule(&self, namespace: &str, name: &str) -> Result<Option<WorkflowSchedule>> {
1341 let row = sqlx::query_as::<_, SqliteScheduleRow>(
1342 "SELECT name, namespace, workflow_type, cron_expr, timezone, input, task_queue, overlap_policy, paused, last_run_at, next_run_at, last_workflow_id, created_at
1343 FROM workflow.schedules WHERE namespace = ? AND name = ?",
1344 )
1345 .bind(namespace)
1346 .bind(name)
1347 .fetch_optional(&self.pool)
1348 .await?;
1349 Ok(row.map(Into::into))
1350 }
1351
1352 async fn list_schedules(&self, namespace: &str) -> Result<Vec<WorkflowSchedule>> {
1353 let rows = sqlx::query_as::<_, SqliteScheduleRow>(
1354 "SELECT name, namespace, workflow_type, cron_expr, timezone, input, task_queue, overlap_policy, paused, last_run_at, next_run_at, last_workflow_id, created_at
1355 FROM workflow.schedules WHERE namespace = ? ORDER BY name",
1356 )
1357 .bind(namespace)
1358 .fetch_all(&self.pool)
1359 .await?;
1360 Ok(rows.into_iter().map(Into::into).collect())
1361 }
1362
1363 async fn update_schedule_last_run(
1364 &self,
1365 namespace: &str,
1366 name: &str,
1367 last_run_at: f64,
1368 next_run_at: f64,
1369 workflow_id: &str,
1370 ) -> Result<()> {
1371 sqlx::query(
1372 "UPDATE workflow.schedules SET last_run_at = ?, next_run_at = ?, last_workflow_id = ? WHERE namespace = ? AND name = ?",
1373 )
1374 .bind(last_run_at)
1375 .bind(next_run_at)
1376 .bind(workflow_id)
1377 .bind(namespace)
1378 .bind(name)
1379 .execute(&self.pool)
1380 .await?;
1381 Ok(())
1382 }
1383
1384 async fn delete_schedule(&self, namespace: &str, name: &str) -> Result<bool> {
1385 let res = sqlx::query("DELETE FROM workflow.schedules WHERE namespace = ? AND name = ?")
1386 .bind(namespace)
1387 .bind(name)
1388 .execute(&self.pool)
1389 .await?;
1390 Ok(res.rows_affected() > 0)
1391 }
1392
1393 async fn list_archivable_workflows(
1394 &self,
1395 cutoff: f64,
1396 limit: i64,
1397 ) -> Result<Vec<WorkflowRecord>> {
1398 let rows = sqlx::query_as::<_, SqliteWorkflowRow>(
1399 "SELECT id, namespace, run_id, workflow_type, task_queue, status, input, result, error, parent_id, claimed_by, search_attributes, archived_at, archive_uri, created_at, updated_at, completed_at
1400 FROM workflow.workflows
1401 WHERE status IN ('COMPLETED', 'FAILED', 'CANCELLED', 'TIMED_OUT')
1402 AND completed_at IS NOT NULL
1403 AND completed_at < ?
1404 AND archived_at IS NULL
1405 ORDER BY completed_at ASC
1406 LIMIT ?",
1407 )
1408 .bind(cutoff)
1409 .bind(limit)
1410 .fetch_all(&self.pool)
1411 .await?;
1412 Ok(rows.into_iter().map(Into::into).collect())
1413 }
1414
1415 async fn mark_archived_and_purge(
1416 &self,
1417 workflow_id: &str,
1418 archive_uri: &str,
1419 archived_at: f64,
1420 ) -> Result<()> {
1421 let mut tx = self.pool.begin().await?;
1422 sqlx::query("DELETE FROM workflow.events WHERE workflow_id = ?")
1423 .bind(workflow_id)
1424 .execute(&mut *tx)
1425 .await?;
1426 sqlx::query("DELETE FROM workflow.activities WHERE workflow_id = ?")
1427 .bind(workflow_id)
1428 .execute(&mut *tx)
1429 .await?;
1430 sqlx::query("DELETE FROM workflow.timers WHERE workflow_id = ?")
1431 .bind(workflow_id)
1432 .execute(&mut *tx)
1433 .await?;
1434 sqlx::query("DELETE FROM workflow.signals WHERE workflow_id = ?")
1435 .bind(workflow_id)
1436 .execute(&mut *tx)
1437 .await?;
1438 sqlx::query("DELETE FROM workflow.snapshots WHERE workflow_id = ?")
1439 .bind(workflow_id)
1440 .execute(&mut *tx)
1441 .await?;
1442 sqlx::query("UPDATE workflow.workflows SET archived_at = ?, archive_uri = ? WHERE id = ?")
1443 .bind(archived_at)
1444 .bind(archive_uri)
1445 .bind(workflow_id)
1446 .execute(&mut *tx)
1447 .await?;
1448 tx.commit().await?;
1449 Ok(())
1450 }
1451
1452 async fn upsert_search_attributes(&self, workflow_id: &str, patch_json: &str) -> Result<()> {
1453 let current: Option<(Option<String>,)> =
1456 sqlx::query_as("SELECT search_attributes FROM workflow.workflows WHERE id = ?")
1457 .bind(workflow_id)
1458 .fetch_optional(&self.pool)
1459 .await?;
1460 let merged = merge_search_attrs(current.and_then(|(s,)| s).as_deref(), patch_json)?;
1461 sqlx::query("UPDATE workflow.workflows SET search_attributes = ? WHERE id = ?")
1462 .bind(merged)
1463 .bind(workflow_id)
1464 .execute(&self.pool)
1465 .await?;
1466 Ok(())
1467 }
1468
1469 async fn update_schedule(
1470 &self,
1471 namespace: &str,
1472 name: &str,
1473 patch: &SchedulePatch,
1474 ) -> Result<Option<WorkflowSchedule>> {
1475 let mut sets: Vec<&'static str> = Vec::new();
1478 if patch.cron_expr.is_some() {
1479 sets.push("cron_expr = ?");
1480 }
1481 if patch.timezone.is_some() {
1482 sets.push("timezone = ?");
1483 }
1484 if patch.input.is_some() {
1485 sets.push("input = ?");
1486 }
1487 if patch.task_queue.is_some() {
1488 sets.push("task_queue = ?");
1489 }
1490 if patch.overlap_policy.is_some() {
1491 sets.push("overlap_policy = ?");
1492 }
1493 if sets.is_empty() {
1495 return self.get_schedule(namespace, name).await;
1496 }
1497
1498 let sql = format!(
1499 "UPDATE workflow.schedules SET {} WHERE namespace = ? AND name = ?",
1500 sets.join(", ")
1501 );
1502 let mut q = sqlx::query(&sql);
1503 if let Some(ref v) = patch.cron_expr {
1504 q = q.bind(v);
1505 }
1506 if let Some(ref v) = patch.timezone {
1507 q = q.bind(v);
1508 }
1509 if let Some(ref v) = patch.input {
1510 q = q.bind(v.to_string());
1511 }
1512 if let Some(ref v) = patch.task_queue {
1513 q = q.bind(v);
1514 }
1515 if let Some(ref v) = patch.overlap_policy {
1516 q = q.bind(v);
1517 }
1518 let res = q.bind(namespace).bind(name).execute(&self.pool).await?;
1519 if res.rows_affected() == 0 {
1520 return Ok(None);
1521 }
1522 self.get_schedule(namespace, name).await
1523 }
1524
1525 async fn set_schedule_paused(
1526 &self,
1527 namespace: &str,
1528 name: &str,
1529 paused: bool,
1530 ) -> Result<Option<WorkflowSchedule>> {
1531 let res = sqlx::query(
1532 "UPDATE workflow.schedules SET paused = ? WHERE namespace = ? AND name = ?",
1533 )
1534 .bind(paused)
1535 .bind(namespace)
1536 .bind(name)
1537 .execute(&self.pool)
1538 .await?;
1539 if res.rows_affected() == 0 {
1540 return Ok(None);
1541 }
1542 self.get_schedule(namespace, name).await
1543 }
1544
1545 async fn register_worker(&self, w: &WorkflowWorker) -> Result<()> {
1548 sqlx::query(
1549 "INSERT OR REPLACE INTO workflow.workers (id, namespace, identity, task_queue, workflows, activities, max_concurrent_workflows, max_concurrent_activities, active_tasks, last_heartbeat, registered_at)
1550 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1551 )
1552 .bind(&w.id)
1553 .bind(&w.namespace)
1554 .bind(&w.identity)
1555 .bind(&w.task_queue)
1556 .bind(&w.workflows)
1557 .bind(&w.activities)
1558 .bind(w.max_concurrent_workflows)
1559 .bind(w.max_concurrent_activities)
1560 .bind(w.active_tasks)
1561 .bind(w.last_heartbeat)
1562 .bind(w.registered_at)
1563 .execute(&self.pool)
1564 .await?;
1565 Ok(())
1566 }
1567
1568 async fn heartbeat_worker(&self, id: &str, now: f64) -> Result<bool> {
1569 let res = sqlx::query("UPDATE workflow.workers SET last_heartbeat = ? WHERE id = ?")
1570 .bind(now)
1571 .bind(id)
1572 .execute(&self.pool)
1573 .await?;
1574 Ok(res.rows_affected() > 0)
1575 }
1576
1577 async fn list_workers(&self, namespace: &str) -> Result<Vec<WorkflowWorker>> {
1578 let rows = sqlx::query_as::<_, SqliteWorkerRow>(
1579 "SELECT id, namespace, identity, task_queue, workflows, activities, max_concurrent_workflows, max_concurrent_activities, active_tasks, last_heartbeat, registered_at
1580 FROM workflow.workers WHERE namespace = ? ORDER BY registered_at",
1581 )
1582 .bind(namespace)
1583 .fetch_all(&self.pool)
1584 .await?;
1585 Ok(rows.into_iter().map(Into::into).collect())
1586 }
1587
1588 async fn remove_dead_workers(&self, cutoff: f64) -> Result<Vec<String>> {
1589 let rows: Vec<(String,)> =
1590 sqlx::query_as("SELECT id FROM workflow.workers WHERE last_heartbeat < ?")
1591 .bind(cutoff)
1592 .fetch_all(&self.pool)
1593 .await?;
1594 let ids: Vec<String> = rows.into_iter().map(|r| r.0).collect();
1595 if !ids.is_empty() {
1596 sqlx::query("DELETE FROM workflow.workers WHERE last_heartbeat < ?")
1597 .bind(cutoff)
1598 .execute(&self.pool)
1599 .await?;
1600 }
1601 Ok(ids)
1602 }
1603
1604 async fn list_child_workflows(&self, parent_id: &str) -> Result<Vec<WorkflowRecord>> {
1607 let rows = sqlx::query_as::<_, SqliteWorkflowRow>(
1608 "SELECT id, namespace, run_id, workflow_type, task_queue, status, input, result, error, parent_id, claimed_by, search_attributes, archived_at, archive_uri, created_at, updated_at, completed_at
1609 FROM workflow.workflows WHERE parent_id = ? ORDER BY created_at ASC",
1610 )
1611 .bind(parent_id)
1612 .fetch_all(&self.pool)
1613 .await?;
1614 Ok(rows.into_iter().map(Into::into).collect())
1615 }
1616
1617 async fn create_snapshot(
1620 &self,
1621 workflow_id: &str,
1622 event_seq: i32,
1623 state_json: &str,
1624 ) -> Result<()> {
1625 sqlx::query(
1626 "INSERT OR REPLACE INTO workflow.snapshots (workflow_id, event_seq, state_json, created_at)
1627 VALUES (?, ?, ?, ?)",
1628 )
1629 .bind(workflow_id)
1630 .bind(event_seq)
1631 .bind(state_json)
1632 .bind(timestamp_now())
1633 .execute(&self.pool)
1634 .await?;
1635 Ok(())
1636 }
1637
1638 async fn get_latest_snapshot(&self, workflow_id: &str) -> Result<Option<WorkflowSnapshot>> {
1639 let row = sqlx::query_as::<_, (String, i32, String, f64)>(
1640 "SELECT workflow_id, event_seq, state_json, created_at
1641 FROM workflow.snapshots WHERE workflow_id = ?
1642 ORDER BY event_seq DESC LIMIT 1",
1643 )
1644 .bind(workflow_id)
1645 .fetch_optional(&self.pool)
1646 .await?;
1647
1648 Ok(row.map(
1649 |(workflow_id, event_seq, state_json, created_at)| WorkflowSnapshot {
1650 workflow_id,
1651 event_seq,
1652 state_json,
1653 created_at,
1654 },
1655 ))
1656 }
1657
1658 async fn get_queue_stats(&self, namespace: &str) -> Result<Vec<QueueStats>> {
1661 let rows = sqlx::query_as::<_, (String, i64, i64)>(
1663 "SELECT a.task_queue,
1664 SUM(CASE WHEN a.status = 'PENDING' THEN 1 ELSE 0 END),
1665 SUM(CASE WHEN a.status = 'RUNNING' THEN 1 ELSE 0 END)
1666 FROM workflow.activities a
1667 INNER JOIN workflow.workflows w ON w.id = a.workflow_id
1668 WHERE w.namespace = ?
1669 GROUP BY a.task_queue",
1670 )
1671 .bind(namespace)
1672 .fetch_all(&self.pool)
1673 .await?;
1674
1675 let mut stats: Vec<QueueStats> = rows
1676 .into_iter()
1677 .map(|(queue, pending, running)| QueueStats {
1678 queue,
1679 pending_activities: pending,
1680 running_activities: running,
1681 workers: 0,
1682 })
1683 .collect();
1684
1685 let worker_rows = sqlx::query_as::<_, (String, i64)>(
1687 "SELECT task_queue, COUNT(*) FROM workflow.workers WHERE namespace = ? GROUP BY task_queue",
1688 )
1689 .bind(namespace)
1690 .fetch_all(&self.pool)
1691 .await?;
1692
1693 for (queue, count) in worker_rows {
1694 if let Some(s) = stats.iter_mut().find(|s| s.queue == queue) {
1695 s.workers = count;
1696 } else {
1697 stats.push(QueueStats {
1698 queue,
1699 pending_activities: 0,
1700 running_activities: 0,
1701 workers: count,
1702 });
1703 }
1704 }
1705
1706 stats.sort_by(|a, b| a.queue.cmp(&b.queue));
1707 Ok(stats)
1708 }
1709
1710 async fn try_acquire_scheduler_lock(&self) -> Result<bool> {
1713 self.refresh_engine_lock().await.ok();
1716 Ok(true)
1717 }
1718}
1719
1720fn timestamp_now() -> f64 {
1721 std::time::SystemTime::now()
1722 .duration_since(std::time::UNIX_EPOCH)
1723 .unwrap()
1724 .as_secs_f64()
1725}
1726
1727pub(crate) fn merge_search_attrs(current: Option<&str>, patch_json: &str) -> Result<String> {
1730 let mut current_map: serde_json::Map<String, serde_json::Value> = current
1731 .and_then(|s| serde_json::from_str::<serde_json::Value>(s).ok())
1732 .and_then(|v| v.as_object().cloned())
1733 .unwrap_or_default();
1734 let patch: serde_json::Value = serde_json::from_str(patch_json)
1735 .map_err(|e| anyhow::anyhow!("invalid search_attributes patch: {e}"))?;
1736 let patch_obj = patch
1737 .as_object()
1738 .ok_or_else(|| anyhow::anyhow!("search_attributes patch must be a JSON object"))?;
1739 for (k, v) in patch_obj {
1740 current_map.insert(k.clone(), v.clone());
1741 }
1742 Ok(serde_json::Value::Object(current_map).to_string())
1743}
1744
1745#[derive(sqlx::FromRow)]
1748struct SqliteWorkflowRow {
1749 id: String,
1750 namespace: String,
1751 run_id: String,
1752 workflow_type: String,
1753 task_queue: String,
1754 status: String,
1755 input: Option<String>,
1756 result: Option<String>,
1757 error: Option<String>,
1758 parent_id: Option<String>,
1759 claimed_by: Option<String>,
1760 search_attributes: Option<String>,
1761 archived_at: Option<f64>,
1762 archive_uri: Option<String>,
1763 created_at: f64,
1764 updated_at: f64,
1765 completed_at: Option<f64>,
1766}
1767
1768impl From<SqliteWorkflowRow> for WorkflowRecord {
1769 fn from(r: SqliteWorkflowRow) -> Self {
1770 Self {
1771 id: r.id,
1772 namespace: r.namespace,
1773 run_id: r.run_id,
1774 workflow_type: r.workflow_type,
1775 task_queue: r.task_queue,
1776 status: r.status,
1777 input: r.input,
1778 result: r.result,
1779 error: r.error,
1780 parent_id: r.parent_id,
1781 claimed_by: r.claimed_by,
1782 search_attributes: r.search_attributes,
1783 archived_at: r.archived_at,
1784 archive_uri: r.archive_uri,
1785 created_at: r.created_at,
1786 updated_at: r.updated_at,
1787 completed_at: r.completed_at,
1788 }
1789 }
1790}
1791
1792#[derive(sqlx::FromRow)]
1793struct SqliteEventRow {
1794 id: i64,
1795 workflow_id: String,
1796 seq: i32,
1797 event_type: String,
1798 payload: Option<String>,
1799 timestamp: f64,
1800}
1801
1802impl From<SqliteEventRow> for WorkflowEvent {
1803 fn from(r: SqliteEventRow) -> Self {
1804 Self {
1805 id: Some(r.id),
1806 workflow_id: r.workflow_id,
1807 seq: r.seq,
1808 event_type: r.event_type,
1809 payload: r.payload,
1810 timestamp: r.timestamp,
1811 }
1812 }
1813}
1814
1815#[derive(sqlx::FromRow)]
1816struct SqliteActivityRow {
1817 id: i64,
1818 workflow_id: String,
1819 seq: i32,
1820 name: String,
1821 task_queue: String,
1822 input: Option<String>,
1823 status: String,
1824 result: Option<String>,
1825 error: Option<String>,
1826 attempt: i32,
1827 max_attempts: i32,
1828 initial_interval_secs: f64,
1829 backoff_coefficient: f64,
1830 start_to_close_secs: f64,
1831 heartbeat_timeout_secs: Option<f64>,
1832 claimed_by: Option<String>,
1833 scheduled_at: f64,
1834 started_at: Option<f64>,
1835 completed_at: Option<f64>,
1836 last_heartbeat: Option<f64>,
1837}
1838
1839impl From<SqliteActivityRow> for WorkflowActivity {
1840 fn from(r: SqliteActivityRow) -> Self {
1841 Self {
1842 id: Some(r.id),
1843 workflow_id: r.workflow_id,
1844 seq: r.seq,
1845 name: r.name,
1846 task_queue: r.task_queue,
1847 input: r.input,
1848 status: r.status,
1849 result: r.result,
1850 error: r.error,
1851 attempt: r.attempt,
1852 max_attempts: r.max_attempts,
1853 initial_interval_secs: r.initial_interval_secs,
1854 backoff_coefficient: r.backoff_coefficient,
1855 start_to_close_secs: r.start_to_close_secs,
1856 heartbeat_timeout_secs: r.heartbeat_timeout_secs,
1857 claimed_by: r.claimed_by,
1858 scheduled_at: r.scheduled_at,
1859 started_at: r.started_at,
1860 completed_at: r.completed_at,
1861 last_heartbeat: r.last_heartbeat,
1862 }
1863 }
1864}
1865
1866#[derive(sqlx::FromRow)]
1867struct SqliteTimerRow {
1868 id: i64,
1869 workflow_id: String,
1870 seq: i32,
1871 fire_at: f64,
1872 fired: bool,
1873}
1874
1875impl From<SqliteTimerRow> for WorkflowTimer {
1876 fn from(r: SqliteTimerRow) -> Self {
1877 Self {
1878 id: Some(r.id),
1879 workflow_id: r.workflow_id,
1880 seq: r.seq,
1881 fire_at: r.fire_at,
1882 fired: r.fired,
1883 }
1884 }
1885}
1886
1887#[derive(sqlx::FromRow)]
1888struct SqliteSignalRow {
1889 id: i64,
1890 workflow_id: String,
1891 name: String,
1892 payload: Option<String>,
1893 consumed: bool,
1894 received_at: f64,
1895}
1896
1897impl From<SqliteSignalRow> for WorkflowSignal {
1898 fn from(r: SqliteSignalRow) -> Self {
1899 Self {
1900 id: Some(r.id),
1901 workflow_id: r.workflow_id,
1902 name: r.name,
1903 payload: r.payload,
1904 consumed: r.consumed,
1905 received_at: r.received_at,
1906 }
1907 }
1908}
1909
1910#[derive(sqlx::FromRow)]
1911struct SqliteScheduleRow {
1912 name: String,
1913 namespace: String,
1914 workflow_type: String,
1915 cron_expr: String,
1916 timezone: String,
1917 input: Option<String>,
1918 task_queue: String,
1919 overlap_policy: String,
1920 paused: bool,
1921 last_run_at: Option<f64>,
1922 next_run_at: Option<f64>,
1923 last_workflow_id: Option<String>,
1924 created_at: f64,
1925}
1926
1927impl From<SqliteScheduleRow> for WorkflowSchedule {
1928 fn from(r: SqliteScheduleRow) -> Self {
1929 Self {
1930 name: r.name,
1931 namespace: r.namespace,
1932 workflow_type: r.workflow_type,
1933 cron_expr: r.cron_expr,
1934 timezone: r.timezone,
1935 input: r.input,
1936 task_queue: r.task_queue,
1937 overlap_policy: r.overlap_policy,
1938 paused: r.paused,
1939 last_run_at: r.last_run_at,
1940 next_run_at: r.next_run_at,
1941 last_workflow_id: r.last_workflow_id,
1942 created_at: r.created_at,
1943 }
1944 }
1945}
1946
1947#[derive(sqlx::FromRow)]
1948struct SqliteWorkerRow {
1949 id: String,
1950 namespace: String,
1951 identity: String,
1952 task_queue: String,
1953 workflows: Option<String>,
1954 activities: Option<String>,
1955 max_concurrent_workflows: i32,
1956 max_concurrent_activities: i32,
1957 active_tasks: i32,
1958 last_heartbeat: f64,
1959 registered_at: f64,
1960}
1961
1962impl From<SqliteWorkerRow> for WorkflowWorker {
1963 fn from(r: SqliteWorkerRow) -> Self {
1964 Self {
1965 id: r.id,
1966 namespace: r.namespace,
1967 identity: r.identity,
1968 task_queue: r.task_queue,
1969 workflows: r.workflows,
1970 activities: r.activities,
1971 max_concurrent_workflows: r.max_concurrent_workflows,
1972 max_concurrent_activities: r.max_concurrent_activities,
1973 active_tasks: r.active_tasks,
1974 last_heartbeat: r.last_heartbeat,
1975 registered_at: r.registered_at,
1976 }
1977 }
1978}