1use anyhow::Result;
2use sqlx::SqlitePool;
3use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};
4
5use crate::store::{
6 NOT_A_SETTLEMENT, NamespaceRecord, NamespaceStats, QueueStats, RetryEvent, WorkflowStore,
7 payload_activity_id, retry_denial, settle_outcome,
8};
9use crate::types::*;
10
11const RETRY_ACTIVITY_SELECT: &str = "SELECT id, workflow_id, seq, name, task_queue, input, status, result, error, attempt, max_attempts, initial_interval_secs, backoff_coefficient, start_to_close_secs, heartbeat_timeout_secs, claimed_by, scheduled_at, started_at, completed_at, last_heartbeat FROM workflow.activities WHERE workflow_id = ? AND status = 'FAILED' ORDER BY seq DESC LIMIT 1";
12const UNSETTLED_ACTIVITY_SELECT: &str = "SELECT a.id, a.workflow_id, a.seq, a.name, a.task_queue, a.input, a.status, a.result, a.error, a.attempt, a.max_attempts, a.initial_interval_secs, a.backoff_coefficient, a.start_to_close_secs, a.heartbeat_timeout_secs, a.claimed_by, a.scheduled_at, a.started_at, a.completed_at, a.last_heartbeat FROM workflow.activities a JOIN workflow.workflows w ON w.id = a.workflow_id WHERE a.status IN ('COMPLETED', 'FAILED') AND w.status NOT IN ('COMPLETED', 'FAILED', 'CANCELLED', 'TIMED_OUT') AND w.archived_at IS NULL AND NOT EXISTS (SELECT 1 FROM workflow.events e WHERE e.workflow_id = a.workflow_id AND e.activity_id = a.id AND e.event_type IN ('ActivityCompleted', 'ActivityFailed')) ORDER BY a.completed_at ASC LIMIT ?";
16const RETRY_ACTIVITY_UPDATE: &str = "UPDATE workflow.activities SET status = 'PENDING', result = NULL, error = NULL, attempt = 1, claimed_by = NULL, scheduled_at = ?, started_at = NULL, completed_at = NULL, last_heartbeat = NULL WHERE id = ? RETURNING id, workflow_id, seq, name, task_queue, input, status, result, error, attempt, max_attempts, initial_interval_secs, backoff_coefficient, start_to_close_secs, heartbeat_timeout_secs, claimed_by, scheduled_at, started_at, completed_at, last_heartbeat";
17
18const SCHEMA: &str = r#"
31CREATE TABLE IF NOT EXISTS workflow.namespaces (
32 name TEXT PRIMARY KEY,
33 created_at REAL NOT NULL
34);
35
36INSERT OR IGNORE INTO workflow.namespaces (name, created_at)
37 VALUES ('main', strftime('%s', 'now'));
38
39CREATE TABLE IF NOT EXISTS workflow.workflows (
40 id TEXT PRIMARY KEY,
41 namespace TEXT NOT NULL DEFAULT 'main',
42 run_id TEXT NOT NULL,
43 workflow_type TEXT NOT NULL,
44 task_queue TEXT NOT NULL DEFAULT 'main',
45 status TEXT NOT NULL DEFAULT 'PENDING',
46 input TEXT,
47 result TEXT,
48 error TEXT,
49 parent_id TEXT,
50 claimed_by TEXT,
51 search_attributes TEXT,
52 archived_at REAL,
53 archive_uri TEXT,
54 -- Workflow-task dispatch (Phase 9): a workflow is "dispatchable" when
55 -- it has new events a worker needs to replay against. Set true on
56 -- start, on activity completion, on timer fire, on signal arrival.
57 -- Cleared when a worker claims the dispatch lease.
58 needs_dispatch INTEGER NOT NULL DEFAULT 0,
59 dispatch_claimed_by TEXT,
60 dispatch_last_heartbeat REAL,
61 created_at REAL NOT NULL,
62 updated_at REAL NOT NULL,
63 completed_at REAL
64);
65CREATE INDEX IF NOT EXISTS workflow.idx_wf_status_queue ON workflows(status, task_queue);
66CREATE INDEX IF NOT EXISTS workflow.idx_wf_namespace ON workflows(namespace);
67CREATE INDEX IF NOT EXISTS workflow.idx_wf_dispatch ON workflows(task_queue, needs_dispatch, dispatch_claimed_by);
68
69CREATE TABLE IF NOT EXISTS workflow.events (
70 id INTEGER PRIMARY KEY AUTOINCREMENT,
71 workflow_id TEXT NOT NULL REFERENCES workflows(id),
72 seq INTEGER NOT NULL,
73 event_type TEXT NOT NULL,
74 payload TEXT,
75 -- Set on ActivityCompleted / ActivityFailed only. Answers "did this
76 -- activity's terminal event land" without parsing payload JSON, which
77 -- is what the settle transaction and the reconciler both need.
78 activity_id INTEGER,
79 timestamp REAL NOT NULL
80);
81CREATE INDEX IF NOT EXISTS workflow.idx_wf_events_lookup ON events(workflow_id, seq);
82
83CREATE TABLE IF NOT EXISTS workflow.activities (
84 id INTEGER PRIMARY KEY AUTOINCREMENT,
85 workflow_id TEXT NOT NULL REFERENCES workflows(id),
86 seq INTEGER NOT NULL,
87 name TEXT NOT NULL,
88 task_queue TEXT NOT NULL DEFAULT 'main',
89 input TEXT,
90 status TEXT NOT NULL DEFAULT 'PENDING',
91 result TEXT,
92 error TEXT,
93 attempt INTEGER NOT NULL DEFAULT 1,
94 max_attempts INTEGER NOT NULL DEFAULT 3,
95 initial_interval_secs REAL NOT NULL DEFAULT 1,
96 backoff_coefficient REAL NOT NULL DEFAULT 2,
97 start_to_close_secs REAL NOT NULL DEFAULT 300,
98 heartbeat_timeout_secs REAL,
99 claimed_by TEXT,
100 scheduled_at REAL NOT NULL,
101 started_at REAL,
102 completed_at REAL,
103 last_heartbeat REAL,
104 UNIQUE (workflow_id, seq)
105);
106CREATE INDEX IF NOT EXISTS workflow.idx_wf_act_pending ON activities(task_queue, status, scheduled_at);
107
108CREATE TABLE IF NOT EXISTS workflow.timers (
109 id INTEGER PRIMARY KEY AUTOINCREMENT,
110 workflow_id TEXT NOT NULL REFERENCES workflows(id),
111 seq INTEGER NOT NULL,
112 fire_at REAL NOT NULL,
113 fired INTEGER NOT NULL DEFAULT 0,
114 UNIQUE (workflow_id, seq)
115);
116CREATE INDEX IF NOT EXISTS workflow.idx_wf_timers_due ON timers(fire_at);
117
118CREATE TABLE IF NOT EXISTS workflow.signals (
119 id INTEGER PRIMARY KEY AUTOINCREMENT,
120 workflow_id TEXT NOT NULL REFERENCES workflows(id),
121 name TEXT NOT NULL,
122 payload TEXT,
123 consumed INTEGER NOT NULL DEFAULT 0,
124 received_at REAL NOT NULL
125);
126CREATE INDEX IF NOT EXISTS workflow.idx_wf_signals_lookup ON signals(workflow_id, name, consumed);
127
128CREATE TABLE IF NOT EXISTS workflow.schedules (
129 name TEXT NOT NULL,
130 namespace TEXT NOT NULL DEFAULT 'main',
131 workflow_type TEXT NOT NULL,
132 cron_expr TEXT NOT NULL,
133 timezone TEXT NOT NULL DEFAULT 'UTC',
134 input TEXT,
135 task_queue TEXT NOT NULL DEFAULT 'main',
136 overlap_policy TEXT NOT NULL DEFAULT 'skip',
137 paused INTEGER NOT NULL DEFAULT 0,
138 last_run_at REAL,
139 next_run_at REAL,
140 last_workflow_id TEXT,
141 created_at REAL NOT NULL,
142 PRIMARY KEY (namespace, name)
143);
144
145CREATE TABLE IF NOT EXISTS workflow.workers (
146 id TEXT PRIMARY KEY,
147 namespace TEXT NOT NULL DEFAULT 'main',
148 identity TEXT NOT NULL,
149 task_queue TEXT NOT NULL,
150 workflows TEXT,
151 activities TEXT,
152 max_concurrent_workflows INTEGER NOT NULL DEFAULT 10,
153 max_concurrent_activities INTEGER NOT NULL DEFAULT 10,
154 active_tasks INTEGER NOT NULL DEFAULT 0,
155 last_heartbeat REAL NOT NULL,
156 registered_at REAL NOT NULL
157);
158
159CREATE TABLE IF NOT EXISTS workflow.snapshots (
160 workflow_id TEXT NOT NULL REFERENCES workflows(id),
161 event_seq INTEGER NOT NULL,
162 state_json TEXT NOT NULL,
163 created_at REAL NOT NULL,
164 PRIMARY KEY (workflow_id, event_seq)
165);
166
167-- workflow.api_keys retired in plan-15 slice 3 (auth tokens come from
168-- the auth module).
169DROP TABLE IF EXISTS workflow.api_keys;
170
171CREATE TABLE IF NOT EXISTS engine.lock (
172 id INTEGER PRIMARY KEY CHECK (id = 1),
173 instance_id TEXT NOT NULL,
174 started_at REAL NOT NULL,
175 last_heartbeat REAL NOT NULL
176);
177
178CREATE TABLE IF NOT EXISTS engine.events (
179 id INTEGER PRIMARY KEY AUTOINCREMENT,
180 ts REAL NOT NULL DEFAULT (CAST(strftime('%s','now') AS REAL)),
181 namespace TEXT NOT NULL,
182 subsystem TEXT NOT NULL,
183 kind TEXT NOT NULL,
184 payload TEXT NOT NULL DEFAULT '{}'
185);
186CREATE INDEX IF NOT EXISTS engine.idx_engine_events_ns_id ON events(namespace, id);
187CREATE INDEX IF NOT EXISTS engine.idx_engine_events_ts_prune ON events(ts);
188"#;
189
190const LOCK_STALE_SECS: f64 = 60.0;
193const LOCK_HEARTBEAT_SECS: u64 = 15;
195
196#[derive(Clone)]
203pub struct SqliteStore {
204 pool: SqlitePool,
205 instance_id: String,
206}
207
208async fn build_default_pool(url: &str) -> Result<SqlitePool> {
221 use std::str::FromStr;
222 use std::sync::atomic::{AtomicU64, Ordering};
223
224 static SEQ: AtomicU64 = AtomicU64::new(0);
225 let suffix = format!(
226 "{}_{}",
227 std::process::id(),
228 SEQ.fetch_add(1, Ordering::Relaxed)
229 );
230 let engine_alias = format!("file:assay_engine_{suffix}?mode=memory&cache=shared");
231 let workflow_alias = format!("file:assay_workflow_{suffix}?mode=memory&cache=shared");
232
233 let opts = SqliteConnectOptions::from_str(url)?.create_if_missing(true);
234
235 let pool = SqlitePoolOptions::new()
236 .max_connections(1)
237 .after_connect(move |conn, _meta| {
238 let engine_alias = engine_alias.clone();
239 let workflow_alias = workflow_alias.clone();
240 Box::pin(async move {
241 use sqlx::Executor;
242 conn.execute(format!("ATTACH DATABASE '{engine_alias}' AS engine").as_str())
243 .await?;
244 conn.execute(format!("ATTACH DATABASE '{workflow_alias}' AS workflow").as_str())
245 .await?;
246 Ok(())
247 })
248 })
249 .connect_with(opts)
250 .await?;
251 Ok(pool)
252}
253
254impl SqliteStore {
255 pub async fn new(url: &str) -> Result<Self> {
261 let pool = build_default_pool(url).await?;
262 Self::from_attached_pool(pool).await
263 }
264
265 pub async fn from_attached_pool(pool: SqlitePool) -> Result<Self> {
270 let instance_id = format!("assay-{:016x}", {
271 use std::collections::hash_map::DefaultHasher;
272 use std::hash::{Hash, Hasher};
273 let mut h = DefaultHasher::new();
274 std::time::SystemTime::now().hash(&mut h);
275 std::process::id().hash(&mut h);
276 h.finish()
277 });
278 let store = Self { pool, instance_id };
279 store.migrate().await?;
280 Ok(store)
281 }
282
283 pub async fn from_pool(pool: SqlitePool) -> Result<Self> {
289 Self::from_attached_pool(pool).await
290 }
291
292 pub fn pool(&self) -> &SqlitePool {
295 &self.pool
296 }
297
298 pub async fn acquire_engine_lock(&self) -> Result<()> {
301 let now = timestamp_now();
302
303 let result = sqlx::query(
305 "INSERT INTO engine.lock (id, instance_id, started_at, last_heartbeat) VALUES (1, ?, ?, ?)",
306 )
307 .bind(&self.instance_id)
308 .bind(now)
309 .bind(now)
310 .execute(&self.pool)
311 .await;
312
313 match result {
314 Ok(_) => Ok(()),
315 Err(_) => {
316 let row: Option<(String, f64)> = sqlx::query_as(
318 "SELECT instance_id, last_heartbeat FROM engine.lock WHERE id = 1",
319 )
320 .fetch_optional(&self.pool)
321 .await?;
322
323 if let Some((existing_id, last_hb)) = row {
324 if now - last_hb > LOCK_STALE_SECS {
325 sqlx::query(
327 "UPDATE engine.lock SET instance_id = ?, started_at = ?, last_heartbeat = ? WHERE id = 1",
328 )
329 .bind(&self.instance_id)
330 .bind(now)
331 .bind(now)
332 .execute(&self.pool)
333 .await?;
334 tracing::warn!(
335 "Took over stale engine lock from {existing_id} (last heartbeat {:.0}s ago)",
336 now - last_hb
337 );
338 Ok(())
339 } else {
340 let age = now - last_hb;
341 anyhow::bail!(
342 "Another assay engine instance is already running (id: {existing_id}, \
343 last heartbeat {age:.0}s ago).\n\n\
344 SQLite only supports a single engine instance. For multi-instance \
345 deployment (Kubernetes, Docker Swarm), use PostgreSQL:\n\n\
346 \x20 assay serve --backend postgres://user:pass@host:5432/dbname"
347 );
348 }
349 } else {
350 anyhow::bail!("Unexpected engine lock state");
351 }
352 }
353 }
354 }
355
356 pub async fn refresh_engine_lock(&self) -> Result<()> {
358 sqlx::query("UPDATE engine.lock SET last_heartbeat = ? WHERE id = 1 AND instance_id = ?")
359 .bind(timestamp_now())
360 .bind(&self.instance_id)
361 .execute(&self.pool)
362 .await?;
363 Ok(())
364 }
365
366 pub async fn release_engine_lock(&self) -> Result<()> {
368 sqlx::query("DELETE FROM engine.lock WHERE id = 1 AND instance_id = ?")
369 .bind(&self.instance_id)
370 .execute(&self.pool)
371 .await?;
372 Ok(())
373 }
374
375 pub fn spawn_lock_heartbeat(self: &std::sync::Arc<Self>) {
377 let store = std::sync::Arc::clone(self);
378 tokio::spawn(async move {
379 let mut tick =
380 tokio::time::interval(std::time::Duration::from_secs(LOCK_HEARTBEAT_SECS));
381 loop {
382 tick.tick().await;
383 if let Err(e) = store.refresh_engine_lock().await {
384 tracing::error!("Engine lock heartbeat failed: {e}");
385 }
386 }
387 });
388 }
389
390 async fn migrate(&self) -> Result<()> {
396 for statement in SCHEMA.split(';') {
397 let trimmed = statement.trim();
398 if !trimmed.is_empty() {
399 sqlx::query(trimmed).execute(&self.pool).await?;
400 }
401 }
402 Self::add_column_if_missing(&self.pool, "workflow.events", "activity_id", "INTEGER")
403 .await?;
404 sqlx::query(
405 "CREATE INDEX IF NOT EXISTS workflow.idx_wf_events_activity ON events(activity_id)",
406 )
407 .execute(&self.pool)
408 .await?;
409 self.backfill_event_activity_ids().await?;
410 Ok(())
411 }
412
413 async fn backfill_event_activity_ids(&self) -> Result<()> {
419 const BATCH: i64 = 500;
420 loop {
421 let rows: Vec<(i64, Option<String>)> = sqlx::query_as(
422 "SELECT id, payload FROM workflow.events
423 WHERE activity_id IS NULL
424 AND event_type IN ('ActivityCompleted', 'ActivityFailed')
425 LIMIT ?",
426 )
427 .bind(BATCH)
428 .fetch_all(&self.pool)
429 .await?;
430 if rows.is_empty() {
431 return Ok(());
432 }
433 let batch_len = rows.len() as i64;
434 for (id, payload) in rows {
435 sqlx::query("UPDATE workflow.events SET activity_id = ? WHERE id = ?")
436 .bind(payload_activity_id(payload.as_deref()))
437 .bind(id)
438 .execute(&self.pool)
439 .await?;
440 }
441 if batch_len < BATCH {
442 return Ok(());
443 }
444 }
445 }
446
447 async fn add_column_if_missing(
456 pool: &SqlitePool,
457 table: &str,
458 column: &str,
459 type_def: &str,
460 ) -> Result<()> {
461 let (schema, bare) = match table.split_once('.') {
462 Some((schema, bare)) => (schema, bare),
463 None => ("main", table),
464 };
465 let exists: Option<(String,)> =
466 sqlx::query_as("SELECT name FROM pragma_table_info(?, ?) WHERE name = ?")
467 .bind(bare)
468 .bind(schema)
469 .bind(column)
470 .fetch_optional(pool)
471 .await?;
472 if exists.is_none() {
473 let sql = format!("ALTER TABLE {table} ADD COLUMN {column} {type_def}");
474 sqlx::query(&sql).execute(pool).await?;
475 }
476 Ok(())
477 }
478}
479
480impl WorkflowStore for SqliteStore {
481 async fn create_namespace(&self, name: &str) -> Result<()> {
484 sqlx::query("INSERT INTO workflow.namespaces (name, created_at) VALUES (?, ?)")
485 .bind(name)
486 .bind(timestamp_now())
487 .execute(&self.pool)
488 .await?;
489 Ok(())
490 }
491
492 async fn list_namespaces(&self) -> Result<Vec<NamespaceRecord>> {
493 let rows = sqlx::query_as::<_, (String, f64)>(
494 "SELECT name, created_at FROM workflow.namespaces ORDER BY name",
495 )
496 .fetch_all(&self.pool)
497 .await?;
498 Ok(rows
499 .into_iter()
500 .map(|(name, created_at)| NamespaceRecord { name, created_at })
501 .collect())
502 }
503
504 async fn delete_namespace(&self, name: &str) -> Result<bool> {
505 let res = sqlx::query("DELETE FROM workflow.namespaces WHERE name = ? AND name != 'main'")
507 .bind(name)
508 .execute(&self.pool)
509 .await?;
510 Ok(res.rows_affected() > 0)
511 }
512
513 async fn get_namespace_stats(&self, namespace: &str) -> Result<NamespaceStats> {
514 let total: (i64,) =
515 sqlx::query_as("SELECT COUNT(*) FROM workflow.workflows WHERE namespace = ?")
516 .bind(namespace)
517 .fetch_one(&self.pool)
518 .await?;
519 let running: (i64,) = sqlx::query_as(
520 "SELECT COUNT(*) FROM workflow.workflows WHERE namespace = ? AND status = 'RUNNING'",
521 )
522 .bind(namespace)
523 .fetch_one(&self.pool)
524 .await?;
525 let pending: (i64,) = sqlx::query_as(
526 "SELECT COUNT(*) FROM workflow.workflows WHERE namespace = ? AND status = 'PENDING'",
527 )
528 .bind(namespace)
529 .fetch_one(&self.pool)
530 .await?;
531 let completed: (i64,) = sqlx::query_as(
532 "SELECT COUNT(*) FROM workflow.workflows WHERE namespace = ? AND status = 'COMPLETED'",
533 )
534 .bind(namespace)
535 .fetch_one(&self.pool)
536 .await?;
537 let failed: (i64,) = sqlx::query_as(
538 "SELECT COUNT(*) FROM workflow.workflows WHERE namespace = ? AND status = 'FAILED'",
539 )
540 .bind(namespace)
541 .fetch_one(&self.pool)
542 .await?;
543 let schedules: (i64,) =
544 sqlx::query_as("SELECT COUNT(*) FROM workflow.schedules WHERE namespace = ?")
545 .bind(namespace)
546 .fetch_one(&self.pool)
547 .await?;
548 let workers: (i64,) =
549 sqlx::query_as("SELECT COUNT(*) FROM workflow.workers WHERE namespace = ?")
550 .bind(namespace)
551 .fetch_one(&self.pool)
552 .await?;
553
554 Ok(NamespaceStats {
555 namespace: namespace.to_string(),
556 total_workflows: total.0,
557 running: running.0,
558 pending: pending.0,
559 completed: completed.0,
560 failed: failed.0,
561 schedules: schedules.0,
562 workers: workers.0,
563 })
564 }
565
566 async fn create_workflow(&self, wf: &WorkflowRecord) -> Result<()> {
569 sqlx::query(
570 "INSERT INTO workflow.workflows (id, namespace, run_id, workflow_type, task_queue, status, input, result, error, parent_id, claimed_by, search_attributes, archived_at, archive_uri, created_at, updated_at, completed_at)
571 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
572 )
573 .bind(&wf.id)
574 .bind(&wf.namespace)
575 .bind(&wf.run_id)
576 .bind(&wf.workflow_type)
577 .bind(&wf.task_queue)
578 .bind(&wf.status)
579 .bind(&wf.input)
580 .bind(&wf.result)
581 .bind(&wf.error)
582 .bind(&wf.parent_id)
583 .bind(&wf.claimed_by)
584 .bind(&wf.search_attributes)
585 .bind(wf.archived_at)
586 .bind(&wf.archive_uri)
587 .bind(wf.created_at)
588 .bind(wf.updated_at)
589 .bind(wf.completed_at)
590 .execute(&self.pool)
591 .await?;
592 Ok(())
593 }
594
595 async fn get_workflow(&self, id: &str) -> Result<Option<WorkflowRecord>> {
596 let row = sqlx::query_as::<_, SqliteWorkflowRow>(
597 "SELECT id, namespace, run_id, workflow_type, task_queue, status, input, result, error, parent_id, claimed_by, search_attributes, archived_at, archive_uri, created_at, updated_at, completed_at FROM workflow.workflows WHERE id = ?",
598 )
599 .bind(id)
600 .fetch_optional(&self.pool)
601 .await?;
602 Ok(row.map(Into::into))
603 }
604
605 async fn list_workflows(
606 &self,
607 namespace: &str,
608 status: Option<WorkflowStatus>,
609 workflow_type: Option<&str>,
610 search_attrs_filter: Option<&str>,
611 limit: i64,
612 offset: i64,
613 ) -> Result<Vec<WorkflowRecord>> {
614 let status_str = status.map(|s| s.to_string());
615
616 let filter_pairs: Vec<(String, serde_json::Value)> = search_attrs_filter
621 .and_then(|s| serde_json::from_str::<serde_json::Value>(s).ok())
622 .and_then(|v| v.as_object().cloned())
623 .map(|m| m.into_iter().collect())
624 .unwrap_or_default();
625
626 let mut sql = String::from(
627 "SELECT id, namespace, run_id, workflow_type, task_queue, status, input, result, error, parent_id, claimed_by, search_attributes, archived_at, archive_uri, created_at, updated_at, completed_at
628 FROM workflow.workflows
629 WHERE namespace = ?
630 AND (? IS NULL OR status = ?)
631 AND (? IS NULL OR workflow_type = ?)",
632 );
633 for _ in &filter_pairs {
634 sql.push_str(" AND json_extract(search_attributes, '$.' || ?) = ?");
635 }
636 sql.push_str(" ORDER BY created_at DESC LIMIT ? OFFSET ?");
637
638 let mut q = sqlx::query_as::<_, SqliteWorkflowRow>(&sql)
639 .bind(namespace)
640 .bind(&status_str)
641 .bind(&status_str)
642 .bind(workflow_type)
643 .bind(workflow_type);
644 for (key, value) in &filter_pairs {
645 q = q.bind(key.clone());
646 match value {
651 serde_json::Value::String(s) => q = q.bind(s.clone()),
652 serde_json::Value::Number(n) => {
653 if let Some(i) = n.as_i64() {
654 q = q.bind(i);
655 } else if let Some(f) = n.as_f64() {
656 q = q.bind(f);
657 } else {
658 q = q.bind(n.to_string());
659 }
660 }
661 serde_json::Value::Bool(b) => q = q.bind(*b as i64),
662 _ => q = q.bind(value.to_string()),
663 }
664 }
665 let rows = q.bind(limit).bind(offset).fetch_all(&self.pool).await?;
666 Ok(rows.into_iter().map(Into::into).collect())
667 }
668
669 async fn update_workflow_status(
670 &self,
671 id: &str,
672 status: WorkflowStatus,
673 result: Option<&str>,
674 error: Option<&str>,
675 ) -> Result<()> {
676 let now = timestamp_now();
677 let completed_at = if status.is_terminal() {
678 Some(now)
679 } else {
680 None
681 };
682 sqlx::query(
683 "UPDATE workflow.workflows SET status = ?, result = COALESCE(?, result), error = COALESCE(?, error), updated_at = ?, completed_at = COALESCE(?, completed_at) WHERE id = ?",
684 )
685 .bind(status.to_string())
686 .bind(result)
687 .bind(error)
688 .bind(now)
689 .bind(completed_at)
690 .bind(id)
691 .execute(&self.pool)
692 .await?;
693 Ok(())
694 }
695
696 async fn claim_workflow(&self, id: &str, worker_id: &str) -> Result<bool> {
697 let res = sqlx::query(
698 "UPDATE workflow.workflows SET claimed_by = ?, status = 'RUNNING', updated_at = ? WHERE id = ? AND claimed_by IS NULL",
699 )
700 .bind(worker_id)
701 .bind(timestamp_now())
702 .bind(id)
703 .execute(&self.pool)
704 .await?;
705 Ok(res.rows_affected() > 0)
706 }
707
708 async fn mark_workflow_dispatchable(&self, workflow_id: &str) -> Result<()> {
709 sqlx::query("UPDATE workflow.workflows SET needs_dispatch = 1 WHERE id = ?")
710 .bind(workflow_id)
711 .execute(&self.pool)
712 .await?;
713 Ok(())
714 }
715
716 async fn claim_workflow_task(
717 &self,
718 task_queue: &str,
719 worker_id: &str,
720 ) -> Result<Option<WorkflowRecord>> {
721 let now = timestamp_now();
722 let row = sqlx::query_as::<_, SqliteWorkflowRow>(
724 "UPDATE workflow.workflows
725 SET dispatch_claimed_by = ?, dispatch_last_heartbeat = ?, needs_dispatch = 0
726 WHERE id = (
727 SELECT id FROM workflow.workflows
728 WHERE task_queue = ?
729 AND needs_dispatch = 1
730 AND dispatch_claimed_by IS NULL
731 AND status NOT IN ('COMPLETED', 'FAILED', 'CANCELLED', 'TIMED_OUT')
732 ORDER BY updated_at ASC
733 LIMIT 1
734 )
735 RETURNING id, namespace, run_id, workflow_type, task_queue, status, input, result, error, parent_id, claimed_by, search_attributes, archived_at, archive_uri, created_at, updated_at, completed_at",
736 )
737 .bind(worker_id)
738 .bind(now)
739 .bind(task_queue)
740 .fetch_optional(&self.pool)
741 .await?;
742 Ok(row.map(Into::into))
743 }
744
745 async fn release_workflow_task(&self, workflow_id: &str, worker_id: &str) -> Result<()> {
746 sqlx::query(
747 "UPDATE workflow.workflows
748 SET dispatch_claimed_by = NULL, dispatch_last_heartbeat = NULL
749 WHERE id = ? AND dispatch_claimed_by = ?",
750 )
751 .bind(workflow_id)
752 .bind(worker_id)
753 .execute(&self.pool)
754 .await?;
755 Ok(())
756 }
757
758 async fn release_stale_dispatch_leases(&self, now: f64, timeout_secs: f64) -> Result<u64> {
759 let res = sqlx::query(
763 "UPDATE workflow.workflows
764 SET dispatch_claimed_by = NULL,
765 dispatch_last_heartbeat = NULL,
766 needs_dispatch = 1
767 WHERE dispatch_claimed_by IS NOT NULL
768 AND (? - dispatch_last_heartbeat) > ?
769 AND status NOT IN ('COMPLETED', 'FAILED', 'CANCELLED', 'TIMED_OUT')",
770 )
771 .bind(now)
772 .bind(timeout_secs)
773 .execute(&self.pool)
774 .await?;
775 Ok(res.rows_affected())
776 }
777
778 async fn append_event(&self, ev: &WorkflowEvent) -> Result<i64> {
781 let res = sqlx::query(
782 "INSERT INTO workflow.events (workflow_id, seq, event_type, payload, timestamp) VALUES (?, ?, ?, ?, ?)",
783 )
784 .bind(&ev.workflow_id)
785 .bind(ev.seq)
786 .bind(&ev.event_type)
787 .bind(&ev.payload)
788 .bind(ev.timestamp)
789 .execute(&self.pool)
790 .await?;
791 Ok(res.last_insert_rowid())
792 }
793
794 async fn list_events(&self, workflow_id: &str) -> Result<Vec<WorkflowEvent>> {
795 let rows = sqlx::query_as::<_, SqliteEventRow>(
796 "SELECT id, workflow_id, seq, event_type, payload, timestamp FROM workflow.events WHERE workflow_id = ? ORDER BY seq ASC",
797 )
798 .bind(workflow_id)
799 .fetch_all(&self.pool)
800 .await?;
801 Ok(rows.into_iter().map(Into::into).collect())
802 }
803
804 async fn list_events_page(
805 &self,
806 workflow_id: &str,
807 cursor: Option<i32>,
808 limit: i64,
809 descending: bool,
810 ) -> Result<Vec<WorkflowEvent>> {
811 let limit = limit.clamp(0, 1_000);
812 if limit == 0 {
813 return Ok(Vec::new());
814 }
815 let rows = if descending {
816 sqlx::query_as::<_, SqliteEventRow>(
817 "SELECT id, workflow_id, seq, event_type, payload, timestamp
818 FROM workflow.events
819 WHERE workflow_id = ? AND (? IS NULL OR seq < ?)
820 ORDER BY seq DESC LIMIT ?",
821 )
822 .bind(workflow_id)
823 .bind(cursor)
824 .bind(cursor)
825 .bind(limit)
826 .fetch_all(&self.pool)
827 .await?
828 } else {
829 sqlx::query_as::<_, SqliteEventRow>(
830 "SELECT id, workflow_id, seq, event_type, payload, timestamp
831 FROM workflow.events
832 WHERE workflow_id = ? AND (? IS NULL OR seq > ?)
833 ORDER BY seq ASC LIMIT ?",
834 )
835 .bind(workflow_id)
836 .bind(cursor)
837 .bind(cursor)
838 .bind(limit)
839 .fetch_all(&self.pool)
840 .await?
841 };
842 Ok(rows.into_iter().map(Into::into).collect())
843 }
844
845 async fn get_event_count(&self, workflow_id: &str) -> Result<i64> {
846 let row: (i64,) =
847 sqlx::query_as("SELECT COUNT(*) FROM workflow.events WHERE workflow_id = ?")
848 .bind(workflow_id)
849 .fetch_one(&self.pool)
850 .await?;
851 Ok(row.0)
852 }
853
854 async fn create_activity(&self, act: &WorkflowActivity) -> Result<i64> {
857 let res = sqlx::query(
858 "INSERT INTO workflow.activities (workflow_id, seq, name, task_queue, input, status, attempt, max_attempts, initial_interval_secs, backoff_coefficient, start_to_close_secs, heartbeat_timeout_secs, scheduled_at)
859 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
860 )
861 .bind(&act.workflow_id)
862 .bind(act.seq)
863 .bind(&act.name)
864 .bind(&act.task_queue)
865 .bind(&act.input)
866 .bind(&act.status)
867 .bind(act.attempt)
868 .bind(act.max_attempts)
869 .bind(act.initial_interval_secs)
870 .bind(act.backoff_coefficient)
871 .bind(act.start_to_close_secs)
872 .bind(act.heartbeat_timeout_secs)
873 .bind(act.scheduled_at)
874 .execute(&self.pool)
875 .await?;
876 Ok(res.last_insert_rowid())
877 }
878
879 async fn get_activity(&self, id: i64) -> Result<Option<WorkflowActivity>> {
880 let row = sqlx::query_as::<_, SqliteActivityRow>(
881 "SELECT id, workflow_id, seq, name, task_queue, input, status, result, error, attempt, max_attempts, initial_interval_secs, backoff_coefficient, start_to_close_secs, heartbeat_timeout_secs, claimed_by, scheduled_at, started_at, completed_at, last_heartbeat
882 FROM workflow.activities WHERE id = ?",
883 )
884 .bind(id)
885 .fetch_optional(&self.pool)
886 .await?;
887 Ok(row.map(Into::into))
888 }
889
890 async fn get_activity_by_workflow_seq(
891 &self,
892 workflow_id: &str,
893 seq: i32,
894 ) -> Result<Option<WorkflowActivity>> {
895 let row = sqlx::query_as::<_, SqliteActivityRow>(
896 "SELECT id, workflow_id, seq, name, task_queue, input, status, result, error, attempt, max_attempts, initial_interval_secs, backoff_coefficient, start_to_close_secs, heartbeat_timeout_secs, claimed_by, scheduled_at, started_at, completed_at, last_heartbeat
897 FROM workflow.activities WHERE workflow_id = ? AND seq = ?",
898 )
899 .bind(workflow_id)
900 .bind(seq)
901 .fetch_optional(&self.pool)
902 .await?;
903 Ok(row.map(Into::into))
904 }
905
906 async fn claim_activity(
907 &self,
908 task_queue: &str,
909 worker_id: &str,
910 ) -> Result<Option<WorkflowActivity>> {
911 let now = timestamp_now();
912 let row = sqlx::query_as::<_, SqliteActivityRow>(
913 "UPDATE workflow.activities SET status = 'RUNNING', claimed_by = ?, started_at = ?
914 WHERE id = (
915 SELECT id FROM workflow.activities
916 WHERE task_queue = ? AND status = 'PENDING'
917 ORDER BY scheduled_at ASC
918 LIMIT 1
919 )
920 RETURNING id, workflow_id, seq, name, task_queue, input, status, result, error, attempt, max_attempts, initial_interval_secs, backoff_coefficient, start_to_close_secs, heartbeat_timeout_secs, claimed_by, scheduled_at, started_at, completed_at, last_heartbeat",
921 )
922 .bind(worker_id)
923 .bind(now)
924 .bind(task_queue)
925 .fetch_optional(&self.pool)
926 .await?;
927 Ok(row.map(Into::into))
928 }
929
930 async fn requeue_activity_for_retry(
931 &self,
932 id: i64,
933 next_attempt: i32,
934 next_scheduled_at: f64,
935 ) -> Result<()> {
936 sqlx::query(
937 "UPDATE workflow.activities
938 SET status = 'PENDING', attempt = ?, scheduled_at = ?,
939 claimed_by = NULL, started_at = NULL, last_heartbeat = NULL,
940 error = NULL
941 WHERE id = ?",
942 )
943 .bind(next_attempt)
944 .bind(next_scheduled_at)
945 .bind(id)
946 .execute(&self.pool)
947 .await?;
948 Ok(())
949 }
950
951 async fn retry_failed_activity(
952 &self,
953 workflow_id: &str,
954 requested_by: &str,
955 reason: &str,
956 requested_at: f64,
957 ) -> Result<RetryFailedActivityResult> {
958 let mut tx = self.pool.begin().await?;
959 let workflow: Option<(String, Option<String>, Option<f64>)> = sqlx::query_as(
960 "SELECT status, parent_id, archived_at FROM workflow.workflows WHERE id = ?",
961 )
962 .bind(workflow_id)
963 .fetch_optional(&mut *tx)
964 .await?;
965 let Some((status, parent_id, archived_at)) = workflow else {
966 return Ok(RetryFailedActivityResult::NotFound);
967 };
968 if let Some(denial) = retry_denial(status, parent_id, archived_at) {
969 return Ok(denial);
970 }
971
972 let failed = sqlx::query_as::<_, SqliteActivityRow>(RETRY_ACTIVITY_SELECT)
973 .bind(workflow_id)
974 .fetch_optional(&mut *tx)
975 .await?;
976 let Some(failed) = failed else {
977 return Ok(RetryFailedActivityResult::NoFailedActivity);
978 };
979 let failed_event_seq: (i32,) = sqlx::query_as(
980 "SELECT seq FROM workflow.events
981 WHERE workflow_id = ? AND event_type = 'ActivityFailed'
982 ORDER BY seq DESC LIMIT 1",
983 )
984 .bind(workflow_id)
985 .fetch_one(&mut *tx)
986 .await?;
987 let invalidated =
988 sqlx::query("DELETE FROM workflow.activities WHERE workflow_id = ? AND seq > ?")
989 .bind(workflow_id)
990 .bind(failed.seq)
991 .execute(&mut *tx)
992 .await?
993 .rows_affected();
994 let activity = sqlx::query_as::<_, SqliteActivityRow>(RETRY_ACTIVITY_UPDATE)
995 .bind(requested_at)
996 .bind(failed.id)
997 .fetch_one(&mut *tx)
998 .await?;
999 sqlx::query("UPDATE workflow.events SET activity_id = ? WHERE activity_id = ?")
1002 .bind(NOT_A_SETTLEMENT)
1003 .bind(failed.id)
1004 .execute(&mut *tx)
1005 .await?;
1006 sqlx::query(
1007 "UPDATE workflow.workflows
1008 SET status = 'WAITING', result = NULL, error = NULL, completed_at = NULL,
1009 updated_at = ?, needs_dispatch = 0, dispatch_claimed_by = NULL,
1010 dispatch_last_heartbeat = NULL
1011 WHERE id = ?",
1012 )
1013 .bind(requested_at)
1014 .bind(workflow_id)
1015 .execute(&mut *tx)
1016 .await?;
1017 let event_seq: (i32,) = sqlx::query_as(
1018 "SELECT COALESCE(MAX(seq), 0) + 1 FROM workflow.events WHERE workflow_id = ?",
1019 )
1020 .bind(workflow_id)
1021 .fetch_one(&mut *tx)
1022 .await?;
1023 let payload = RetryEvent {
1024 activity_id: failed.id,
1025 activity_seq: failed.seq,
1026 activity_name: &failed.name,
1027 failed_event_seq: failed_event_seq.0,
1028 requested_by,
1029 reason,
1030 invalidated_activities: invalidated,
1031 }
1032 .payload();
1033 sqlx::query(
1034 "INSERT INTO workflow.events (workflow_id, seq, event_type, payload, timestamp)
1035 VALUES (?, ?, 'ActivityRetryRequested', ?, ?)",
1036 )
1037 .bind(workflow_id)
1038 .bind(event_seq.0)
1039 .bind(payload.to_string())
1040 .bind(requested_at)
1041 .execute(&mut *tx)
1042 .await?;
1043 tx.commit().await?;
1044 Ok(RetryFailedActivityResult::Retried(Box::new(
1045 RetriedActivity {
1046 activity: activity.into(),
1047 invalidated_activities: invalidated,
1048 },
1049 )))
1050 }
1051
1052 async fn complete_activity(
1053 &self,
1054 id: i64,
1055 result: Option<&str>,
1056 error: Option<&str>,
1057 failed: bool,
1058 ) -> Result<()> {
1059 let status = if failed { "FAILED" } else { "COMPLETED" };
1060 sqlx::query(
1061 "UPDATE workflow.activities SET status = ?, result = ?, error = ?, completed_at = ? WHERE id = ?",
1062 )
1063 .bind(status)
1064 .bind(result)
1065 .bind(error)
1066 .bind(timestamp_now())
1067 .bind(id)
1068 .execute(&self.pool)
1069 .await?;
1070 Ok(())
1071 }
1072
1073 async fn settle_activity(&self, settlement: &ActivitySettlement<'_>) -> Result<SettleOutcome> {
1074 let mut tx = self.pool.begin().await?;
1075 let current: Option<(String,)> =
1076 sqlx::query_as("SELECT status FROM workflow.activities WHERE id = ?")
1077 .bind(settlement.activity_id)
1078 .fetch_optional(&mut *tx)
1079 .await?;
1080 let Some((status,)) = current else {
1081 return Ok(SettleOutcome::Unknown);
1082 };
1083 let settled = matches!(status.as_str(), "COMPLETED" | "FAILED");
1084 let event_id: Option<(i64,)> = sqlx::query_as(
1085 "SELECT id FROM workflow.events
1086 WHERE workflow_id = ? AND activity_id = ?
1087 AND event_type IN ('ActivityCompleted', 'ActivityFailed')
1088 LIMIT 1",
1089 )
1090 .bind(settlement.workflow_id)
1091 .bind(settlement.activity_id)
1092 .fetch_optional(&mut *tx)
1093 .await?;
1094
1095 if !settled {
1096 sqlx::query(
1097 "UPDATE workflow.activities
1098 SET status = ?, result = ?, error = ?, completed_at = ?
1099 WHERE id = ?",
1100 )
1101 .bind(if settlement.failed {
1102 "FAILED"
1103 } else {
1104 "COMPLETED"
1105 })
1106 .bind(settlement.result)
1107 .bind(settlement.error)
1108 .bind(settlement.now)
1109 .bind(settlement.activity_id)
1110 .execute(&mut *tx)
1111 .await?;
1112 }
1113 if !settled || event_id.is_none() {
1117 let seq: (i32,) = sqlx::query_as(
1118 "SELECT COALESCE(MAX(seq), 0) + 1 FROM workflow.events WHERE workflow_id = ?",
1119 )
1120 .bind(settlement.workflow_id)
1121 .fetch_one(&mut *tx)
1122 .await?;
1123 sqlx::query(
1124 "INSERT INTO workflow.events (workflow_id, seq, event_type, payload, activity_id, timestamp)
1125 VALUES (?, ?, ?, ?, ?, ?)",
1126 )
1127 .bind(settlement.workflow_id)
1128 .bind(seq.0)
1129 .bind(settlement.event_type)
1130 .bind(settlement.payload)
1131 .bind(settlement.activity_id)
1132 .bind(settlement.now)
1133 .execute(&mut *tx)
1134 .await?;
1135 }
1136 sqlx::query("UPDATE workflow.workflows SET needs_dispatch = 1 WHERE id = ?")
1137 .bind(settlement.workflow_id)
1138 .execute(&mut *tx)
1139 .await?;
1140 tx.commit().await?;
1141 Ok(settle_outcome(settled, event_id.is_some()))
1142 }
1143
1144 async fn list_unsettled_activities(&self, limit: i64) -> Result<Vec<WorkflowActivity>> {
1145 let rows = sqlx::query_as::<_, SqliteActivityRow>(UNSETTLED_ACTIVITY_SELECT)
1146 .bind(limit)
1147 .fetch_all(&self.pool)
1148 .await?;
1149 Ok(rows.into_iter().map(Into::into).collect())
1150 }
1151
1152 async fn heartbeat_activity(&self, id: i64, _details: Option<&str>) -> Result<()> {
1153 sqlx::query("UPDATE workflow.activities SET last_heartbeat = ? WHERE id = ?")
1154 .bind(timestamp_now())
1155 .bind(id)
1156 .execute(&self.pool)
1157 .await?;
1158 Ok(())
1159 }
1160
1161 async fn get_timed_out_activities(&self, now: f64) -> Result<Vec<WorkflowActivity>> {
1162 let rows = sqlx::query_as::<_, SqliteActivityRow>(
1163 "SELECT id, workflow_id, seq, name, task_queue, input, status, result, error, attempt, max_attempts, initial_interval_secs, backoff_coefficient, start_to_close_secs, heartbeat_timeout_secs, claimed_by, scheduled_at, started_at, completed_at, last_heartbeat
1164 FROM workflow.activities
1165 WHERE status = 'RUNNING'
1166 AND heartbeat_timeout_secs IS NOT NULL
1167 AND (? - COALESCE(last_heartbeat, started_at)) > heartbeat_timeout_secs",
1168 )
1169 .bind(now)
1170 .fetch_all(&self.pool)
1171 .await?;
1172 Ok(rows.into_iter().map(Into::into).collect())
1173 }
1174
1175 async fn create_timer(&self, timer: &WorkflowTimer) -> Result<i64> {
1178 let res = sqlx::query(
1181 "INSERT OR IGNORE INTO workflow.timers (workflow_id, seq, fire_at, fired) VALUES (?, ?, ?, 0)",
1182 )
1183 .bind(&timer.workflow_id)
1184 .bind(timer.seq)
1185 .bind(timer.fire_at)
1186 .execute(&self.pool)
1187 .await?;
1188
1189 let id = res.last_insert_rowid();
1190 if id != 0 {
1191 return Ok(id);
1192 }
1193
1194 let (existing_id,): (i64,) =
1196 sqlx::query_as("SELECT id FROM workflow.timers WHERE workflow_id = ? AND seq = ?")
1197 .bind(&timer.workflow_id)
1198 .bind(timer.seq)
1199 .fetch_one(&self.pool)
1200 .await?;
1201 Ok(existing_id)
1202 }
1203
1204 async fn cancel_pending_activities(&self, workflow_id: &str) -> Result<u64> {
1205 let res = sqlx::query(
1206 "UPDATE workflow.activities SET status = 'CANCELLED', completed_at = ?
1207 WHERE workflow_id = ? AND status = 'PENDING'",
1208 )
1209 .bind(timestamp_now())
1210 .bind(workflow_id)
1211 .execute(&self.pool)
1212 .await?;
1213 Ok(res.rows_affected())
1214 }
1215
1216 async fn cancel_pending_timers(&self, workflow_id: &str) -> Result<u64> {
1217 let res = sqlx::query(
1218 "UPDATE workflow.timers SET fired = 1
1219 WHERE workflow_id = ? AND fired = 0",
1220 )
1221 .bind(workflow_id)
1222 .execute(&self.pool)
1223 .await?;
1224 Ok(res.rows_affected())
1225 }
1226
1227 async fn get_timer_by_workflow_seq(
1228 &self,
1229 workflow_id: &str,
1230 seq: i32,
1231 ) -> Result<Option<WorkflowTimer>> {
1232 let row = sqlx::query_as::<_, SqliteTimerRow>(
1233 "SELECT id, workflow_id, seq, fire_at, fired
1234 FROM workflow.timers WHERE workflow_id = ? AND seq = ?",
1235 )
1236 .bind(workflow_id)
1237 .bind(seq)
1238 .fetch_optional(&self.pool)
1239 .await?;
1240 Ok(row.map(Into::into))
1241 }
1242
1243 async fn fire_due_timers(&self, now: f64) -> Result<Vec<WorkflowTimer>> {
1244 let rows = sqlx::query_as::<_, SqliteTimerRow>(
1245 "UPDATE workflow.timers SET fired = 1
1246 WHERE fired = 0 AND fire_at <= ?
1247 RETURNING id, workflow_id, seq, fire_at, fired",
1248 )
1249 .bind(now)
1250 .fetch_all(&self.pool)
1251 .await?;
1252 Ok(rows.into_iter().map(Into::into).collect())
1253 }
1254
1255 async fn send_signal(&self, sig: &WorkflowSignal) -> Result<i64> {
1258 let res = sqlx::query(
1259 "INSERT INTO workflow.signals (workflow_id, name, payload, consumed, received_at) VALUES (?, ?, ?, 0, ?)",
1260 )
1261 .bind(&sig.workflow_id)
1262 .bind(&sig.name)
1263 .bind(&sig.payload)
1264 .bind(sig.received_at)
1265 .execute(&self.pool)
1266 .await?;
1267 Ok(res.last_insert_rowid())
1268 }
1269
1270 async fn deliver_signal(&self, sig: &WorkflowSignal, payload_json: &str) -> Result<i64> {
1271 let mut tx = self.pool.begin().await?;
1272 let res = sqlx::query(
1273 "INSERT INTO workflow.signals (workflow_id, name, payload, consumed, received_at) VALUES (?, ?, ?, 0, ?)",
1274 )
1275 .bind(&sig.workflow_id)
1276 .bind(&sig.name)
1277 .bind(&sig.payload)
1278 .bind(sig.received_at)
1279 .execute(&mut *tx)
1280 .await?;
1281 let signal_id = res.last_insert_rowid();
1282 let seq: (i32,) = sqlx::query_as(
1283 "SELECT COALESCE(MAX(seq), 0) + 1 FROM workflow.events WHERE workflow_id = ?",
1284 )
1285 .bind(&sig.workflow_id)
1286 .fetch_one(&mut *tx)
1287 .await?;
1288 sqlx::query(
1289 "INSERT INTO workflow.events (workflow_id, seq, event_type, payload, timestamp)
1290 VALUES (?, ?, 'SignalReceived', ?, ?)",
1291 )
1292 .bind(&sig.workflow_id)
1293 .bind(seq.0)
1294 .bind(payload_json)
1295 .bind(sig.received_at)
1296 .execute(&mut *tx)
1297 .await?;
1298 sqlx::query("UPDATE workflow.workflows SET needs_dispatch = 1 WHERE id = ?")
1299 .bind(&sig.workflow_id)
1300 .execute(&mut *tx)
1301 .await?;
1302 tx.commit().await?;
1303 Ok(signal_id)
1304 }
1305
1306 async fn consume_signals(&self, workflow_id: &str, name: &str) -> Result<Vec<WorkflowSignal>> {
1307 let rows = sqlx::query_as::<_, SqliteSignalRow>(
1308 "UPDATE workflow.signals SET consumed = 1
1309 WHERE workflow_id = ? AND name = ? AND consumed = 0
1310 RETURNING id, workflow_id, name, payload, consumed, received_at",
1311 )
1312 .bind(workflow_id)
1313 .bind(name)
1314 .fetch_all(&self.pool)
1315 .await?;
1316 Ok(rows.into_iter().map(Into::into).collect())
1317 }
1318
1319 async fn create_schedule(&self, sched: &WorkflowSchedule) -> Result<()> {
1322 sqlx::query(
1323 "INSERT INTO workflow.schedules (name, namespace, workflow_type, cron_expr, timezone, input, task_queue, overlap_policy, paused, last_run_at, next_run_at, last_workflow_id, created_at)
1324 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1325 )
1326 .bind(&sched.name)
1327 .bind(&sched.namespace)
1328 .bind(&sched.workflow_type)
1329 .bind(&sched.cron_expr)
1330 .bind(&sched.timezone)
1331 .bind(&sched.input)
1332 .bind(&sched.task_queue)
1333 .bind(&sched.overlap_policy)
1334 .bind(sched.paused)
1335 .bind(sched.last_run_at)
1336 .bind(crate::scheduler::seed_next_run(sched))
1337 .bind(&sched.last_workflow_id)
1338 .bind(sched.created_at)
1339 .execute(&self.pool)
1340 .await?;
1341 Ok(())
1342 }
1343
1344 async fn get_schedule(&self, namespace: &str, name: &str) -> Result<Option<WorkflowSchedule>> {
1345 let row = sqlx::query_as::<_, SqliteScheduleRow>(
1346 "SELECT name, namespace, workflow_type, cron_expr, timezone, input, task_queue, overlap_policy, paused, last_run_at, next_run_at, last_workflow_id, created_at
1347 FROM workflow.schedules WHERE namespace = ? AND name = ?",
1348 )
1349 .bind(namespace)
1350 .bind(name)
1351 .fetch_optional(&self.pool)
1352 .await?;
1353 Ok(row.map(Into::into))
1354 }
1355
1356 async fn list_schedules(&self, namespace: &str) -> Result<Vec<WorkflowSchedule>> {
1357 let rows = sqlx::query_as::<_, SqliteScheduleRow>(
1358 "SELECT name, namespace, workflow_type, cron_expr, timezone, input, task_queue, overlap_policy, paused, last_run_at, next_run_at, last_workflow_id, created_at
1359 FROM workflow.schedules WHERE namespace = ? ORDER BY name",
1360 )
1361 .bind(namespace)
1362 .fetch_all(&self.pool)
1363 .await?;
1364 Ok(rows.into_iter().map(Into::into).collect())
1365 }
1366
1367 async fn update_schedule_last_run(
1368 &self,
1369 namespace: &str,
1370 name: &str,
1371 last_run_at: f64,
1372 next_run_at: f64,
1373 workflow_id: &str,
1374 ) -> Result<()> {
1375 sqlx::query(
1376 "UPDATE workflow.schedules SET last_run_at = ?, next_run_at = ?, last_workflow_id = ? WHERE namespace = ? AND name = ?",
1377 )
1378 .bind(last_run_at)
1379 .bind(next_run_at)
1380 .bind(workflow_id)
1381 .bind(namespace)
1382 .bind(name)
1383 .execute(&self.pool)
1384 .await?;
1385 Ok(())
1386 }
1387
1388 async fn delete_schedule(&self, namespace: &str, name: &str) -> Result<bool> {
1389 let res = sqlx::query("DELETE FROM workflow.schedules WHERE namespace = ? AND name = ?")
1390 .bind(namespace)
1391 .bind(name)
1392 .execute(&self.pool)
1393 .await?;
1394 Ok(res.rows_affected() > 0)
1395 }
1396
1397 async fn list_archivable_workflows(
1398 &self,
1399 cutoff: f64,
1400 limit: i64,
1401 ) -> Result<Vec<WorkflowRecord>> {
1402 let rows = sqlx::query_as::<_, SqliteWorkflowRow>(
1403 "SELECT id, namespace, run_id, workflow_type, task_queue, status, input, result, error, parent_id, claimed_by, search_attributes, archived_at, archive_uri, created_at, updated_at, completed_at
1404 FROM workflow.workflows
1405 WHERE status IN ('COMPLETED', 'FAILED', 'CANCELLED', 'TIMED_OUT')
1406 AND completed_at IS NOT NULL
1407 AND completed_at < ?
1408 AND archived_at IS NULL
1409 ORDER BY completed_at ASC
1410 LIMIT ?",
1411 )
1412 .bind(cutoff)
1413 .bind(limit)
1414 .fetch_all(&self.pool)
1415 .await?;
1416 Ok(rows.into_iter().map(Into::into).collect())
1417 }
1418
1419 async fn mark_archived_and_purge(
1420 &self,
1421 workflow_id: &str,
1422 archive_uri: &str,
1423 archived_at: f64,
1424 ) -> Result<()> {
1425 let mut tx = self.pool.begin().await?;
1426 sqlx::query("DELETE FROM workflow.events WHERE workflow_id = ?")
1427 .bind(workflow_id)
1428 .execute(&mut *tx)
1429 .await?;
1430 sqlx::query("DELETE FROM workflow.activities WHERE workflow_id = ?")
1431 .bind(workflow_id)
1432 .execute(&mut *tx)
1433 .await?;
1434 sqlx::query("DELETE FROM workflow.timers WHERE workflow_id = ?")
1435 .bind(workflow_id)
1436 .execute(&mut *tx)
1437 .await?;
1438 sqlx::query("DELETE FROM workflow.signals WHERE workflow_id = ?")
1439 .bind(workflow_id)
1440 .execute(&mut *tx)
1441 .await?;
1442 sqlx::query("DELETE FROM workflow.snapshots WHERE workflow_id = ?")
1443 .bind(workflow_id)
1444 .execute(&mut *tx)
1445 .await?;
1446 sqlx::query("UPDATE workflow.workflows SET archived_at = ?, archive_uri = ? WHERE id = ?")
1447 .bind(archived_at)
1448 .bind(archive_uri)
1449 .bind(workflow_id)
1450 .execute(&mut *tx)
1451 .await?;
1452 tx.commit().await?;
1453 Ok(())
1454 }
1455
1456 async fn upsert_search_attributes(&self, workflow_id: &str, patch_json: &str) -> Result<()> {
1457 let current: Option<(Option<String>,)> =
1460 sqlx::query_as("SELECT search_attributes FROM workflow.workflows WHERE id = ?")
1461 .bind(workflow_id)
1462 .fetch_optional(&self.pool)
1463 .await?;
1464 let merged = merge_search_attrs(current.and_then(|(s,)| s).as_deref(), patch_json)?;
1465 sqlx::query("UPDATE workflow.workflows SET search_attributes = ? WHERE id = ?")
1466 .bind(merged)
1467 .bind(workflow_id)
1468 .execute(&self.pool)
1469 .await?;
1470 Ok(())
1471 }
1472
1473 async fn update_schedule(
1474 &self,
1475 namespace: &str,
1476 name: &str,
1477 patch: &SchedulePatch,
1478 ) -> Result<Option<WorkflowSchedule>> {
1479 let mut sets: Vec<&'static str> = Vec::new();
1482 if patch.cron_expr.is_some() {
1483 sets.push("cron_expr = ?");
1484 }
1485 if patch.timezone.is_some() {
1486 sets.push("timezone = ?");
1487 }
1488 if patch.input.is_some() {
1489 sets.push("input = ?");
1490 }
1491 if patch.task_queue.is_some() {
1492 sets.push("task_queue = ?");
1493 }
1494 if patch.overlap_policy.is_some() {
1495 sets.push("overlap_policy = ?");
1496 }
1497 if sets.is_empty() {
1499 return self.get_schedule(namespace, name).await;
1500 }
1501
1502 let sql = format!(
1503 "UPDATE workflow.schedules SET {} WHERE namespace = ? AND name = ?",
1504 sets.join(", ")
1505 );
1506 let mut q = sqlx::query(&sql);
1507 if let Some(ref v) = patch.cron_expr {
1508 q = q.bind(v);
1509 }
1510 if let Some(ref v) = patch.timezone {
1511 q = q.bind(v);
1512 }
1513 if let Some(ref v) = patch.input {
1514 q = q.bind(v.to_string());
1515 }
1516 if let Some(ref v) = patch.task_queue {
1517 q = q.bind(v);
1518 }
1519 if let Some(ref v) = patch.overlap_policy {
1520 q = q.bind(v);
1521 }
1522 let res = q.bind(namespace).bind(name).execute(&self.pool).await?;
1523 if res.rows_affected() == 0 {
1524 return Ok(None);
1525 }
1526 self.get_schedule(namespace, name).await
1527 }
1528
1529 async fn set_schedule_paused(
1530 &self,
1531 namespace: &str,
1532 name: &str,
1533 paused: bool,
1534 ) -> Result<Option<WorkflowSchedule>> {
1535 let res = sqlx::query(
1536 "UPDATE workflow.schedules SET paused = ? WHERE namespace = ? AND name = ?",
1537 )
1538 .bind(paused)
1539 .bind(namespace)
1540 .bind(name)
1541 .execute(&self.pool)
1542 .await?;
1543 if res.rows_affected() == 0 {
1544 return Ok(None);
1545 }
1546 self.get_schedule(namespace, name).await
1547 }
1548
1549 async fn register_worker(&self, w: &WorkflowWorker) -> Result<()> {
1552 sqlx::query(
1553 "INSERT OR REPLACE INTO workflow.workers (id, namespace, identity, task_queue, workflows, activities, max_concurrent_workflows, max_concurrent_activities, active_tasks, last_heartbeat, registered_at)
1554 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
1555 )
1556 .bind(&w.id)
1557 .bind(&w.namespace)
1558 .bind(&w.identity)
1559 .bind(&w.task_queue)
1560 .bind(&w.workflows)
1561 .bind(&w.activities)
1562 .bind(w.max_concurrent_workflows)
1563 .bind(w.max_concurrent_activities)
1564 .bind(w.active_tasks)
1565 .bind(w.last_heartbeat)
1566 .bind(w.registered_at)
1567 .execute(&self.pool)
1568 .await?;
1569 Ok(())
1570 }
1571
1572 async fn heartbeat_worker(&self, id: &str, now: f64) -> Result<bool> {
1573 let res = sqlx::query("UPDATE workflow.workers SET last_heartbeat = ? WHERE id = ?")
1574 .bind(now)
1575 .bind(id)
1576 .execute(&self.pool)
1577 .await?;
1578 Ok(res.rows_affected() > 0)
1579 }
1580
1581 async fn list_workers(&self, namespace: &str) -> Result<Vec<WorkflowWorker>> {
1582 let rows = sqlx::query_as::<_, SqliteWorkerRow>(
1583 "SELECT id, namespace, identity, task_queue, workflows, activities, max_concurrent_workflows, max_concurrent_activities, active_tasks, last_heartbeat, registered_at
1584 FROM workflow.workers WHERE namespace = ? ORDER BY registered_at",
1585 )
1586 .bind(namespace)
1587 .fetch_all(&self.pool)
1588 .await?;
1589 Ok(rows.into_iter().map(Into::into).collect())
1590 }
1591
1592 async fn remove_dead_workers(&self, cutoff: f64) -> Result<Vec<String>> {
1593 let rows: Vec<(String,)> =
1594 sqlx::query_as("SELECT id FROM workflow.workers WHERE last_heartbeat < ?")
1595 .bind(cutoff)
1596 .fetch_all(&self.pool)
1597 .await?;
1598 let ids: Vec<String> = rows.into_iter().map(|r| r.0).collect();
1599 if !ids.is_empty() {
1600 sqlx::query("DELETE FROM workflow.workers WHERE last_heartbeat < ?")
1601 .bind(cutoff)
1602 .execute(&self.pool)
1603 .await?;
1604 }
1605 Ok(ids)
1606 }
1607
1608 async fn list_child_workflows(&self, parent_id: &str) -> Result<Vec<WorkflowRecord>> {
1611 let rows = sqlx::query_as::<_, SqliteWorkflowRow>(
1612 "SELECT id, namespace, run_id, workflow_type, task_queue, status, input, result, error, parent_id, claimed_by, search_attributes, archived_at, archive_uri, created_at, updated_at, completed_at
1613 FROM workflow.workflows WHERE parent_id = ? ORDER BY created_at ASC",
1614 )
1615 .bind(parent_id)
1616 .fetch_all(&self.pool)
1617 .await?;
1618 Ok(rows.into_iter().map(Into::into).collect())
1619 }
1620
1621 async fn create_snapshot(
1624 &self,
1625 workflow_id: &str,
1626 event_seq: i32,
1627 state_json: &str,
1628 ) -> Result<()> {
1629 sqlx::query(
1630 "INSERT OR REPLACE INTO workflow.snapshots (workflow_id, event_seq, state_json, created_at)
1631 VALUES (?, ?, ?, ?)",
1632 )
1633 .bind(workflow_id)
1634 .bind(event_seq)
1635 .bind(state_json)
1636 .bind(timestamp_now())
1637 .execute(&self.pool)
1638 .await?;
1639 Ok(())
1640 }
1641
1642 async fn get_latest_snapshot(&self, workflow_id: &str) -> Result<Option<WorkflowSnapshot>> {
1643 let row = sqlx::query_as::<_, (String, i32, String, f64)>(
1644 "SELECT workflow_id, event_seq, state_json, created_at
1645 FROM workflow.snapshots WHERE workflow_id = ?
1646 ORDER BY event_seq DESC LIMIT 1",
1647 )
1648 .bind(workflow_id)
1649 .fetch_optional(&self.pool)
1650 .await?;
1651
1652 Ok(row.map(
1653 |(workflow_id, event_seq, state_json, created_at)| WorkflowSnapshot {
1654 workflow_id,
1655 event_seq,
1656 state_json,
1657 created_at,
1658 },
1659 ))
1660 }
1661
1662 async fn get_queue_stats(&self, namespace: &str) -> Result<Vec<QueueStats>> {
1665 let rows = sqlx::query_as::<_, (String, i64, i64)>(
1667 "SELECT a.task_queue,
1668 SUM(CASE WHEN a.status = 'PENDING' THEN 1 ELSE 0 END),
1669 SUM(CASE WHEN a.status = 'RUNNING' THEN 1 ELSE 0 END)
1670 FROM workflow.activities a
1671 INNER JOIN workflow.workflows w ON w.id = a.workflow_id
1672 WHERE w.namespace = ?
1673 GROUP BY a.task_queue",
1674 )
1675 .bind(namespace)
1676 .fetch_all(&self.pool)
1677 .await?;
1678
1679 let mut stats: Vec<QueueStats> = rows
1680 .into_iter()
1681 .map(|(queue, pending, running)| QueueStats {
1682 queue,
1683 pending_activities: pending,
1684 running_activities: running,
1685 workers: 0,
1686 })
1687 .collect();
1688
1689 let worker_rows = sqlx::query_as::<_, (String, i64)>(
1691 "SELECT task_queue, COUNT(*) FROM workflow.workers WHERE namespace = ? GROUP BY task_queue",
1692 )
1693 .bind(namespace)
1694 .fetch_all(&self.pool)
1695 .await?;
1696
1697 for (queue, count) in worker_rows {
1698 if let Some(s) = stats.iter_mut().find(|s| s.queue == queue) {
1699 s.workers = count;
1700 } else {
1701 stats.push(QueueStats {
1702 queue,
1703 pending_activities: 0,
1704 running_activities: 0,
1705 workers: count,
1706 });
1707 }
1708 }
1709
1710 stats.sort_by(|a, b| a.queue.cmp(&b.queue));
1711 Ok(stats)
1712 }
1713
1714 async fn try_acquire_scheduler_lock(&self) -> Result<bool> {
1717 self.refresh_engine_lock().await.ok();
1720 Ok(true)
1721 }
1722}
1723
1724fn timestamp_now() -> f64 {
1725 std::time::SystemTime::now()
1726 .duration_since(std::time::UNIX_EPOCH)
1727 .unwrap()
1728 .as_secs_f64()
1729}
1730
1731pub(crate) fn merge_search_attrs(current: Option<&str>, patch_json: &str) -> Result<String> {
1734 let mut current_map: serde_json::Map<String, serde_json::Value> = current
1735 .and_then(|s| serde_json::from_str::<serde_json::Value>(s).ok())
1736 .and_then(|v| v.as_object().cloned())
1737 .unwrap_or_default();
1738 let patch: serde_json::Value = serde_json::from_str(patch_json)
1739 .map_err(|e| anyhow::anyhow!("invalid search_attributes patch: {e}"))?;
1740 let patch_obj = patch
1741 .as_object()
1742 .ok_or_else(|| anyhow::anyhow!("search_attributes patch must be a JSON object"))?;
1743 for (k, v) in patch_obj {
1744 current_map.insert(k.clone(), v.clone());
1745 }
1746 Ok(serde_json::Value::Object(current_map).to_string())
1747}
1748
1749#[derive(sqlx::FromRow)]
1752struct SqliteWorkflowRow {
1753 id: String,
1754 namespace: String,
1755 run_id: String,
1756 workflow_type: String,
1757 task_queue: String,
1758 status: String,
1759 input: Option<String>,
1760 result: Option<String>,
1761 error: Option<String>,
1762 parent_id: Option<String>,
1763 claimed_by: Option<String>,
1764 search_attributes: Option<String>,
1765 archived_at: Option<f64>,
1766 archive_uri: Option<String>,
1767 created_at: f64,
1768 updated_at: f64,
1769 completed_at: Option<f64>,
1770}
1771
1772impl From<SqliteWorkflowRow> for WorkflowRecord {
1773 fn from(r: SqliteWorkflowRow) -> Self {
1774 Self {
1775 id: r.id,
1776 namespace: r.namespace,
1777 run_id: r.run_id,
1778 workflow_type: r.workflow_type,
1779 task_queue: r.task_queue,
1780 status: r.status,
1781 input: r.input,
1782 result: r.result,
1783 error: r.error,
1784 parent_id: r.parent_id,
1785 claimed_by: r.claimed_by,
1786 search_attributes: r.search_attributes,
1787 archived_at: r.archived_at,
1788 archive_uri: r.archive_uri,
1789 created_at: r.created_at,
1790 updated_at: r.updated_at,
1791 completed_at: r.completed_at,
1792 }
1793 }
1794}
1795
1796#[derive(sqlx::FromRow)]
1797struct SqliteEventRow {
1798 id: i64,
1799 workflow_id: String,
1800 seq: i32,
1801 event_type: String,
1802 payload: Option<String>,
1803 timestamp: f64,
1804}
1805
1806impl From<SqliteEventRow> for WorkflowEvent {
1807 fn from(r: SqliteEventRow) -> Self {
1808 Self {
1809 id: Some(r.id),
1810 workflow_id: r.workflow_id,
1811 seq: r.seq,
1812 event_type: r.event_type,
1813 payload: r.payload,
1814 timestamp: r.timestamp,
1815 }
1816 }
1817}
1818
1819#[derive(sqlx::FromRow)]
1820struct SqliteActivityRow {
1821 id: i64,
1822 workflow_id: String,
1823 seq: i32,
1824 name: String,
1825 task_queue: String,
1826 input: Option<String>,
1827 status: String,
1828 result: Option<String>,
1829 error: Option<String>,
1830 attempt: i32,
1831 max_attempts: i32,
1832 initial_interval_secs: f64,
1833 backoff_coefficient: f64,
1834 start_to_close_secs: f64,
1835 heartbeat_timeout_secs: Option<f64>,
1836 claimed_by: Option<String>,
1837 scheduled_at: f64,
1838 started_at: Option<f64>,
1839 completed_at: Option<f64>,
1840 last_heartbeat: Option<f64>,
1841}
1842
1843impl From<SqliteActivityRow> for WorkflowActivity {
1844 fn from(r: SqliteActivityRow) -> Self {
1845 Self {
1846 id: Some(r.id),
1847 workflow_id: r.workflow_id,
1848 seq: r.seq,
1849 name: r.name,
1850 task_queue: r.task_queue,
1851 input: r.input,
1852 status: r.status,
1853 result: r.result,
1854 error: r.error,
1855 attempt: r.attempt,
1856 max_attempts: r.max_attempts,
1857 initial_interval_secs: r.initial_interval_secs,
1858 backoff_coefficient: r.backoff_coefficient,
1859 start_to_close_secs: r.start_to_close_secs,
1860 heartbeat_timeout_secs: r.heartbeat_timeout_secs,
1861 claimed_by: r.claimed_by,
1862 scheduled_at: r.scheduled_at,
1863 started_at: r.started_at,
1864 completed_at: r.completed_at,
1865 last_heartbeat: r.last_heartbeat,
1866 }
1867 }
1868}
1869
1870#[derive(sqlx::FromRow)]
1871struct SqliteTimerRow {
1872 id: i64,
1873 workflow_id: String,
1874 seq: i32,
1875 fire_at: f64,
1876 fired: bool,
1877}
1878
1879impl From<SqliteTimerRow> for WorkflowTimer {
1880 fn from(r: SqliteTimerRow) -> Self {
1881 Self {
1882 id: Some(r.id),
1883 workflow_id: r.workflow_id,
1884 seq: r.seq,
1885 fire_at: r.fire_at,
1886 fired: r.fired,
1887 }
1888 }
1889}
1890
1891#[derive(sqlx::FromRow)]
1892struct SqliteSignalRow {
1893 id: i64,
1894 workflow_id: String,
1895 name: String,
1896 payload: Option<String>,
1897 consumed: bool,
1898 received_at: f64,
1899}
1900
1901impl From<SqliteSignalRow> for WorkflowSignal {
1902 fn from(r: SqliteSignalRow) -> Self {
1903 Self {
1904 id: Some(r.id),
1905 workflow_id: r.workflow_id,
1906 name: r.name,
1907 payload: r.payload,
1908 consumed: r.consumed,
1909 received_at: r.received_at,
1910 }
1911 }
1912}
1913
1914#[derive(sqlx::FromRow)]
1915struct SqliteScheduleRow {
1916 name: String,
1917 namespace: String,
1918 workflow_type: String,
1919 cron_expr: String,
1920 timezone: String,
1921 input: Option<String>,
1922 task_queue: String,
1923 overlap_policy: String,
1924 paused: bool,
1925 last_run_at: Option<f64>,
1926 next_run_at: Option<f64>,
1927 last_workflow_id: Option<String>,
1928 created_at: f64,
1929}
1930
1931impl From<SqliteScheduleRow> for WorkflowSchedule {
1932 fn from(r: SqliteScheduleRow) -> Self {
1933 Self {
1934 name: r.name,
1935 namespace: r.namespace,
1936 workflow_type: r.workflow_type,
1937 cron_expr: r.cron_expr,
1938 timezone: r.timezone,
1939 input: r.input,
1940 task_queue: r.task_queue,
1941 overlap_policy: r.overlap_policy,
1942 paused: r.paused,
1943 last_run_at: r.last_run_at,
1944 next_run_at: r.next_run_at,
1945 last_workflow_id: r.last_workflow_id,
1946 created_at: r.created_at,
1947 }
1948 }
1949}
1950
1951#[derive(sqlx::FromRow)]
1952struct SqliteWorkerRow {
1953 id: String,
1954 namespace: String,
1955 identity: String,
1956 task_queue: String,
1957 workflows: Option<String>,
1958 activities: Option<String>,
1959 max_concurrent_workflows: i32,
1960 max_concurrent_activities: i32,
1961 active_tasks: i32,
1962 last_heartbeat: f64,
1963 registered_at: f64,
1964}
1965
1966impl From<SqliteWorkerRow> for WorkflowWorker {
1967 fn from(r: SqliteWorkerRow) -> Self {
1968 Self {
1969 id: r.id,
1970 namespace: r.namespace,
1971 identity: r.identity,
1972 task_queue: r.task_queue,
1973 workflows: r.workflows,
1974 activities: r.activities,
1975 max_concurrent_workflows: r.max_concurrent_workflows,
1976 max_concurrent_activities: r.max_concurrent_activities,
1977 active_tasks: r.active_tasks,
1978 last_heartbeat: r.last_heartbeat,
1979 registered_at: r.registered_at,
1980 }
1981 }
1982}