fluxpro-engine 0.1.10

A feature-gated workflow and process engine
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
//! Atomic task snapshots, fenced transition commits, and instance startup.

use super::FluxproDbServiceImpl;
use crate::models::commands::start_process_instance::StartProcessInstance;
use crate::models::context_map::context_map::{ContextMap, ContextValue};
use crate::models::context_map::context_patcher::{ContextPatcher, ContextPatcherOp};
use crate::models::execution_log::ExecutionLogEvent;
use crate::models::id_field::IdField;
use crate::models::process_def::{Node, ProcessDefinition, SetStageWithReason};
use crate::models::queue::queue_task::{FluxproQueueTask, FluxproQueueTaskDefinition};
use chrono::{DateTime, Utc};
use sqlx::{PgConnection, Postgres, Row, Transaction};
use uuid::Uuid;

/// Consistent input for an execution attempt, read while its lease is valid.
#[derive(Clone)]
pub struct TaskExecutionSnapshot {
    /// Instance revision used to reject stale execution results.
    pub revision: i64,
    /// Identity of the current node visit, distinct from its workflow node ID.
    pub visit_id: Option<Uuid>,
    /// Whether an event has already completed the current wait.
    pub wait_completed: bool,
    /// Wait visit bound to this timeout or signal, if any.
    pub task_visit_id: Option<Uuid>,
    /// Definition version permanently associated with this instance.
    pub definition: ProcessDefinition,
    /// Persisted current node; absent before the first node entry.
    pub current_node: Option<Node>,
    /// Context visible to this execution attempt.
    pub context: ContextMap,
}

/// A successor or timeout to insert in the same transaction as its predecessor completes.
pub struct ScheduledTask {
    /// Existing wire-compatible task payload.
    pub task: FluxproQueueTask,
    /// Earliest execution time; `None` means immediately.
    pub run_after: Option<DateTime<Utc>>,
    /// Originating visit for timeout events; node tasks leave this unset.
    pub visit_id: Option<Uuid>,
    /// Signals which cancel this timeout.
    pub cancel_signals: Vec<IdField>,
}

/// Database effects prepared without holding a transaction across host handler calls.
#[derive(Default)]
pub struct TaskExecutionChanges {
    /// Node to enter, using the source queue UUID as its stable visit ID.
    pub enter_node: Option<IdField>,
    /// Optional stage assignment and its reason.
    pub stage: Option<SetStageWithReason>,
    /// Context keys to merge into the default scope.
    pub context: Option<ContextMap>,
    /// Ordered handler changes; removals delete stored keys in the default scope.
    pub context_patch: Option<ContextPatcher>,
    /// Stops execution and retains this task with a durable incident (kind, reason).
    pub incident: Option<(String, String)>,
    /// Consumes the current wait and cancels its remaining timeout tasks.
    pub close_wait: bool,
    /// Successors and timeouts committed with this transition.
    pub tasks: Vec<ScheduledTask>,
    /// Durable events committed with their corresponding state changes.
    pub events: Vec<ExecutionLogEvent>,
    /// Retains and releases the source task for retry instead of deleting it.
    pub retry_at: Option<DateTime<Utc>>,
}

async fn lock_task(
    tx: &mut Transaction<'_, Postgres>,
    task: &FluxproQueueTaskDefinition,
) -> anyhow::Result<Option<Uuid>> {
    let row = sqlx::query(
        "select node_visit_id, task from fluxpro.queue_runner \
         where uuid=$1 and lock_key=$2 and locked_by > clock_timestamp() for update",
    )
    .bind(task.uuid)
    .bind(&task.lock_key)
    .fetch_optional(&mut **tx)
    .await?
    .ok_or_else(|| anyhow::anyhow!("task {} no longer has a valid lease", task.uuid))?;
    let stored: FluxproQueueTask = serde_json::from_value(row.try_get("task")?)?;
    anyhow::ensure!(
        serde_json::to_value(stored)? == serde_json::to_value(&task.task)?,
        "task payload changed after dequeue"
    );
    Ok(row.try_get("node_visit_id")?)
}

pub(super) async fn load(
    db: &FluxproDbServiceImpl,
    task: &FluxproQueueTaskDefinition,
) -> anyhow::Result<TaskExecutionSnapshot> {
    let mut tx = db.db_pool.begin().await?;
    // All transition operations lock the instance before the queue row.
    let instance = sqlx::query(
        "select execution_state, revision, node_visit_id, wait_completed, current_node_ref, process_def_uuid \
         from fluxpro.process_instance where token=$1 for update",
    )
    .bind(task.task.process_token().get_id())
    .fetch_one(&mut *tx)
    .await?;
    anyhow::ensure!(
        instance.try_get::<String, _>("execution_state")? == "running",
        "instance is suspended"
    );
    let mut task_visit_id = lock_task(&mut tx, task).await?;
    let definition_uuid: Uuid = instance.try_get("process_def_uuid")?;
    let raw: serde_json::Value =
        sqlx::query_scalar("select definition from fluxpro.process_definition where uuid=$1")
            .bind(definition_uuid)
            .fetch_one(&mut *tx)
            .await?;
    let mut definition: ProcessDefinition = serde_json::from_value(raw)?;
    definition.uuid = Some(definition_uuid);
    let current_ref: Option<Uuid> = instance.try_get("current_node_ref")?;
    let current_node = if let Some(current_ref) = current_ref {
        let raw = sqlx::query_scalar::<_, serde_json::Value>(
            "select definition from fluxpro.process_def_node where uuid=$1 and process_def_uuid=$2",
        )
        .bind(current_ref)
        .bind(definition_uuid)
        .fetch_one(&mut *tx)
        .await?;
        Some(serde_json::from_value::<Node>(raw)?)
    } else {
        None
    };
    let visit_id: Option<Uuid> = instance.try_get("node_visit_id")?;
    if task_visit_id.is_none()
        && matches!(&task.task, FluxproQueueTask::ProcessSignal { signal, .. }
            if accepts_signal(current_node.as_ref(), &signal.signal))
    {
        task_visit_id = visit_id;
        sqlx::query("update fluxpro.queue_runner set node_visit_id=$2 where uuid=$1")
            .bind(task.uuid)
            .bind(task_visit_id)
            .execute(&mut *tx)
            .await?;
    }
    let rows = sqlx::query(
        "select name, value from fluxpro.process_instance_context_variable \
         where process_instance_uuid=(select uuid from fluxpro.process_instance where token=$1)",
    )
    .bind(task.task.process_token().get_id())
    .fetch_all(&mut *tx)
    .await?;
    let mut context = ContextMap::default();
    for row in rows {
        context.0.insert(
            IdField::new(row.try_get::<String, _>("name")?)?,
            serde_json::from_value::<ContextValue>(row.try_get("value")?)?,
        );
    }
    let snapshot = TaskExecutionSnapshot {
        revision: instance.try_get("revision")?,
        visit_id,
        wait_completed: instance.try_get("wait_completed")?,
        task_visit_id,
        definition,
        current_node,
        context,
    };
    tx.commit().await?;
    Ok(snapshot)
}

pub(super) fn accepts_signal(node: Option<&Node>, signal: &IdField) -> bool {
    match node {
        Some(Node::UserTask {
            wait_for: Some(wait),
            ..
        })
        | Some(Node::Wait { wait_for: wait, .. }) => wait.signals().contains(signal),
        _ => false,
    }
}

pub(super) async fn apply_patch(
    connection: &mut PgConnection,
    instance_uuid: Uuid,
    patch: &ContextPatcher,
) -> anyhow::Result<()> {
    for op in patch.ops() {
        match op {
            ContextPatcherOp::Set { key, value } => {
                merge_context(
                    &mut *connection,
                    instance_uuid,
                    &ContextMap([(key.clone(), value.clone())].into()),
                )
                .await?;
            }
            ContextPatcherOp::Remove { key } => {
                sqlx::query("delete from fluxpro.process_instance_context_variable where process_instance_uuid=$1 and scope='_' and name=$2")
                        .bind(instance_uuid).bind(key.get_id()).execute(&mut *connection).await?;
            }
        }
    }
    Ok(())
}

async fn merge_context(
    connection: &mut PgConnection,
    instance_uuid: Uuid,
    context: &ContextMap,
) -> anyhow::Result<()> {
    for (key, value) in &context.0 {
        sqlx::query(
            "insert into fluxpro.process_instance_context_variable \
             (process_instance_uuid, scope, name, value) values ($1, '_', $2, $3) \
             on conflict (process_instance_uuid, scope, name) do update set value=excluded.value",
        )
        .bind(instance_uuid)
        .bind(key.get_id())
        .bind(serde_json::to_value(value)?)
        .execute(&mut *connection)
        .await?;
    }
    Ok(())
}

async fn assign_stage(
    connection: &mut PgConnection,
    instance_uuid: Uuid,
    definition_uuid: Uuid,
    stage: &SetStageWithReason,
) -> anyhow::Result<()> {
    let stage_uuid: Uuid = sqlx::query_scalar(
        "select uuid from fluxpro.process_stage where process_def_uuid=$1 and stage_id=$2",
    )
    .bind(definition_uuid)
    .bind(stage.stage.get_id())
    .fetch_one(&mut *connection)
    .await?;
    sqlx::query("update fluxpro.process_instance set current_stage=$2, current_stage_reason=$3 where uuid=$1")
        .bind(instance_uuid).bind(stage_uuid).bind(&stage.reason).execute(&mut *connection).await?;
    sqlx::query("insert into fluxpro.process_instance_stage_log (process_instance_uuid, stage_uuid, reason) values ($1,$2,$3)")
        .bind(instance_uuid).bind(stage_uuid).bind(&stage.reason).execute(&mut *connection).await?;
    Ok(())
}

async fn enqueue(connection: &mut PgConnection, pending: &ScheduledTask) -> anyhow::Result<()> {
    let target = match &pending.task {
        FluxproQueueTask::ProcessNode { node, .. } => Some(node.id()),
        FluxproQueueTask::ProcessEvent { on_time, .. } => Some(on_time),
        FluxproQueueTask::ProcessSignal { .. } => None,
    };
    if let Some(target) = target {
        let _: Uuid = sqlx::query_scalar(
            "select n.uuid from fluxpro.process_def_node n \
            join fluxpro.process_instance i on i.process_def_uuid=n.process_def_uuid \
            where i.token=$1 and n.node_id=$2",
        )
        .bind(pending.task.process_token().get_id())
        .bind(target.get_id())
        .fetch_one(&mut *connection)
        .await?;
    }

    let uuid: Uuid = sqlx::query_scalar(
        "insert into fluxpro.queue_runner (task, run_after, node_visit_id) values ($1,$2,$3) returning uuid",
    )
    .bind(serde_json::to_value(&pending.task)?)
    .bind(pending.run_after.unwrap_or_else(Utc::now))
    .bind(pending.visit_id)
    .fetch_one(&mut *connection)
    .await?;
    if let FluxproQueueTask::ProcessEvent {
        process_token,
        node,
        ..
    } = &pending.task
    {
        for signal in &pending.cancel_signals {
            sqlx::query("insert into fluxpro.queue_cancel_task (process_token,node_id,event_id,task_uuid) values ($1,$2,$3,$4)")
                .bind(process_token.get_id()).bind(node.id().get_id()).bind(signal.get_id()).bind(uuid)
                .execute(&mut *connection).await?;
        }
    }
    Ok(())
}

async fn record_event(
    connection: &mut PgConnection,
    instance_uuid: Uuid,
    event: &ExecutionLogEvent,
) -> anyhow::Result<()> {
    sqlx::query("insert into fluxpro.process_instance_log \
        (process_instance_uuid,level,event_type,source,message,node_id,handler_id,queue_task_uuid,attempt,error_kind,error_message,details) \
        values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)")
        .bind(instance_uuid).bind(event.level.to_string()).bind(&event.event_type).bind(&event.source)
        .bind(&event.message).bind(&event.node_id).bind(&event.handler_id).bind(event.queue_task_uuid)
        .bind(event.attempt).bind(&event.error_kind).bind(&event.error_message).bind(&event.details)
        .execute(connection).await?;
    Ok(())
}

pub(super) async fn commit(
    db: &FluxproDbServiceImpl,
    task: &FluxproQueueTaskDefinition,
    snapshot: &TaskExecutionSnapshot,
    changes: TaskExecutionChanges,
) -> anyhow::Result<()> {
    let mut tx = db.db_pool.begin().await?;
    let result: anyhow::Result<()> = async {
    let instance = sqlx::query(
        "select uuid, execution_state, process_def_uuid, revision, node_visit_id, wait_completed \
        from fluxpro.process_instance where token=$1 for update",
    )
    .bind(task.task.process_token().get_id())
    .fetch_one(&mut *tx)
    .await?;
    lock_task(&mut tx, task).await?;
    anyhow::ensure!(
        instance.try_get::<String, _>("execution_state")? == "running",
        "instance is suspended"
    );
    anyhow::ensure!(
        instance.try_get::<i64, _>("revision")? == snapshot.revision,
        "instance changed during task execution"
    );
    let instance_uuid: Uuid = instance.try_get("uuid")?;
    let definition_uuid: Uuid = instance.try_get("process_def_uuid")?;
    if changes.close_wait {
        anyhow::ensure!(
            snapshot.visit_id.is_some() && !instance.try_get::<bool, _>("wait_completed")?,
            "wait already completed"
        );
        anyhow::ensure!(
            instance.try_get::<Option<Uuid>, _>("node_visit_id")? == snapshot.visit_id,
            "wait visit changed"
        );
        // Remove sibling timeouts, but retain the source row until the final lease check.
        sqlx::query("delete from fluxpro.queue_cancel_task where task_uuid in \
            (select uuid from fluxpro.queue_runner where node_visit_id=$1 and task ? 'ProcessEvent' and uuid<>$2)")
            .bind(snapshot.visit_id).bind(task.uuid).execute(&mut *tx).await?;
        sqlx::query("delete from fluxpro.queue_runner where node_visit_id=$1 and task ? 'ProcessEvent' and uuid<>$2")
            .bind(snapshot.visit_id).bind(task.uuid).execute(&mut *tx).await?;
        sqlx::query("update fluxpro.process_instance set wait_completed=true where uuid=$1")
            .bind(instance_uuid)
            .execute(&mut *tx)
            .await?;
    }
    if let Some(node_id) = &changes.enter_node {
        let node_uuid: Uuid = sqlx::query_scalar(
            "select uuid from fluxpro.process_def_node where process_def_uuid=$1 and node_id=$2",
        )
        .bind(definition_uuid)
        .bind(node_id.get_id())
        .fetch_one(&mut *tx)
        .await?;
        sqlx::query("update fluxpro.process_instance set current_node_ref=$2, node_visit_id=$3, wait_completed=false where uuid=$1")
            .bind(instance_uuid).bind(node_uuid).bind(task.uuid).execute(&mut *tx).await?;
    }
    if let Some(context) = changes
        .context
        .as_ref()
        .filter(|_| changes.context_patch.is_none())
    {
        merge_context(&mut tx, instance_uuid, context).await?;
    }
    if let Some(patch) = &changes.context_patch {
        apply_patch(&mut tx, instance_uuid, patch).await?;
    }

    if let Some(stage) = &changes.stage {
        assign_stage(&mut tx, instance_uuid, definition_uuid, stage).await?;
    }
    for pending in &changes.tasks {
        anyhow::ensure!(
            pending.task.process_token() == task.task.process_token(),
            "successor belongs to another instance"
        );
        enqueue(&mut tx, pending).await?;
    }
    for event in &changes.events {
        record_event(&mut tx, instance_uuid, event).await?;
    }
    sqlx::query("update fluxpro.process_instance set revision=revision+1 where uuid=$1")
        .bind(instance_uuid)
        .execute(&mut *tx)
        .await?;
    if let Some((kind, reason)) = &changes.incident {
        anyhow::ensure!(
            changes.tasks.is_empty() && changes.retry_at.is_none(),
            "incident cannot also schedule successors or retries"
        );
        sqlx::query("insert into fluxpro.process_incident(process_instance_uuid,task_uuid,task,kind,reason,attempt) values ($1,$2,$3,$4,$5,$6)")
            .bind(instance_uuid).bind(task.uuid).bind(serde_json::to_value(&task.task)?).bind(kind).bind(reason).bind(task.attempts)
            .execute(&mut *tx).await?;
        sqlx::query(
            "update fluxpro.process_instance set execution_state='suspended' where uuid=$1",
        )
        .bind(instance_uuid)
        .execute(&mut *tx)
        .await?;
    } else if changes.retry_at.is_none() {
        sqlx::query("update fluxpro.process_instance set recovery_task_uuid=null where uuid=$1 and recovery_task_uuid=$2")
            .bind(instance_uuid).bind(task.uuid).execute(&mut *tx).await?;
    }
    // clock_timestamp checks actual elapsed time, not transaction-start time.
    let affected = if let Some(retry_at) = changes
        .retry_at
        .or_else(|| changes.incident.as_ref().map(|_| Utc::now()))
    {
        sqlx::query("update fluxpro.queue_runner set run_after=$3,lock_key='',locked_at=null,locked_by=null \
            where uuid=$1 and lock_key=$2 and locked_by>clock_timestamp()")
            .bind(task.uuid).bind(&task.lock_key).bind(retry_at).execute(&mut *tx).await?.rows_affected()
    } else {
        sqlx::query("delete from fluxpro.queue_cancel_task where task_uuid=$1")
            .bind(task.uuid)
            .execute(&mut *tx)
            .await?;
        sqlx::query("delete from fluxpro.queue_runner where uuid=$1 and lock_key=$2 and locked_by>clock_timestamp()")
            .bind(task.uuid).bind(&task.lock_key).execute(&mut *tx).await?.rows_affected()
    };
    anyhow::ensure!(affected == 1, "task lease expired before transition commit");
        Ok(())
    }
    .await;
    if let Err(error) = result {
        // Drop only queues SQLx rollback. Release all row locks before a caller
        // retries or another worker tries to reclaim this task with SKIP LOCKED.
        tx.rollback().await?;
        return Err(error);
    }
    tx.commit().await?;
    Ok(())
}

pub(super) async fn start(
    db: &FluxproDbServiceImpl,
    definition: &ProcessDefinition,
    command: StartProcessInstance,
) -> anyhow::Result<IdField> {
    let definition_uuid = definition
        .uuid
        .ok_or_else(|| anyhow::anyhow!("definition has no database identity"))?;
    let start = definition
        .nodes
        .iter()
        .find(|node| node.is_start())
        .ok_or_else(|| anyhow::anyhow!("Start node missing"))?;
    let initial = definition
        .stages
        .iter()
        .find(|stage| stage.is_initial())
        .ok_or_else(|| anyhow::anyhow!("initial stage missing"))?;
    let token = IdField::generate();
    let mut tx = db.db_pool.begin().await?;
    // Publication may have changed since version lookup. Lock it through startup.
    sqlx::query("select uuid from fluxpro.process_definition where uuid=$1 and lower(status)='active' and effective_from<=clock_timestamp() and (deprecated_at is null or deprecated_at>clock_timestamp()) for share")
        .bind(definition_uuid).fetch_one(&mut *tx).await?;
    let instance_uuid: Uuid = sqlx::query_scalar("insert into fluxpro.process_instance (process_def_uuid,process_id,token) values ($1,$2,$3) returning uuid")
        .bind(definition_uuid).bind(command.process_id.get_id()).bind(token.get_id()).fetch_one(&mut *tx).await?;
    merge_context(&mut tx, instance_uuid, &command.context).await?;
    assign_stage(
        &mut tx,
        instance_uuid,
        definition_uuid,
        &SetStageWithReason {
            stage: initial.id().clone(),
            reason: None,
        },
    )
    .await?;
    enqueue(
        &mut tx,
        &ScheduledTask {
            task: FluxproQueueTask::ProcessNode {
                process_token: token.clone(),
                node: start.clone(),
            },
            run_after: None,
            visit_id: None,
            cancel_signals: vec![],
        },
    )
    .await?;
    record_event(
        &mut tx,
        instance_uuid,
        &ExecutionLogEvent::info(
            "instance.created",
            "process_service",
            "Process instance created",
        ),
    )
    .await?;
    record_event(
        &mut tx,
        instance_uuid,
        &ExecutionLogEvent::info(
            "stage.changed",
            "process_service",
            format!("Process moved to stage {}", initial.id()),
        ),
    )
    .await?;
    tx.commit().await?;
    Ok(token)
}