1use crate::execution_identity::{ExecutionIdentityV1, ExecutionResultReceiptV1};
2use anyhow::{Context, Result};
3use async_trait::async_trait;
4use serde::{Deserialize, Serialize};
5use std::collections::BTreeMap;
6use std::path::{Path, PathBuf};
7use tokio::io::AsyncWriteExt;
8use tokio::sync::Mutex;
9
10const MAX_DECISION_LEDGER_BYTES: usize = 16 * 1024 * 1024;
11const DECISION_LEDGER_SCHEMA_VERSION: u32 = 1;
12
13#[derive(Debug, Clone, PartialEq, Eq)]
14pub enum FlowDecisionClaimOutcome {
15 Claimed { attempt: u32 },
16 Completed,
17 Busy { lease_expires_at_ms: u64 },
18 Conflict,
19}
20
21#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
28#[serde(rename_all = "snake_case")]
29pub enum FlowDecisionClaimState {
30 Missing,
32 Pending {
34 lease_expires_at_ms: u64,
35 attempts: u32,
36 },
37 Completed { completed_at_ms: Option<u64> },
39 Unavailable,
41}
42
43impl FlowDecisionClaimState {
44 pub fn is_live_at(&self, now_ms: u64) -> bool {
46 matches!(
47 self,
48 Self::Pending {
49 lease_expires_at_ms,
50 ..
51 } if *lease_expires_at_ms > now_ms
52 )
53 }
54}
55
56#[async_trait]
57pub trait FlowDecisionLedger: Send + Sync {
58 async fn claim(
59 &self,
60 decision_id: &str,
61 request_hash: &str,
62 owner_id: &str,
63 now_ms: u64,
64 lease_ms: u64,
65 ) -> Result<FlowDecisionClaimOutcome>;
66
67 async fn renew(
70 &self,
71 decision_id: &str,
72 request_hash: &str,
73 owner_id: &str,
74 now_ms: u64,
75 lease_ms: u64,
76 ) -> Result<bool>;
77
78 async fn claim_with_identity(
82 &self,
83 decision_id: &str,
84 request_hash: &str,
85 identity: &ExecutionIdentityV1,
86 owner_id: &str,
87 now_ms: u64,
88 lease_ms: u64,
89 ) -> Result<FlowDecisionClaimOutcome> {
90 identity
91 .validate()
92 .map_err(|error| anyhow::anyhow!(error))?;
93 self.claim(decision_id, request_hash, owner_id, now_ms, lease_ms)
94 .await
95 }
96
97 async fn inspect(
103 &self,
104 _decision_id: &str,
105 _request_hash: &str,
106 ) -> Result<FlowDecisionClaimState> {
107 Ok(FlowDecisionClaimState::Unavailable)
108 }
109
110 async fn inspect_with_identity(
112 &self,
113 decision_id: &str,
114 request_hash: &str,
115 identity: &ExecutionIdentityV1,
116 ) -> Result<FlowDecisionClaimState> {
117 identity
118 .validate()
119 .map_err(|error| anyhow::anyhow!(error))?;
120 self.inspect(decision_id, request_hash).await
121 }
122
123 async fn renew_with_identity(
126 &self,
127 decision_id: &str,
128 request_hash: &str,
129 identity: &ExecutionIdentityV1,
130 owner_id: &str,
131 now_ms: u64,
132 lease_ms: u64,
133 ) -> Result<bool> {
134 identity
135 .validate()
136 .map_err(|error| anyhow::anyhow!(error))?;
137 self.renew(decision_id, request_hash, owner_id, now_ms, lease_ms)
138 .await
139 }
140
141 async fn complete(
142 &self,
143 decision_id: &str,
144 request_hash: &str,
145 owner_id: &str,
146 completed_at_ms: u64,
147 ) -> Result<()>;
148
149 async fn complete_with_identity(
154 &self,
155 decision_id: &str,
156 request_hash: &str,
157 identity: &ExecutionIdentityV1,
158 owner_id: &str,
159 completed_at_ms: u64,
160 ) -> Result<()> {
161 identity
162 .validate()
163 .map_err(|error| anyhow::anyhow!(error))?;
164 self.complete(decision_id, request_hash, owner_id, completed_at_ms)
165 .await
166 }
167
168 async fn complete_with_receipt(
171 &self,
172 decision_id: &str,
173 request_hash: &str,
174 identity: &ExecutionIdentityV1,
175 owner_id: &str,
176 receipt: &ExecutionResultReceiptV1,
177 completed_at_ms: u64,
178 ) -> Result<()> {
179 identity
180 .validate()
181 .map_err(|error| anyhow::anyhow!(error))?;
182 receipt.validate().map_err(|error| anyhow::anyhow!(error))?;
183 if &receipt.identity != identity {
184 anyhow::bail!("decision result receipt identity conflicts with its claim");
185 }
186 self.complete(decision_id, request_hash, owner_id, completed_at_ms)
187 .await
188 }
189
190 async fn release(&self, decision_id: &str, request_hash: &str, owner_id: &str) -> Result<()>;
191
192 async fn release_with_identity(
195 &self,
196 decision_id: &str,
197 request_hash: &str,
198 identity: &ExecutionIdentityV1,
199 owner_id: &str,
200 ) -> Result<()> {
201 identity
202 .validate()
203 .map_err(|error| anyhow::anyhow!(error))?;
204 self.release(decision_id, request_hash, owner_id).await
205 }
206
207 async fn completed_receipt(
209 &self,
210 _decision_id: &str,
211 ) -> Result<Option<ExecutionResultReceiptV1>> {
212 Ok(None)
213 }
214
215 async fn prune_completed(&self, _before_ms: u64) -> Result<usize> {
217 anyhow::bail!("Flow decision ledger does not support receipt pruning")
218 }
219}
220
221#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
222#[serde(rename_all = "snake_case")]
223enum ClaimStatus {
224 Pending,
225 Completed,
226}
227
228#[derive(Debug, Clone, Serialize, Deserialize)]
229struct ClaimRecord {
230 request_hash: String,
231 #[serde(default, skip_serializing_if = "Option::is_none")]
232 execution_identity: Option<ExecutionIdentityV1>,
233 status: ClaimStatus,
234 owner_id: String,
235 lease_expires_at_ms: u64,
236 attempts: u32,
237 #[serde(default, skip_serializing_if = "Option::is_none")]
238 completed_at_ms: Option<u64>,
239 #[serde(default, skip_serializing_if = "Option::is_none")]
240 result_receipt: Option<ExecutionResultReceiptV1>,
241}
242
243#[derive(Debug, Default, Serialize, Deserialize)]
244struct DecisionLedgerFile {
245 schema_version: u32,
246 records: BTreeMap<String, ClaimRecord>,
247}
248
249#[derive(Serialize)]
250struct DecisionLedgerFileRef<'a> {
251 schema_version: u32,
252 records: &'a BTreeMap<String, ClaimRecord>,
253}
254
255#[derive(Debug, Default)]
256pub struct MemoryFlowDecisionLedger {
257 records: Mutex<BTreeMap<String, ClaimRecord>>,
258}
259
260impl MemoryFlowDecisionLedger {
261 pub fn new() -> Self {
262 Self::default()
263 }
264}
265
266#[async_trait]
267impl FlowDecisionLedger for MemoryFlowDecisionLedger {
268 async fn claim(
269 &self,
270 decision_id: &str,
271 request_hash: &str,
272 owner_id: &str,
273 now_ms: u64,
274 lease_ms: u64,
275 ) -> Result<FlowDecisionClaimOutcome> {
276 let mut records = self.records.lock().await;
277 claim_record(
278 &mut records,
279 decision_id,
280 request_hash,
281 None,
282 owner_id,
283 now_ms,
284 lease_ms,
285 )
286 }
287
288 async fn renew(
289 &self,
290 decision_id: &str,
291 request_hash: &str,
292 owner_id: &str,
293 now_ms: u64,
294 lease_ms: u64,
295 ) -> Result<bool> {
296 let mut records = self.records.lock().await;
297 Ok(renew_record(
298 &mut records,
299 decision_id,
300 request_hash,
301 None,
302 owner_id,
303 now_ms,
304 lease_ms,
305 ))
306 }
307
308 async fn inspect(
309 &self,
310 decision_id: &str,
311 request_hash: &str,
312 ) -> Result<FlowDecisionClaimState> {
313 let records = self.records.lock().await;
314 inspect_record(&records, decision_id, request_hash, None)
315 }
316
317 async fn inspect_with_identity(
318 &self,
319 decision_id: &str,
320 request_hash: &str,
321 identity: &ExecutionIdentityV1,
322 ) -> Result<FlowDecisionClaimState> {
323 identity
324 .validate()
325 .map_err(|error| anyhow::anyhow!(error))?;
326 let records = self.records.lock().await;
327 inspect_record(&records, decision_id, request_hash, Some(identity))
328 }
329
330 async fn complete(
331 &self,
332 decision_id: &str,
333 request_hash: &str,
334 owner_id: &str,
335 completed_at_ms: u64,
336 ) -> Result<()> {
337 let mut records = self.records.lock().await;
338 complete_record(
339 &mut records,
340 decision_id,
341 request_hash,
342 None,
343 owner_id,
344 None,
345 completed_at_ms,
346 )
347 }
348
349 async fn complete_with_identity(
350 &self,
351 decision_id: &str,
352 request_hash: &str,
353 identity: &ExecutionIdentityV1,
354 owner_id: &str,
355 completed_at_ms: u64,
356 ) -> Result<()> {
357 identity
358 .validate()
359 .map_err(|error| anyhow::anyhow!(error))?;
360 let mut records = self.records.lock().await;
361 complete_record(
362 &mut records,
363 decision_id,
364 request_hash,
365 Some(identity),
366 owner_id,
367 None,
368 completed_at_ms,
369 )
370 }
371
372 async fn release(&self, decision_id: &str, request_hash: &str, owner_id: &str) -> Result<()> {
373 let mut records = self.records.lock().await;
374 release_record(&mut records, decision_id, request_hash, None, owner_id)
375 }
376
377 async fn claim_with_identity(
378 &self,
379 decision_id: &str,
380 request_hash: &str,
381 identity: &ExecutionIdentityV1,
382 owner_id: &str,
383 now_ms: u64,
384 lease_ms: u64,
385 ) -> Result<FlowDecisionClaimOutcome> {
386 identity
387 .validate()
388 .map_err(|error| anyhow::anyhow!(error))?;
389 let mut records = self.records.lock().await;
390 claim_record(
391 &mut records,
392 decision_id,
393 request_hash,
394 Some(identity),
395 owner_id,
396 now_ms,
397 lease_ms,
398 )
399 }
400
401 async fn renew_with_identity(
402 &self,
403 decision_id: &str,
404 request_hash: &str,
405 identity: &ExecutionIdentityV1,
406 owner_id: &str,
407 now_ms: u64,
408 lease_ms: u64,
409 ) -> Result<bool> {
410 identity
411 .validate()
412 .map_err(|error| anyhow::anyhow!(error))?;
413 let mut records = self.records.lock().await;
414 Ok(renew_record(
415 &mut records,
416 decision_id,
417 request_hash,
418 Some(identity),
419 owner_id,
420 now_ms,
421 lease_ms,
422 ))
423 }
424
425 async fn complete_with_receipt(
426 &self,
427 decision_id: &str,
428 request_hash: &str,
429 identity: &ExecutionIdentityV1,
430 owner_id: &str,
431 receipt: &ExecutionResultReceiptV1,
432 completed_at_ms: u64,
433 ) -> Result<()> {
434 validate_receipt(identity, receipt)?;
435 let mut records = self.records.lock().await;
436 complete_record(
437 &mut records,
438 decision_id,
439 request_hash,
440 Some(identity),
441 owner_id,
442 Some(receipt),
443 completed_at_ms,
444 )
445 }
446
447 async fn release_with_identity(
448 &self,
449 decision_id: &str,
450 request_hash: &str,
451 identity: &ExecutionIdentityV1,
452 owner_id: &str,
453 ) -> Result<()> {
454 identity
455 .validate()
456 .map_err(|error| anyhow::anyhow!(error))?;
457 let mut records = self.records.lock().await;
458 release_record(
459 &mut records,
460 decision_id,
461 request_hash,
462 Some(identity),
463 owner_id,
464 )
465 }
466
467 async fn completed_receipt(
468 &self,
469 decision_id: &str,
470 ) -> Result<Option<ExecutionResultReceiptV1>> {
471 let records = self.records.lock().await;
472 Ok(records
473 .get(decision_id)
474 .and_then(|record| record.result_receipt.clone()))
475 }
476
477 async fn prune_completed(&self, before_ms: u64) -> Result<usize> {
478 let mut records = self.records.lock().await;
479 Ok(prune_records(&mut records, before_ms))
480 }
481}
482
483#[derive(Debug)]
484pub struct FileFlowDecisionLedger {
485 root: PathBuf,
486 process_lock: Mutex<()>,
487}
488
489impl FileFlowDecisionLedger {
490 pub fn new(root: impl Into<PathBuf>) -> Self {
491 Self {
492 root: root.into(),
493 process_lock: Mutex::new(()),
494 }
495 }
496
497 fn data_path(&self) -> PathBuf {
498 self.root.join("flow-decisions.json")
499 }
500
501 async fn acquire_file_lock(&self) -> Result<std::fs::File> {
502 tokio::fs::create_dir_all(&self.root)
503 .await
504 .with_context(|| format!("create decision ledger `{}`", self.root.display()))?;
505 let path = self.root.join(".flow-decisions.lock");
506 tokio::task::spawn_blocking(move || {
507 use fs2::FileExt;
508 let file = std::fs::OpenOptions::new()
509 .create(true)
510 .truncate(false)
511 .read(true)
512 .write(true)
513 .open(&path)
514 .with_context(|| format!("open decision ledger lock `{}`", path.display()))?;
515 file.lock_exclusive()
516 .with_context(|| format!("lock decision ledger `{}`", path.display()))?;
517 Ok(file)
518 })
519 .await
520 .context("decision ledger lock task failed")?
521 }
522
523 async fn mutate<T>(
524 &self,
525 mutation: impl FnOnce(&mut BTreeMap<String, ClaimRecord>) -> Result<T>,
526 ) -> Result<T> {
527 let _process_guard = self.process_lock.lock().await;
528 let _file_guard = self.acquire_file_lock().await?;
529 let mut records = read_records(&self.data_path()).await?;
530 let result = mutation(&mut records)?;
531 write_records(&self.data_path(), &records).await?;
532 Ok(result)
533 }
534}
535
536#[async_trait]
537impl FlowDecisionLedger for FileFlowDecisionLedger {
538 async fn claim(
539 &self,
540 decision_id: &str,
541 request_hash: &str,
542 owner_id: &str,
543 now_ms: u64,
544 lease_ms: u64,
545 ) -> Result<FlowDecisionClaimOutcome> {
546 self.mutate(|records| {
547 claim_record(
548 records,
549 decision_id,
550 request_hash,
551 None,
552 owner_id,
553 now_ms,
554 lease_ms,
555 )
556 })
557 .await
558 }
559
560 async fn renew(
561 &self,
562 decision_id: &str,
563 request_hash: &str,
564 owner_id: &str,
565 now_ms: u64,
566 lease_ms: u64,
567 ) -> Result<bool> {
568 self.mutate(|records| {
569 Ok(renew_record(
570 records,
571 decision_id,
572 request_hash,
573 None,
574 owner_id,
575 now_ms,
576 lease_ms,
577 ))
578 })
579 .await
580 }
581
582 async fn inspect(
583 &self,
584 decision_id: &str,
585 request_hash: &str,
586 ) -> Result<FlowDecisionClaimState> {
587 let records = read_records(&self.data_path()).await?;
588 inspect_record(&records, decision_id, request_hash, None)
589 }
590
591 async fn inspect_with_identity(
592 &self,
593 decision_id: &str,
594 request_hash: &str,
595 identity: &ExecutionIdentityV1,
596 ) -> Result<FlowDecisionClaimState> {
597 identity
598 .validate()
599 .map_err(|error| anyhow::anyhow!(error))?;
600 let records = read_records(&self.data_path()).await?;
601 inspect_record(&records, decision_id, request_hash, Some(identity))
602 }
603
604 async fn complete(
605 &self,
606 decision_id: &str,
607 request_hash: &str,
608 owner_id: &str,
609 completed_at_ms: u64,
610 ) -> Result<()> {
611 self.mutate(|records| {
612 complete_record(
613 records,
614 decision_id,
615 request_hash,
616 None,
617 owner_id,
618 None,
619 completed_at_ms,
620 )
621 })
622 .await
623 }
624
625 async fn complete_with_identity(
626 &self,
627 decision_id: &str,
628 request_hash: &str,
629 identity: &ExecutionIdentityV1,
630 owner_id: &str,
631 completed_at_ms: u64,
632 ) -> Result<()> {
633 identity
634 .validate()
635 .map_err(|error| anyhow::anyhow!(error))?;
636 self.mutate(|records| {
637 complete_record(
638 records,
639 decision_id,
640 request_hash,
641 Some(identity),
642 owner_id,
643 None,
644 completed_at_ms,
645 )
646 })
647 .await
648 }
649
650 async fn release(&self, decision_id: &str, request_hash: &str, owner_id: &str) -> Result<()> {
651 self.mutate(|records| release_record(records, decision_id, request_hash, None, owner_id))
652 .await
653 }
654
655 async fn claim_with_identity(
656 &self,
657 decision_id: &str,
658 request_hash: &str,
659 identity: &ExecutionIdentityV1,
660 owner_id: &str,
661 now_ms: u64,
662 lease_ms: u64,
663 ) -> Result<FlowDecisionClaimOutcome> {
664 identity
665 .validate()
666 .map_err(|error| anyhow::anyhow!(error))?;
667 self.mutate(|records| {
668 claim_record(
669 records,
670 decision_id,
671 request_hash,
672 Some(identity),
673 owner_id,
674 now_ms,
675 lease_ms,
676 )
677 })
678 .await
679 }
680
681 async fn renew_with_identity(
682 &self,
683 decision_id: &str,
684 request_hash: &str,
685 identity: &ExecutionIdentityV1,
686 owner_id: &str,
687 now_ms: u64,
688 lease_ms: u64,
689 ) -> Result<bool> {
690 identity
691 .validate()
692 .map_err(|error| anyhow::anyhow!(error))?;
693 self.mutate(|records| {
694 Ok(renew_record(
695 records,
696 decision_id,
697 request_hash,
698 Some(identity),
699 owner_id,
700 now_ms,
701 lease_ms,
702 ))
703 })
704 .await
705 }
706
707 async fn complete_with_receipt(
708 &self,
709 decision_id: &str,
710 request_hash: &str,
711 identity: &ExecutionIdentityV1,
712 owner_id: &str,
713 receipt: &ExecutionResultReceiptV1,
714 completed_at_ms: u64,
715 ) -> Result<()> {
716 validate_receipt(identity, receipt)?;
717 self.mutate(|records| {
718 complete_record(
719 records,
720 decision_id,
721 request_hash,
722 Some(identity),
723 owner_id,
724 Some(receipt),
725 completed_at_ms,
726 )
727 })
728 .await
729 }
730
731 async fn release_with_identity(
732 &self,
733 decision_id: &str,
734 request_hash: &str,
735 identity: &ExecutionIdentityV1,
736 owner_id: &str,
737 ) -> Result<()> {
738 identity
739 .validate()
740 .map_err(|error| anyhow::anyhow!(error))?;
741 self.mutate(|records| {
742 release_record(records, decision_id, request_hash, Some(identity), owner_id)
743 })
744 .await
745 }
746
747 async fn completed_receipt(
748 &self,
749 decision_id: &str,
750 ) -> Result<Option<ExecutionResultReceiptV1>> {
751 let records = read_records(&self.data_path()).await?;
752 Ok(records
753 .get(decision_id)
754 .and_then(|record| record.result_receipt.clone()))
755 }
756
757 async fn prune_completed(&self, before_ms: u64) -> Result<usize> {
758 self.mutate(|records| Ok(prune_records(records, before_ms)))
759 .await
760 }
761}
762
763fn claim_record(
764 records: &mut BTreeMap<String, ClaimRecord>,
765 decision_id: &str,
766 request_hash: &str,
767 execution_identity: Option<&ExecutionIdentityV1>,
768 owner_id: &str,
769 now_ms: u64,
770 lease_ms: u64,
771) -> Result<FlowDecisionClaimOutcome> {
772 let lease_expires_at_ms = now_ms.saturating_add(lease_ms.max(1));
773 match records.get_mut(decision_id) {
774 None => {
775 records.insert(
776 decision_id.to_string(),
777 ClaimRecord {
778 request_hash: request_hash.to_string(),
779 execution_identity: execution_identity.cloned(),
780 status: ClaimStatus::Pending,
781 owner_id: owner_id.to_string(),
782 lease_expires_at_ms,
783 attempts: 1,
784 completed_at_ms: None,
785 result_receipt: None,
786 },
787 );
788 Ok(FlowDecisionClaimOutcome::Claimed { attempt: 1 })
789 }
790 Some(record) if record.request_hash != request_hash => {
791 Ok(FlowDecisionClaimOutcome::Conflict)
792 }
793 Some(record) if identity_conflicts(record, execution_identity) => {
794 Ok(FlowDecisionClaimOutcome::Conflict)
795 }
796 Some(record) if record.status == ClaimStatus::Completed => {
797 if record.execution_identity.is_none() {
798 record.execution_identity = execution_identity.cloned();
799 }
800 Ok(FlowDecisionClaimOutcome::Completed)
801 }
802 Some(record) if record.lease_expires_at_ms > now_ms && record.owner_id != owner_id => {
803 Ok(FlowDecisionClaimOutcome::Busy {
804 lease_expires_at_ms: record.lease_expires_at_ms,
805 })
806 }
807 Some(record) => {
808 if record.execution_identity.is_none() {
809 record.execution_identity = execution_identity.cloned();
810 }
811 record.owner_id = owner_id.to_string();
812 record.lease_expires_at_ms = lease_expires_at_ms;
813 record.attempts = record.attempts.saturating_add(1);
814 Ok(FlowDecisionClaimOutcome::Claimed {
815 attempt: record.attempts,
816 })
817 }
818 }
819}
820
821fn complete_record(
822 records: &mut BTreeMap<String, ClaimRecord>,
823 decision_id: &str,
824 request_hash: &str,
825 execution_identity: Option<&ExecutionIdentityV1>,
826 owner_id: &str,
827 result_receipt: Option<&ExecutionResultReceiptV1>,
828 completed_at_ms: u64,
829) -> Result<()> {
830 let record = records
831 .get_mut(decision_id)
832 .with_context(|| format!("decision claim `{decision_id}` does not exist"))?;
833 if record.request_hash != request_hash {
834 anyhow::bail!("decision `{decision_id}` request hash conflicts with its claim");
835 }
836 if identity_conflicts(record, execution_identity) {
837 anyhow::bail!("decision `{decision_id}` execution identity conflicts with its claim");
838 }
839 if record.status == ClaimStatus::Completed {
840 if let Some(receipt) = result_receipt {
841 if let Some(existing) = record.result_receipt.as_ref() {
842 if existing != receipt {
843 anyhow::bail!(
844 "decision `{decision_id}` result receipt conflicts with its completion"
845 );
846 }
847 } else {
848 record.result_receipt = Some(receipt.clone());
849 }
850 }
851 if record.execution_identity.is_none() {
852 record.execution_identity = execution_identity.cloned();
853 }
854 return Ok(());
855 }
856 if record.owner_id != owner_id {
857 anyhow::bail!("decision `{decision_id}` is owned by another dispatcher");
858 }
859 if record.lease_expires_at_ms == 0 || record.lease_expires_at_ms <= completed_at_ms {
860 anyhow::bail!("decision `{decision_id}` claim lease expired before completion");
861 }
862 if record.execution_identity.is_none() {
863 record.execution_identity = execution_identity.cloned();
864 }
865 record.status = ClaimStatus::Completed;
866 record.lease_expires_at_ms = 0;
867 record.completed_at_ms = Some(completed_at_ms);
868 record.result_receipt = result_receipt.cloned();
869 Ok(())
870}
871
872fn renew_record(
873 records: &mut BTreeMap<String, ClaimRecord>,
874 decision_id: &str,
875 request_hash: &str,
876 execution_identity: Option<&ExecutionIdentityV1>,
877 owner_id: &str,
878 now_ms: u64,
879 lease_ms: u64,
880) -> bool {
881 let Some(record) = records.get_mut(decision_id) else {
882 return false;
883 };
884 if record.request_hash != request_hash
885 || record.status != ClaimStatus::Pending
886 || record.owner_id != owner_id
887 || record.lease_expires_at_ms <= now_ms
888 {
889 return false;
890 }
891 if identity_conflicts(record, execution_identity) {
892 return false;
893 }
894 record.lease_expires_at_ms = now_ms.saturating_add(lease_ms.max(1));
895 true
896}
897
898fn release_record(
899 records: &mut BTreeMap<String, ClaimRecord>,
900 decision_id: &str,
901 request_hash: &str,
902 execution_identity: Option<&ExecutionIdentityV1>,
903 owner_id: &str,
904) -> Result<()> {
905 let Some(record) = records.get_mut(decision_id) else {
906 return Ok(());
907 };
908 if record.request_hash != request_hash
909 || record.status == ClaimStatus::Completed
910 || identity_conflicts(record, execution_identity)
911 {
912 return Ok(());
913 }
914 if record.owner_id == owner_id {
915 record.owner_id.clear();
916 record.lease_expires_at_ms = 0;
917 }
918 Ok(())
919}
920
921fn identity_conflicts(
922 record: &ClaimRecord,
923 execution_identity: Option<&ExecutionIdentityV1>,
924) -> bool {
925 matches!(
926 (record.execution_identity.as_ref(), execution_identity),
927 (Some(stored), Some(supplied)) if stored != supplied
928 )
929}
930
931fn inspect_record(
932 records: &BTreeMap<String, ClaimRecord>,
933 decision_id: &str,
934 request_hash: &str,
935 execution_identity: Option<&ExecutionIdentityV1>,
936) -> Result<FlowDecisionClaimState> {
937 let Some(record) = records.get(decision_id) else {
938 return Ok(FlowDecisionClaimState::Missing);
939 };
940 if record.request_hash != request_hash {
941 anyhow::bail!("decision `{decision_id}` request hash conflicts with its claim");
942 }
943 if identity_conflicts(record, execution_identity) {
944 anyhow::bail!("decision `{decision_id}` execution identity conflicts with its claim");
945 }
946 Ok(match record.status {
947 ClaimStatus::Pending => FlowDecisionClaimState::Pending {
948 lease_expires_at_ms: record.lease_expires_at_ms,
949 attempts: record.attempts,
950 },
951 ClaimStatus::Completed => FlowDecisionClaimState::Completed {
952 completed_at_ms: record.completed_at_ms,
953 },
954 })
955}
956
957fn validate_receipt(
958 identity: &ExecutionIdentityV1,
959 receipt: &ExecutionResultReceiptV1,
960) -> Result<()> {
961 identity
962 .validate()
963 .map_err(|error| anyhow::anyhow!(error))?;
964 receipt.validate().map_err(|error| anyhow::anyhow!(error))?;
965 if &receipt.identity != identity {
966 anyhow::bail!("decision result receipt identity conflicts with its claim");
967 }
968 Ok(())
969}
970
971fn prune_records(records: &mut BTreeMap<String, ClaimRecord>, before_ms: u64) -> usize {
972 let before = records.len();
973 records.retain(|_, record| {
974 record.status != ClaimStatus::Completed
975 || record.completed_at_ms.unwrap_or(u64::MAX) >= before_ms
976 });
977 before - records.len()
978}
979
980async fn read_records(path: &Path) -> Result<BTreeMap<String, ClaimRecord>> {
981 let bytes = match crate::bounded_io::read_file_bounded_async(path, MAX_DECISION_LEDGER_BYTES)
982 .await
983 {
984 Ok(bytes) => bytes,
985 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(BTreeMap::new()),
986 Err(error) if error.kind() == std::io::ErrorKind::InvalidData => {
987 anyhow::bail!("decision ledger exceeds {MAX_DECISION_LEDGER_BYTES} bytes")
988 }
989 Err(error) => return Err(error).context("read decision ledger"),
990 };
991 let ledger: DecisionLedgerFile =
992 serde_json::from_slice(&bytes).context("decode decision ledger")?;
993 if ledger.schema_version > DECISION_LEDGER_SCHEMA_VERSION {
994 anyhow::bail!(
995 "decision ledger schema {} is newer than supported schema {}",
996 ledger.schema_version,
997 DECISION_LEDGER_SCHEMA_VERSION
998 );
999 }
1000 for (decision_id, record) in &ledger.records {
1001 if let Some(identity) = record.execution_identity.as_ref() {
1002 identity
1003 .validate()
1004 .with_context(|| format!("validate identity for decision `{decision_id}`"))?;
1005 }
1006 if let Some(receipt) = record.result_receipt.as_ref() {
1007 receipt
1008 .validate()
1009 .with_context(|| format!("validate result receipt for decision `{decision_id}`"))?;
1010 if record.status != ClaimStatus::Completed {
1011 anyhow::bail!("decision `{decision_id}` has a result receipt before completion");
1012 }
1013 let Some(identity) = record.execution_identity.as_ref() else {
1014 anyhow::bail!(
1015 "decision `{decision_id}` result receipt has no execution identity binding"
1016 );
1017 };
1018 if &receipt.identity != identity {
1019 anyhow::bail!(
1020 "decision `{decision_id}` result receipt identity conflicts with its claim"
1021 );
1022 }
1023 }
1024 }
1025 Ok(ledger.records)
1026}
1027
1028async fn write_records(path: &Path, records: &BTreeMap<String, ClaimRecord>) -> Result<()> {
1029 let bytes = serde_json::to_vec(&DecisionLedgerFileRef {
1030 schema_version: DECISION_LEDGER_SCHEMA_VERSION,
1031 records,
1032 })
1033 .context("encode decision ledger")?;
1034 if bytes.len() > MAX_DECISION_LEDGER_BYTES {
1035 anyhow::bail!("decision ledger exceeds {MAX_DECISION_LEDGER_BYTES} bytes");
1036 }
1037 let temp = path.with_extension(format!("tmp-{}", uuid::Uuid::new_v4()));
1038 let result = async {
1039 let mut file = tokio::fs::File::create(&temp)
1040 .await
1041 .context("create decision ledger generation")?;
1042 file.write_all(&bytes)
1043 .await
1044 .context("write decision ledger")?;
1045 file.sync_all().await.context("sync decision ledger")?;
1046 drop(file);
1047 let temp_copy = temp.clone();
1048 let path = path.to_path_buf();
1049 tokio::task::spawn_blocking(move || {
1050 tempfile::TempPath::try_from_path(temp_copy)?
1051 .persist(path)
1052 .map_err(|error| error.error)
1053 })
1054 .await
1055 .context("publish decision ledger task failed")??;
1056 Ok::<_, anyhow::Error>(())
1057 }
1058 .await;
1059 if result.is_err() {
1060 let _ = tokio::fs::remove_file(temp).await;
1061 }
1062 result
1063}
1064
1065#[cfg(test)]
1066mod tests {
1067 use super::*;
1068 use crate::evaluation::digest_bytes;
1069
1070 fn identity(tag: &str) -> ExecutionIdentityV1 {
1071 ExecutionIdentityV1::derive("a3s.test.flow-decision", &serde_json::json!({ "tag": tag }))
1072 .unwrap()
1073 }
1074
1075 fn receipt(identity: ExecutionIdentityV1) -> ExecutionResultReceiptV1 {
1076 ExecutionResultReceiptV1::new(
1077 identity,
1078 digest_bytes("a3s.test.flow-evidence", b"event"),
1079 crate::execution_identity::ExecutionResultOutcomeV1::Succeeded,
1080 Some(digest_bytes("a3s.test.flow-result", b"result")),
1081 6,
1082 )
1083 .unwrap()
1084 }
1085
1086 #[tokio::test]
1087 async fn identity_fences_claim_renew_release_and_completion() {
1088 let ledger = MemoryFlowDecisionLedger::new();
1089 let first = identity("first");
1090 let other = identity("other");
1091 assert_eq!(
1092 ledger
1093 .inspect_with_identity("decision", "hash", &first)
1094 .await
1095 .unwrap(),
1096 FlowDecisionClaimState::Missing
1097 );
1098 assert_eq!(
1099 ledger
1100 .claim_with_identity("decision", "hash", &first, "owner", 100, 50)
1101 .await
1102 .unwrap(),
1103 FlowDecisionClaimOutcome::Claimed { attempt: 1 }
1104 );
1105 assert_eq!(
1106 ledger
1107 .inspect_with_identity("decision", "hash", &first)
1108 .await
1109 .unwrap(),
1110 FlowDecisionClaimState::Pending {
1111 lease_expires_at_ms: 150,
1112 attempts: 1,
1113 }
1114 );
1115 assert!(ledger
1116 .inspect_with_identity("decision", "hash", &other)
1117 .await
1118 .is_err());
1119 assert!(!ledger
1120 .renew_with_identity("decision", "hash", &other, "owner", 110, 50)
1121 .await
1122 .unwrap());
1123 ledger
1124 .release_with_identity("decision", "hash", &other, "owner")
1125 .await
1126 .unwrap();
1127 assert!(ledger
1128 .renew_with_identity("decision", "hash", &first, "owner", 110, 50)
1129 .await
1130 .unwrap());
1131 let mismatched_receipt = receipt(other);
1132 assert!(ledger
1133 .complete_with_receipt(
1134 "decision",
1135 "hash",
1136 &first,
1137 "owner",
1138 &mismatched_receipt,
1139 120,
1140 )
1141 .await
1142 .is_err());
1143 let result_receipt = receipt(first.clone());
1144 ledger
1145 .complete_with_receipt("decision", "hash", &first, "owner", &result_receipt, 121)
1146 .await
1147 .unwrap();
1148 assert_eq!(
1149 ledger.completed_receipt("decision").await.unwrap(),
1150 Some(result_receipt)
1151 );
1152 assert_eq!(
1153 ledger
1154 .inspect_with_identity("decision", "hash", &first)
1155 .await
1156 .unwrap(),
1157 FlowDecisionClaimState::Completed {
1158 completed_at_ms: Some(121),
1159 }
1160 );
1161 assert!(!ledger
1162 .renew_with_identity("decision", "hash", &first, "owner", 130, 50)
1163 .await
1164 .unwrap());
1165 }
1166
1167 #[tokio::test]
1168 async fn file_ledger_persists_identity_and_result_receipt() {
1169 let directory = tempfile::tempdir().unwrap();
1170 let ledger = FileFlowDecisionLedger::new(directory.path());
1171 let execution_identity = identity("persisted");
1172 let result_receipt = receipt(execution_identity.clone());
1173 ledger
1174 .claim_with_identity("decision", "hash", &execution_identity, "owner", 100, 50)
1175 .await
1176 .unwrap();
1177 ledger
1178 .complete_with_receipt(
1179 "decision",
1180 "hash",
1181 &execution_identity,
1182 "owner",
1183 &result_receipt,
1184 120,
1185 )
1186 .await
1187 .unwrap();
1188 let reopened = FileFlowDecisionLedger::new(directory.path());
1189 assert_eq!(
1190 reopened.completed_receipt("decision").await.unwrap(),
1191 Some(result_receipt)
1192 );
1193 assert_eq!(
1194 reopened
1195 .claim_with_identity("decision", "hash", &execution_identity, "other", 130, 50,)
1196 .await
1197 .unwrap(),
1198 FlowDecisionClaimOutcome::Completed
1199 );
1200 assert_eq!(
1201 reopened
1202 .inspect_with_identity("decision", "hash", &execution_identity)
1203 .await
1204 .unwrap(),
1205 FlowDecisionClaimState::Completed {
1206 completed_at_ms: Some(120),
1207 }
1208 );
1209 }
1210
1211 #[tokio::test]
1212 async fn expired_worker_cannot_complete_after_takeover() {
1213 let ledger = MemoryFlowDecisionLedger::new();
1214 let first = identity("expired-first");
1215 let second = first.clone();
1218 ledger
1219 .claim_with_identity("decision", "hash", &first, "first-owner", 100, 20)
1220 .await
1221 .unwrap();
1222 let first_receipt = receipt(first.clone());
1223 assert!(ledger
1224 .complete_with_receipt(
1225 "decision",
1226 "hash",
1227 &first,
1228 "first-owner",
1229 &first_receipt,
1230 121,
1231 )
1232 .await
1233 .is_err());
1234 assert!(ledger
1235 .complete_with_identity("decision", "hash", &first, "other", 120)
1236 .await
1237 .is_err());
1238 assert_eq!(
1239 ledger
1240 .claim_with_identity("decision", "hash", &second, "second-owner", 121, 20)
1241 .await
1242 .unwrap(),
1243 FlowDecisionClaimOutcome::Claimed { attempt: 2 }
1244 );
1245 assert!(ledger
1246 .complete_with_receipt(
1247 "decision",
1248 "hash",
1249 &first,
1250 "first-owner",
1251 &first_receipt,
1252 122,
1253 )
1254 .await
1255 .is_err());
1256 let second_receipt = receipt(second.clone());
1257 ledger
1258 .complete_with_identity("decision", "hash", &second, "second-owner", 123)
1259 .await
1260 .unwrap();
1261 ledger
1262 .complete_with_receipt(
1263 "decision",
1264 "hash",
1265 &second,
1266 "second-owner",
1267 &second_receipt,
1268 123,
1269 )
1270 .await
1271 .unwrap();
1272 assert_eq!(
1273 ledger.completed_receipt("decision").await.unwrap(),
1274 Some(second_receipt)
1275 );
1276 }
1277
1278 #[tokio::test]
1279 async fn legacy_file_record_without_identity_remains_claimable() {
1280 let directory = tempfile::tempdir().unwrap();
1281 let path = directory.path().join("flow-decisions.json");
1282 let legacy = serde_json::json!({
1283 "schema_version": 1,
1284 "records": {
1285 "decision": {
1286 "request_hash": "hash",
1287 "status": "pending",
1288 "owner_id": "old-owner",
1289 "lease_expires_at_ms": 0,
1290 "attempts": 1
1291 }
1292 }
1293 });
1294 tokio::fs::write(&path, serde_json::to_vec(&legacy).unwrap())
1295 .await
1296 .unwrap();
1297 let ledger = FileFlowDecisionLedger::new(directory.path());
1298 let execution_identity = identity("legacy-upgrade");
1299 assert_eq!(
1300 ledger
1301 .claim_with_identity(
1302 "decision",
1303 "hash",
1304 &execution_identity,
1305 "new-owner",
1306 100,
1307 50,
1308 )
1309 .await
1310 .unwrap(),
1311 FlowDecisionClaimOutcome::Claimed { attempt: 2 }
1312 );
1313 let result_receipt = receipt(execution_identity.clone());
1314 ledger
1315 .complete_with_receipt(
1316 "decision",
1317 "hash",
1318 &execution_identity,
1319 "new-owner",
1320 &result_receipt,
1321 120,
1322 )
1323 .await
1324 .unwrap();
1325 assert_eq!(
1326 ledger.completed_receipt("decision").await.unwrap(),
1327 Some(result_receipt)
1328 );
1329 }
1330}