1use std::collections::BTreeSet;
4use std::error::Error;
5use std::fmt;
6use std::future::Future;
7use std::num::NonZeroU64;
8use std::pin::Pin;
9use std::sync::atomic::{AtomicU64, Ordering};
10use std::time::SystemTime;
11
12use oxide_batch_core::{
13 BatchStatus, DefinitionIdentity, DefinitionRevision, DefinitionUpgrade, DomainError,
14 ExecutionMetadata, ExecutionTimestamps, ExecutionVersion, ExitStatus, FailureCategory,
15 FailureId, FailureSummary, IdentifierKind, JobExecution, JobExecutionId, JobInstance,
16 JobInstanceId, JobInstanceKey, JobName, LifecycleError, LifecycleTransition, NodeId,
17 RecoveryDecisionId, StartLimit, StepExecution, StepExecutionId, StepName, StepPartitionId,
18};
19
20use crate::{
21 ActorRef, FlowDecision, FlowDecisionRequest, FlowStepState, FlowTransitionKind, OperationId,
22 OperatorAction, OperatorRecord, OperatorRecordDraft, OwnerToken, PartitionAggregate,
23 PartitionAggregationError, PartitionPlanEntry, PurgeCounts, PurgePlan, PurgePlanRequest,
24 PurgeSurvey, ReasonCode, RetentionAction, RetentionHold, RetentionRecord, RetentionRecordDraft,
25 StepPartition,
26};
27
28const MAX_RECOVERY_REASON_BYTES: usize = 64;
29const MAX_OPERATOR_REFERENCE_BYTES: usize = 128;
30
31pub type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
37
38pub trait Clock: Send + Sync {
43 fn now(&self) -> SystemTime;
45}
46
47#[derive(Clone, Copy, Debug, Default)]
49pub struct SystemClock;
50
51impl Clock for SystemClock {
52 fn now(&self) -> SystemTime {
53 SystemTime::now()
54 }
55}
56
57pub trait IdGenerator: Send + Sync {
62 fn next_job_instance_id(&self) -> Result<JobInstanceId, IdGenerationError>;
69
70 fn next_job_execution_id(&self) -> Result<JobExecutionId, IdGenerationError>;
77
78 fn next_step_execution_id(&self) -> Result<StepExecutionId, IdGenerationError>;
85
86 fn next_failure_id(&self) -> Result<FailureId, IdGenerationError>;
93}
94
95#[derive(Debug)]
101pub struct SequentialIdGenerator {
102 next: AtomicU64,
103}
104
105impl SequentialIdGenerator {
106 #[must_use]
108 pub const fn new(first: NonZeroU64) -> Self {
109 Self {
110 next: AtomicU64::new(first.get()),
111 }
112 }
113
114 fn next_raw(&self, kind: IdentifierKind) -> Result<u64, IdGenerationError> {
115 self.next
116 .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
117 if current == 0 {
118 None
119 } else {
120 Some(current.checked_add(1).unwrap_or(0))
121 }
122 })
123 .map_err(|_| IdGenerationError::Exhausted { kind })
124 }
125}
126
127impl IdGenerator for SequentialIdGenerator {
128 fn next_job_instance_id(&self) -> Result<JobInstanceId, IdGenerationError> {
129 JobInstanceId::new(self.next_raw(IdentifierKind::JobInstance)?)
130 .map_err(IdGenerationError::Invalid)
131 }
132
133 fn next_job_execution_id(&self) -> Result<JobExecutionId, IdGenerationError> {
134 JobExecutionId::new(self.next_raw(IdentifierKind::JobExecution)?)
135 .map_err(IdGenerationError::Invalid)
136 }
137
138 fn next_step_execution_id(&self) -> Result<StepExecutionId, IdGenerationError> {
139 StepExecutionId::new(self.next_raw(IdentifierKind::StepExecution)?)
140 .map_err(IdGenerationError::Invalid)
141 }
142
143 fn next_failure_id(&self) -> Result<FailureId, IdGenerationError> {
144 FailureId::new(self.next_raw(IdentifierKind::Failure)?).map_err(IdGenerationError::Invalid)
145 }
146}
147
148#[derive(Clone, Debug, Eq, PartialEq)]
150#[non_exhaustive]
151pub enum IdGenerationError {
152 Exhausted {
154 kind: IdentifierKind,
156 },
157 Invalid(DomainError),
159}
160
161impl fmt::Display for IdGenerationError {
162 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
163 match self {
164 Self::Exhausted { kind } => write!(formatter, "{kind} identifier source is exhausted"),
165 Self::Invalid(error) => write!(formatter, "generated identifier was invalid: {error}"),
166 }
167 }
168}
169
170impl Error for IdGenerationError {
171 fn source(&self) -> Option<&(dyn Error + 'static)> {
172 match self {
173 Self::Invalid(error) => Some(error),
174 Self::Exhausted { .. } => None,
175 }
176 }
177}
178
179#[derive(Clone, Debug, Eq, PartialEq)]
181#[non_exhaustive]
182pub enum JobInstanceSelection {
183 Created(JobInstance),
185 Existing(JobInstance),
187}
188
189impl JobInstanceSelection {
190 #[must_use]
192 pub const fn instance(&self) -> &JobInstance {
193 match self {
194 Self::Created(instance) | Self::Existing(instance) => instance,
195 }
196 }
197
198 #[must_use]
200 pub const fn was_created(&self) -> bool {
201 matches!(self, Self::Created(_))
202 }
203}
204
205#[derive(Clone, Copy, Debug, Eq, PartialEq)]
207#[non_exhaustive]
208pub enum RecoveryDisposition {
209 MarkFailed,
211 Abandon,
213}
214
215impl RecoveryDisposition {
216 #[must_use]
218 pub const fn resulting_status(self) -> BatchStatus {
219 match self {
220 Self::MarkFailed => BatchStatus::Failed,
221 Self::Abandon => BatchStatus::Abandoned,
222 }
223 }
224}
225
226#[derive(Clone, Eq, PartialEq)]
228pub struct RecoveryRequest {
229 expected_version: ExecutionVersion,
230 disposition: RecoveryDisposition,
231 reason_code: String,
232 operator_reference: String,
233 evidence_digest: [u8; 32],
234 failure: Option<FailureSummary>,
235}
236
237impl RecoveryRequest {
238 pub fn mark_failed(
248 expected_version: ExecutionVersion,
249 reason_code: impl Into<String>,
250 operator_reference: impl Into<String>,
251 evidence_digest: [u8; 32],
252 failure_category: FailureCategory,
253 failure_id: FailureId,
254 ) -> Result<Self, RecoveryRequestError> {
255 Self::new(
256 expected_version,
257 RecoveryDisposition::MarkFailed,
258 reason_code,
259 operator_reference,
260 evidence_digest,
261 Some(FailureSummary::new(failure_category, failure_id)),
262 )
263 }
264
265 pub fn abandon(
272 expected_version: ExecutionVersion,
273 reason_code: impl Into<String>,
274 operator_reference: impl Into<String>,
275 evidence_digest: [u8; 32],
276 ) -> Result<Self, RecoveryRequestError> {
277 Self::new(
278 expected_version,
279 RecoveryDisposition::Abandon,
280 reason_code,
281 operator_reference,
282 evidence_digest,
283 None,
284 )
285 }
286
287 fn new(
288 expected_version: ExecutionVersion,
289 disposition: RecoveryDisposition,
290 reason_code: impl Into<String>,
291 operator_reference: impl Into<String>,
292 evidence_digest: [u8; 32],
293 failure: Option<FailureSummary>,
294 ) -> Result<Self, RecoveryRequestError> {
295 let reason_code = reason_code.into();
296 validate_recovery_text(
297 &reason_code,
298 RecoveryField::ReasonCode,
299 MAX_RECOVERY_REASON_BYTES,
300 )?;
301 let operator_reference = operator_reference.into();
302 validate_recovery_text(
303 &operator_reference,
304 RecoveryField::OperatorReference,
305 MAX_OPERATOR_REFERENCE_BYTES,
306 )?;
307 Ok(Self {
308 expected_version,
309 disposition,
310 reason_code,
311 operator_reference,
312 evidence_digest,
313 failure,
314 })
315 }
316
317 #[must_use]
319 pub const fn expected_version(&self) -> ExecutionVersion {
320 self.expected_version
321 }
322
323 #[must_use]
325 pub const fn disposition(&self) -> RecoveryDisposition {
326 self.disposition
327 }
328
329 #[must_use]
331 pub fn reason_code(&self) -> &str {
332 &self.reason_code
333 }
334
335 #[must_use]
337 pub fn operator_reference(&self) -> &str {
338 &self.operator_reference
339 }
340
341 #[must_use]
343 pub const fn evidence_digest(&self) -> &[u8; 32] {
344 &self.evidence_digest
345 }
346
347 #[must_use]
349 pub const fn failure(&self) -> Option<FailureSummary> {
350 self.failure
351 }
352}
353
354impl fmt::Debug for RecoveryRequest {
355 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
356 formatter
357 .debug_struct("RecoveryRequest")
358 .field("expected_version", &self.expected_version)
359 .field("disposition", &self.disposition)
360 .field("reason_code", &self.reason_code)
361 .field("operator_reference", &self.operator_reference)
362 .field("evidence_digest", &"<redacted>")
363 .field("failure", &self.failure)
364 .finish()
365 }
366}
367
368#[derive(Clone, Eq, PartialEq)]
370pub struct RecoveryDecision {
371 id: RecoveryDecisionId,
372 job_execution_id: JobExecutionId,
373 execution_version: ExecutionVersion,
374 prior_status: BatchStatus,
375 resulting_status: BatchStatus,
376 reason_code: String,
377 operator_reference: String,
378 evidence_digest: [u8; 32],
379 decided_at: SystemTime,
380}
381
382impl RecoveryDecision {
383 #[allow(clippy::too_many_arguments)]
385 #[doc(hidden)]
386 #[must_use]
387 pub fn new(
388 id: RecoveryDecisionId,
389 job_execution_id: JobExecutionId,
390 execution_version: ExecutionVersion,
391 prior_status: BatchStatus,
392 resulting_status: BatchStatus,
393 reason_code: String,
394 operator_reference: String,
395 evidence_digest: [u8; 32],
396 decided_at: SystemTime,
397 ) -> Self {
398 Self {
399 id,
400 job_execution_id,
401 execution_version,
402 prior_status,
403 resulting_status,
404 reason_code,
405 operator_reference,
406 evidence_digest,
407 decided_at,
408 }
409 }
410
411 #[must_use]
413 pub const fn id(&self) -> RecoveryDecisionId {
414 self.id
415 }
416
417 #[must_use]
419 pub const fn job_execution_id(&self) -> JobExecutionId {
420 self.job_execution_id
421 }
422
423 #[must_use]
425 pub const fn execution_version(&self) -> ExecutionVersion {
426 self.execution_version
427 }
428
429 #[must_use]
431 pub const fn prior_status(&self) -> BatchStatus {
432 self.prior_status
433 }
434
435 #[must_use]
437 pub const fn resulting_status(&self) -> BatchStatus {
438 self.resulting_status
439 }
440
441 #[must_use]
443 pub fn reason_code(&self) -> &str {
444 &self.reason_code
445 }
446
447 #[must_use]
449 pub fn operator_reference(&self) -> &str {
450 &self.operator_reference
451 }
452
453 #[must_use]
455 pub const fn evidence_digest(&self) -> &[u8; 32] {
456 &self.evidence_digest
457 }
458
459 #[must_use]
461 pub const fn decided_at(&self) -> SystemTime {
462 self.decided_at
463 }
464}
465
466impl fmt::Debug for RecoveryDecision {
467 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
468 formatter
469 .debug_struct("RecoveryDecision")
470 .field("id", &self.id)
471 .field("job_execution_id", &self.job_execution_id)
472 .field("execution_version", &self.execution_version)
473 .field("prior_status", &self.prior_status)
474 .field("resulting_status", &self.resulting_status)
475 .field("reason_code", &self.reason_code)
476 .field("operator_reference", &self.operator_reference)
477 .field("evidence_digest", &"<redacted>")
478 .field("decided_at", &self.decided_at)
479 .finish()
480 }
481}
482
483#[derive(Clone, Debug, Eq, PartialEq)]
485pub struct RecoveryResult {
486 execution: JobExecution,
487 decision: RecoveryDecision,
488}
489
490impl RecoveryResult {
491 #[doc(hidden)]
493 #[must_use]
494 pub const fn new(execution: JobExecution, decision: RecoveryDecision) -> Self {
495 Self {
496 execution,
497 decision,
498 }
499 }
500
501 #[must_use]
503 pub const fn execution(&self) -> &JobExecution {
504 &self.execution
505 }
506
507 #[must_use]
509 pub const fn decision(&self) -> &RecoveryDecision {
510 &self.decision
511 }
512}
513
514#[derive(Clone, Copy, Debug, Eq, PartialEq)]
516#[non_exhaustive]
517pub enum RecoveryField {
518 ReasonCode,
520 OperatorReference,
522}
523
524#[derive(Clone, Debug, Eq, PartialEq)]
526#[non_exhaustive]
527pub enum RecoveryRequestError {
528 Empty {
530 field: RecoveryField,
532 },
533 TooLong {
535 field: RecoveryField,
537 max_bytes: usize,
539 },
540 SurroundingWhitespace {
542 field: RecoveryField,
544 },
545 ControlCharacter {
547 field: RecoveryField,
549 },
550}
551
552impl fmt::Display for RecoveryRequestError {
553 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
554 match self {
555 Self::Empty { field } => write!(formatter, "{field:?} must not be empty"),
556 Self::TooLong { field, max_bytes } => {
557 write!(formatter, "{field:?} exceeds {max_bytes} bytes")
558 }
559 Self::SurroundingWhitespace { field } => {
560 write!(formatter, "{field:?} has surrounding whitespace")
561 }
562 Self::ControlCharacter { field } => {
563 write!(formatter, "{field:?} contains a control character")
564 }
565 }
566 }
567}
568
569impl Error for RecoveryRequestError {}
570
571fn validate_recovery_text(
572 value: &str,
573 field: RecoveryField,
574 max_bytes: usize,
575) -> Result<(), RecoveryRequestError> {
576 if value.is_empty() {
577 return Err(RecoveryRequestError::Empty { field });
578 }
579 if value.len() > max_bytes {
580 return Err(RecoveryRequestError::TooLong { field, max_bytes });
581 }
582 if value.trim() != value {
583 return Err(RecoveryRequestError::SurroundingWhitespace { field });
584 }
585 if value.chars().any(char::is_control) {
586 return Err(RecoveryRequestError::ControlCharacter { field });
587 }
588 Ok(())
589}
590
591#[doc(hidden)]
598pub fn recovered_execution(
599 prior: &JobExecution,
600 request: &RecoveryRequest,
601 decided_at: SystemTime,
602) -> Result<JobExecution, RepositoryError> {
603 if prior.version() != request.expected_version() {
604 return Err(RepositoryError::Lifecycle(LifecycleError::StaleVersion {
605 expected: request.expected_version(),
606 actual: prior.version(),
607 }));
608 }
609 let prior_status = prior.metadata().status();
610 if !matches!(
611 prior_status,
612 BatchStatus::Starting | BatchStatus::Started | BatchStatus::Stopping | BatchStatus::Unknown
613 ) {
614 return Err(RepositoryError::RecoveryNotAllowed {
615 id: prior.id(),
616 status: prior_status,
617 });
618 }
619 let current_time = prior.metadata().timestamps();
620 let timestamps = ExecutionTimestamps::new(
621 current_time.created_at(),
622 current_time.started_at(),
623 Some(decided_at),
624 )?;
625 let resulting_status = request.disposition().resulting_status();
626 let metadata = ExecutionMetadata::new(
627 resulting_status,
628 prior.metadata().exit_status().clone(),
629 timestamps,
630 prior.metadata().counts(),
631 request.failure(),
632 )?;
633 Ok(JobExecution::from_snapshot(
634 prior.id(),
635 prior.job_instance_id(),
636 metadata,
637 prior.version().next()?,
638 ))
639}
640
641pub trait JobRepository: Send + Sync {
646 fn connection_capacity(&self) -> u32 {
652 1
653 }
654
655 fn descriptor(&self) -> RepositoryDescriptor {
662 RepositoryDescriptor::new(0, [])
663 }
664
665 fn begin<'a>(
669 &'a self,
670 ) -> BoxFuture<'a, Result<Box<dyn RepositoryUnitOfWork + 'a>, RepositoryError>>;
671}
672
673#[derive(Clone, Debug, Eq, PartialEq)]
675pub struct ExecutionControl {
676 execution: JobExecution,
677 owner_matches: bool,
678 stop_requested: bool,
679}
680
681impl ExecutionControl {
682 #[doc(hidden)]
687 #[must_use]
688 pub const fn new(execution: JobExecution, owner_matches: bool, stop_requested: bool) -> Self {
689 Self {
690 execution,
691 owner_matches,
692 stop_requested,
693 }
694 }
695
696 #[must_use]
698 pub const fn execution(&self) -> &JobExecution {
699 &self.execution
700 }
701
702 #[must_use]
704 pub const fn owner_matches(&self) -> bool {
705 self.owner_matches
706 }
707
708 #[must_use]
710 pub const fn stop_requested(&self) -> bool {
711 self.stop_requested
712 }
713}
714
715pub trait RepositoryUnitOfWork: Send {
721 fn register_definition_upgrade<'a>(
723 &'a mut self,
724 job_name: &'a JobName,
725 upgrade: &'a DefinitionUpgrade,
726 ) -> BoxFuture<'a, Result<(), RepositoryError>>;
727
728 fn select_or_create_job_instance<'a>(
730 &'a mut self,
731 key: &'a JobInstanceKey,
732 ) -> BoxFuture<'a, Result<JobInstanceSelection, RepositoryError>>;
733
734 fn create_job_execution(
740 &mut self,
741 job_instance_id: JobInstanceId,
742 ) -> BoxFuture<'_, Result<JobExecution, RepositoryError>>;
743
744 fn create_job_execution_with_definition<'a>(
749 &'a mut self,
750 job_instance_id: JobInstanceId,
751 definition: &'a DefinitionIdentity,
752 ) -> BoxFuture<'a, Result<JobExecution, RepositoryError>>;
753
754 fn create_step_execution<'a>(
756 &'a mut self,
757 job_execution_id: JobExecutionId,
758 step_name: &'a StepName,
759 ) -> BoxFuture<'a, Result<StepExecution, RepositoryError>>;
760
761 fn create_flow_step_execution<'a>(
768 &'a mut self,
769 _job_execution_id: JobExecutionId,
770 _step_name: &'a StepName,
771 _node_id: &'a NodeId,
772 _start_limit: StartLimit,
773 ) -> BoxFuture<'a, Result<StepExecution, RepositoryError>> {
774 Box::pin(async { Err(RepositoryError::FlowStateCorrupt) })
775 }
776
777 fn transition_job_execution(
779 &mut self,
780 id: JobExecutionId,
781 expected_version: ExecutionVersion,
782 transition: LifecycleTransition,
783 ) -> BoxFuture<'_, Result<JobExecution, RepositoryError>>;
784
785 fn enrich_job_exit_status<'a>(
787 &'a mut self,
788 id: JobExecutionId,
789 expected_version: ExecutionVersion,
790 exit_status: &'a ExitStatus,
791 ) -> BoxFuture<'a, Result<JobExecution, RepositoryError>>;
792
793 fn transition_step_execution(
795 &mut self,
796 id: StepExecutionId,
797 expected_version: ExecutionVersion,
798 transition: LifecycleTransition,
799 ) -> BoxFuture<'_, Result<StepExecution, RepositoryError>>;
800
801 fn enrich_step_exit_status<'a>(
803 &'a mut self,
804 id: StepExecutionId,
805 expected_version: ExecutionVersion,
806 exit_status: &'a ExitStatus,
807 ) -> BoxFuture<'a, Result<StepExecution, RepositoryError>>;
808
809 fn find_job_instance<'a>(
811 &'a mut self,
812 key: &'a JobInstanceKey,
813 ) -> BoxFuture<'a, Result<Option<JobInstance>, RepositoryError>>;
814
815 fn get_job_instance(
817 &mut self,
818 id: JobInstanceId,
819 ) -> BoxFuture<'_, Result<Option<JobInstance>, RepositoryError>>;
820
821 fn get_job_execution(
823 &mut self,
824 id: JobExecutionId,
825 ) -> BoxFuture<'_, Result<Option<JobExecution>, RepositoryError>>;
826
827 fn job_executions(
829 &mut self,
830 job_instance_id: JobInstanceId,
831 ) -> BoxFuture<'_, Result<Vec<JobExecution>, RepositoryError>>;
832
833 fn get_step_execution(
835 &mut self,
836 id: StepExecutionId,
837 ) -> BoxFuture<'_, Result<Option<StepExecution>, RepositoryError>>;
838
839 fn step_executions(
841 &mut self,
842 job_execution_id: JobExecutionId,
843 ) -> BoxFuture<'_, Result<Vec<StepExecution>, RepositoryError>>;
844
845 fn latest_flow_step<'a>(
847 &'a mut self,
848 _job_instance_id: JobInstanceId,
849 _node_id: &'a NodeId,
850 ) -> BoxFuture<'a, Result<Option<FlowStepState>, RepositoryError>> {
851 Box::pin(async { Err(RepositoryError::FlowStateCorrupt) })
852 }
853
854 fn append_flow_decision<'a>(
856 &'a mut self,
857 _request: &'a FlowDecisionRequest,
858 ) -> BoxFuture<'a, Result<FlowDecision, RepositoryError>> {
859 Box::pin(async { Err(RepositoryError::FlowStateCorrupt) })
860 }
861
862 fn find_reusable_flow_decision<'a>(
864 &'a mut self,
865 _job_instance_id: JobInstanceId,
866 _node_id: &'a NodeId,
867 _plan_fingerprint: &'a [u8; 32],
868 _input_digest: &'a [u8; 32],
869 _kind: FlowTransitionKind,
870 ) -> BoxFuture<'a, Result<Option<FlowDecision>, RepositoryError>> {
871 Box::pin(async { Err(RepositoryError::FlowStateCorrupt) })
872 }
873
874 fn flow_decisions(
876 &mut self,
877 _job_execution_id: JobExecutionId,
878 ) -> BoxFuture<'_, Result<Vec<FlowDecision>, RepositoryError>> {
879 Box::pin(async { Err(RepositoryError::FlowStateCorrupt) })
880 }
881
882 fn create_step_partition_plan<'a>(
888 &'a mut self,
889 _step_execution_id: StepExecutionId,
890 _entries: &'a [PartitionPlanEntry],
891 ) -> BoxFuture<'a, Result<Vec<StepPartition>, RepositoryError>> {
892 Box::pin(async {
893 Err(RepositoryError::UnsupportedCapability {
894 capability: RepositoryCapability::StepPartitions,
895 })
896 })
897 }
898
899 fn step_partition_plan(
901 &mut self,
902 _step_execution_id: StepExecutionId,
903 ) -> BoxFuture<'_, Result<Vec<StepPartition>, RepositoryError>> {
904 Box::pin(async {
905 Err(RepositoryError::UnsupportedCapability {
906 capability: RepositoryCapability::StepPartitions,
907 })
908 })
909 }
910
911 fn restart_step_partition_plan(
919 &mut self,
920 _source_step_execution_id: StepExecutionId,
921 _target_step_execution_id: StepExecutionId,
922 ) -> BoxFuture<'_, Result<Vec<StepPartition>, RepositoryError>> {
923 Box::pin(async {
924 Err(RepositoryError::UnsupportedCapability {
925 capability: RepositoryCapability::StepPartitions,
926 })
927 })
928 }
929
930 fn assign_step_partition(
932 &mut self,
933 _id: StepPartitionId,
934 _expected_version: ExecutionVersion,
935 _worker_step_execution_id: StepExecutionId,
936 ) -> BoxFuture<'_, Result<StepPartition, RepositoryError>> {
937 Box::pin(async {
938 Err(RepositoryError::UnsupportedCapability {
939 capability: RepositoryCapability::StepPartitions,
940 })
941 })
942 }
943
944 fn complete_step_partition(
951 &mut self,
952 _id: StepPartitionId,
953 _expected_version: ExecutionVersion,
954 _worker_step_execution_id: StepExecutionId,
955 ) -> BoxFuture<'_, Result<StepPartition, RepositoryError>> {
956 Box::pin(async {
957 Err(RepositoryError::UnsupportedCapability {
958 capability: RepositoryCapability::StepPartitions,
959 })
960 })
961 }
962
963 fn aggregate_step_partitions(
970 &mut self,
971 _step_execution_id: StepExecutionId,
972 _expected_version: ExecutionVersion,
973 _transitioned_at: SystemTime,
974 ) -> BoxFuture<'_, Result<StepExecution, RepositoryError>> {
975 Box::pin(async {
976 Err(RepositoryError::UnsupportedCapability {
977 capability: RepositoryCapability::StepPartitions,
978 })
979 })
980 }
981
982 fn recover_job_execution<'a>(
984 &'a mut self,
985 id: JobExecutionId,
986 request: &'a RecoveryRequest,
987 ) -> BoxFuture<'a, Result<RecoveryResult, RepositoryError>>;
988
989 fn recovery_decision(
991 &mut self,
992 id: JobExecutionId,
993 ) -> BoxFuture<'_, Result<Option<RecoveryDecision>, RepositoryError>>;
994
995 fn find_operator_request<'a>(
1000 &'a mut self,
1001 _action: OperatorAction,
1002 _operation_id: &'a OperationId,
1003 ) -> BoxFuture<'a, Result<Option<OperatorRecord>, RepositoryError>> {
1004 Box::pin(async {
1005 Err(RepositoryError::UnsupportedCapability {
1006 capability: RepositoryCapability::OperatorRequests,
1007 })
1008 })
1009 }
1010
1011 fn append_operator_request<'a>(
1013 &'a mut self,
1014 _draft: &'a OperatorRecordDraft,
1015 ) -> BoxFuture<'a, Result<OperatorRecord, RepositoryError>> {
1016 Box::pin(async {
1017 Err(RepositoryError::UnsupportedCapability {
1018 capability: RepositoryCapability::OperatorRequests,
1019 })
1020 })
1021 }
1022
1023 fn request_execution_stop<'a>(
1029 &'a mut self,
1030 _id: JobExecutionId,
1031 _expected_version: ExecutionVersion,
1032 _actor: &'a ActorRef,
1033 _requested_at: SystemTime,
1034 ) -> BoxFuture<'a, Result<JobExecution, RepositoryError>> {
1035 Box::pin(async {
1036 Err(RepositoryError::UnsupportedCapability {
1037 capability: RepositoryCapability::StopRequests,
1038 })
1039 })
1040 }
1041
1042 fn claim_execution_owner<'a>(
1047 &'a mut self,
1048 _id: JobExecutionId,
1049 _expected_version: ExecutionVersion,
1050 _owner: &'a OwnerToken,
1051 _claimed_at: SystemTime,
1052 ) -> BoxFuture<'a, Result<JobExecution, RepositoryError>> {
1053 Box::pin(async {
1054 Err(RepositoryError::UnsupportedCapability {
1055 capability: RepositoryCapability::ExecutionOwnership,
1056 })
1057 })
1058 }
1059
1060 fn observe_execution_control<'a>(
1066 &'a mut self,
1067 _id: JobExecutionId,
1068 _owner: &'a OwnerToken,
1069 _observed_at: SystemTime,
1070 ) -> BoxFuture<'a, Result<ExecutionControl, RepositoryError>> {
1071 Box::pin(async {
1072 Err(RepositoryError::UnsupportedCapability {
1073 capability: RepositoryCapability::ExecutionOwnership,
1074 })
1075 })
1076 }
1077
1078 fn job_instance_hold(
1080 &mut self,
1081 _id: JobInstanceId,
1082 ) -> BoxFuture<'_, Result<Option<RetentionHold>, RepositoryError>> {
1083 Box::pin(async {
1084 Err(RepositoryError::UnsupportedCapability {
1085 capability: RepositoryCapability::InstanceHolds,
1086 })
1087 })
1088 }
1089
1090 fn place_instance_hold<'a>(
1092 &'a mut self,
1093 _id: JobInstanceId,
1094 _actor: &'a ActorRef,
1095 _reason: &'a ReasonCode,
1096 _placed_at: SystemTime,
1097 ) -> BoxFuture<'a, Result<RetentionHold, RepositoryError>> {
1098 Box::pin(async {
1099 Err(RepositoryError::UnsupportedCapability {
1100 capability: RepositoryCapability::InstanceHolds,
1101 })
1102 })
1103 }
1104
1105 fn release_instance_hold(
1107 &mut self,
1108 _id: JobInstanceId,
1109 ) -> BoxFuture<'_, Result<Option<RetentionHold>, RepositoryError>> {
1110 Box::pin(async {
1111 Err(RepositoryError::UnsupportedCapability {
1112 capability: RepositoryCapability::InstanceHolds,
1113 })
1114 })
1115 }
1116
1117 fn find_retention_action<'a>(
1119 &'a mut self,
1120 _action: RetentionAction,
1121 _operation_id: &'a OperationId,
1122 ) -> BoxFuture<'a, Result<Option<RetentionRecord>, RepositoryError>> {
1123 Box::pin(async {
1124 Err(RepositoryError::UnsupportedCapability {
1125 capability: RepositoryCapability::RetentionPurge,
1126 })
1127 })
1128 }
1129
1130 fn append_retention_action<'a>(
1132 &'a mut self,
1133 _draft: &'a RetentionRecordDraft,
1134 ) -> BoxFuture<'a, Result<RetentionRecord, RepositoryError>> {
1135 Box::pin(async {
1136 Err(RepositoryError::UnsupportedCapability {
1137 capability: RepositoryCapability::RetentionPurge,
1138 })
1139 })
1140 }
1141
1142 fn purge_survey<'a>(
1144 &'a mut self,
1145 _request: &'a PurgePlanRequest,
1146 ) -> BoxFuture<'a, Result<PurgeSurvey, RepositoryError>> {
1147 Box::pin(async {
1148 Err(RepositoryError::UnsupportedCapability {
1149 capability: RepositoryCapability::RetentionPurge,
1150 })
1151 })
1152 }
1153
1154 fn apply_purge<'a>(
1159 &'a mut self,
1160 _plan: &'a PurgePlan,
1161 ) -> BoxFuture<'a, Result<PurgeCounts, RepositoryError>> {
1162 Box::pin(async {
1163 Err(RepositoryError::UnsupportedCapability {
1164 capability: RepositoryCapability::RetentionPurge,
1165 })
1166 })
1167 }
1168
1169 fn commit<'a>(self: Box<Self>) -> BoxFuture<'a, Result<(), RepositoryError>>
1171 where
1172 Self: 'a;
1173
1174 fn rollback<'a>(self: Box<Self>) -> BoxFuture<'a, Result<(), RepositoryError>>
1178 where
1179 Self: 'a;
1180}
1181
1182#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
1187#[non_exhaustive]
1188pub enum RepositoryCapability {
1189 OperatorRequests,
1191 StopRequests,
1193 ExecutionOwnership,
1195 InstanceHolds,
1197 RetentionPurge,
1199 StepPartitions,
1201}
1202
1203impl RepositoryCapability {
1204 #[must_use]
1206 pub const fn as_str(self) -> &'static str {
1207 match self {
1208 Self::OperatorRequests => "operator requests",
1209 Self::StopRequests => "durable stop requests",
1210 Self::ExecutionOwnership => "execution ownership evidence",
1211 Self::InstanceHolds => "instance holds",
1212 Self::RetentionPurge => "retention purge",
1213 Self::StepPartitions => "durable step partitions",
1214 }
1215 }
1216}
1217
1218#[derive(Clone, Debug, Eq, PartialEq)]
1230pub struct RepositoryDescriptor {
1231 descriptor_version: u32,
1232 schema_version: u32,
1233 capabilities: BTreeSet<RepositoryCapability>,
1234}
1235
1236impl RepositoryDescriptor {
1237 pub const CURRENT_VERSION: u32 = 1;
1239
1240 #[must_use]
1247 pub fn new(
1248 schema_version: u32,
1249 capabilities: impl IntoIterator<Item = RepositoryCapability>,
1250 ) -> Self {
1251 Self {
1252 descriptor_version: Self::CURRENT_VERSION,
1253 schema_version,
1254 capabilities: capabilities.into_iter().collect(),
1255 }
1256 }
1257
1258 #[must_use]
1260 pub const fn descriptor_version(&self) -> u32 {
1261 self.descriptor_version
1262 }
1263
1264 #[must_use]
1266 pub const fn schema_version(&self) -> u32 {
1267 self.schema_version
1268 }
1269
1270 #[must_use]
1272 pub fn declares(&self, capability: RepositoryCapability) -> bool {
1273 self.capabilities.contains(&capability)
1274 }
1275
1276 #[must_use]
1278 pub fn capabilities(&self) -> impl ExactSizeIterator<Item = RepositoryCapability> + '_ {
1279 self.capabilities.iter().copied()
1280 }
1281
1282 pub fn require(&self, capability: RepositoryCapability) -> Result<(), RepositoryError> {
1290 if self.declares(capability) {
1291 Ok(())
1292 } else {
1293 Err(RepositoryError::UnsupportedCapability { capability })
1294 }
1295 }
1296}
1297
1298impl fmt::Display for RepositoryCapability {
1299 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1300 formatter.write_str(self.as_str())
1301 }
1302}
1303
1304#[derive(Clone, Debug, Eq, PartialEq)]
1306#[non_exhaustive]
1307pub enum RepositoryError {
1308 SchemaUninitialized,
1310 MigrationRequired {
1312 current: u32,
1314 supported: u32,
1316 },
1317 NewerSchema {
1319 current: u32,
1321 supported: u32,
1323 },
1324 IdentifierOutOfRange {
1326 kind: IdentifierKind,
1328 value: u64,
1330 },
1331 JobInstanceNotFound {
1333 id: JobInstanceId,
1335 },
1336 JobExecutionNotFound {
1338 id: JobExecutionId,
1340 },
1341 StepExecutionNotFound {
1343 id: StepExecutionId,
1345 },
1346 StepPartitionNotFound {
1348 id: StepPartitionId,
1350 },
1351 EmptyPartitionPlan,
1353 PartitionPlanTooLarge {
1355 max: usize,
1357 },
1358 DuplicatePartitionKey,
1360 PartitionPlanExists {
1362 step_execution_id: StepExecutionId,
1364 },
1365 PartitionPlanNotCommitted {
1367 step_execution_id: StepExecutionId,
1369 },
1370 PartitionUpdateNotAllowed {
1372 id: StepPartitionId,
1374 status: BatchStatus,
1376 },
1377 PartitionWorkerMismatch {
1379 partition_id: StepPartitionId,
1381 worker_step_execution_id: StepExecutionId,
1383 },
1384 PartitionWorkerAlreadyAssigned {
1386 worker_step_execution_id: StepExecutionId,
1388 },
1389 PartitionWorkerStale {
1391 partition_id: StepPartitionId,
1393 worker_step_execution_id: StepExecutionId,
1395 },
1396 PartitionParentNotActive {
1398 step_execution_id: StepExecutionId,
1400 status: BatchStatus,
1402 },
1403 PartitionAggregationIncomplete {
1405 step_execution_id: StepExecutionId,
1407 status: BatchStatus,
1409 },
1410 PartitionStateCorrupt,
1412 DuplicateIdentifier {
1414 kind: IdentifierKind,
1416 value: u64,
1418 },
1419 CompletedInstance {
1421 id: JobInstanceId,
1423 },
1424 AbandonedInstance {
1426 id: JobInstanceId,
1428 },
1429 ExecutionAlreadyActive {
1431 instance_id: JobInstanceId,
1433 execution_id: JobExecutionId,
1435 status: BatchStatus,
1437 },
1438 DefinitionDrift {
1440 job_name: JobName,
1442 revision: DefinitionRevision,
1444 },
1445 DefinitionJobMismatch {
1447 expected: JobName,
1449 actual: JobName,
1451 },
1452 IncompatibleDefinition {
1454 instance_id: JobInstanceId,
1456 },
1457 UnsupportedManifestVersion {
1459 format: u16,
1461 },
1462 InvalidDefinitionUpgrade {
1464 execution_id: JobExecutionId,
1466 },
1467 DefinitionUpgradeConflict {
1469 job_name: JobName,
1471 },
1472 RestartStateNotFound {
1474 execution_id: JobExecutionId,
1476 step_name: StepName,
1478 },
1479 FaultStateCorrupt,
1484 StartLimitExceeded {
1486 instance_id: JobInstanceId,
1488 node_id: NodeId,
1490 limit: StartLimit,
1492 },
1493 FlowStateCorrupt,
1495 RecoveryNotAllowed {
1497 id: JobExecutionId,
1499 status: BatchStatus,
1501 },
1502 ExecutionOwned {
1504 id: JobExecutionId,
1506 },
1507 ExecutionOwnershipNotAllowed {
1509 id: JobExecutionId,
1511 status: BatchStatus,
1513 },
1514 Domain(DomainError),
1516 Identifier(IdGenerationError),
1518 Lifecycle(LifecycleError),
1520 RetentionPlanStale,
1524 UnsupportedCapability {
1526 capability: RepositoryCapability,
1528 },
1529 ConcurrentModification,
1531 CommitOutcomeUnknown,
1536 Unavailable,
1538}
1539
1540impl fmt::Display for RepositoryError {
1541 #[allow(clippy::too_many_lines)]
1542 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1543 match self {
1544 Self::SchemaUninitialized => {
1545 formatter.write_str("PostgreSQL metadata schema is not initialized")
1546 }
1547 Self::MigrationRequired { current, supported } => write!(
1548 formatter,
1549 "PostgreSQL metadata schema version {current} requires migration to {supported}"
1550 ),
1551 Self::NewerSchema { current, supported } => write!(
1552 formatter,
1553 "PostgreSQL metadata schema version {current} is newer than supported version {supported}"
1554 ),
1555 Self::IdentifierOutOfRange { kind, value } => {
1556 write!(
1557 formatter,
1558 "{kind} identifier {value} exceeds PostgreSQL bigint"
1559 )
1560 }
1561 Self::JobInstanceNotFound { id } => {
1562 write!(formatter, "job instance {id} was not found")
1563 }
1564 Self::JobExecutionNotFound { id } => {
1565 write!(formatter, "job execution {id} was not found")
1566 }
1567 Self::StepExecutionNotFound { id } => {
1568 write!(formatter, "step execution {id} was not found")
1569 }
1570 Self::StepPartitionNotFound { id } => {
1571 write!(formatter, "step partition {id} was not found")
1572 }
1573 Self::EmptyPartitionPlan => {
1574 formatter.write_str("partition plan must contain at least one entry")
1575 }
1576 Self::PartitionPlanTooLarge { max } => {
1577 write!(formatter, "partition plan exceeds {max} entries")
1578 }
1579 Self::DuplicatePartitionKey => {
1580 formatter.write_str("partition plan contains a duplicate key")
1581 }
1582 Self::PartitionPlanExists { step_execution_id } => write!(
1583 formatter,
1584 "step execution {step_execution_id} already has a partition plan"
1585 ),
1586 Self::PartitionPlanNotCommitted { step_execution_id } => write!(
1587 formatter,
1588 "step execution {step_execution_id} partition plan must commit before assignment"
1589 ),
1590 Self::PartitionUpdateNotAllowed { id, status } => write!(
1591 formatter,
1592 "step partition {id} cannot be updated from {status}"
1593 ),
1594 Self::PartitionWorkerMismatch {
1595 partition_id,
1596 worker_step_execution_id,
1597 } => write!(
1598 formatter,
1599 "worker step execution {worker_step_execution_id} does not belong to partition {partition_id}"
1600 ),
1601 Self::PartitionWorkerAlreadyAssigned {
1602 worker_step_execution_id,
1603 } => write!(
1604 formatter,
1605 "worker step execution {worker_step_execution_id} is already assigned to a partition"
1606 ),
1607 Self::PartitionWorkerStale {
1608 partition_id,
1609 worker_step_execution_id,
1610 } => write!(
1611 formatter,
1612 "worker step execution {worker_step_execution_id} is not the current worker for partition {partition_id}"
1613 ),
1614 Self::PartitionParentNotActive {
1615 step_execution_id,
1616 status,
1617 } => write!(
1618 formatter,
1619 "partition parent step execution {step_execution_id} cannot mutate children from {status}"
1620 ),
1621 Self::PartitionAggregationIncomplete {
1622 step_execution_id,
1623 status,
1624 } => write!(
1625 formatter,
1626 "step execution {step_execution_id} cannot aggregate a child in {status}"
1627 ),
1628 Self::PartitionStateCorrupt => {
1629 formatter.write_str("durable partition state is unusable and no work may begin")
1630 }
1631 Self::DuplicateIdentifier { kind, value } => {
1632 write!(formatter, "{kind} identifier {value} already exists")
1633 }
1634 Self::CompletedInstance { id } => {
1635 write!(formatter, "job instance {id} is already completed")
1636 }
1637 Self::AbandonedInstance { id } => {
1638 write!(formatter, "job instance {id} is abandoned")
1639 }
1640 Self::ExecutionAlreadyActive {
1641 instance_id,
1642 execution_id,
1643 status,
1644 } => write!(
1645 formatter,
1646 "job instance {instance_id} already has execution {execution_id} in {status}"
1647 ),
1648 Self::DefinitionDrift { job_name, revision } => write!(
1649 formatter,
1650 "job {job_name} definition revision {} has drifted",
1651 revision.as_str()
1652 ),
1653 Self::DefinitionJobMismatch { expected, actual } => write!(
1654 formatter,
1655 "definition for job {actual} cannot be used for job {expected}"
1656 ),
1657 Self::IncompatibleDefinition { instance_id } => write!(
1658 formatter,
1659 "job instance {instance_id} has no direct compatible definition"
1660 ),
1661 Self::UnsupportedManifestVersion { format } => {
1662 write!(
1663 formatter,
1664 "definition manifest format {format} is unsupported"
1665 )
1666 }
1667 Self::InvalidDefinitionUpgrade { execution_id } => write!(
1668 formatter,
1669 "definition upgrade for execution {execution_id} is incomplete"
1670 ),
1671 Self::DefinitionUpgradeConflict { job_name } => {
1672 write!(formatter, "job {job_name} definition upgrade conflicts")
1673 }
1674 Self::RestartStateNotFound {
1675 execution_id,
1676 step_name,
1677 } => write!(
1678 formatter,
1679 "restart execution {execution_id} has no durable source for step {step_name}"
1680 ),
1681 Self::FaultStateCorrupt => {
1682 formatter.write_str("durable fault state is unusable and no work may begin")
1683 }
1684 Self::StartLimitExceeded {
1685 instance_id,
1686 node_id,
1687 limit,
1688 } => write!(
1689 formatter,
1690 "job instance {instance_id} exhausted start limit {} for node {}",
1691 limit.get(),
1692 node_id.as_str()
1693 ),
1694 Self::FlowStateCorrupt => {
1695 formatter.write_str("durable flow history is unusable and no work may begin")
1696 }
1697 Self::RecoveryNotAllowed { id, status } => {
1698 write!(
1699 formatter,
1700 "job execution {id} in {status} cannot be recovered"
1701 )
1702 }
1703 Self::ExecutionOwned { id } => {
1704 write!(formatter, "job execution {id} is owned by another process")
1705 }
1706 Self::ExecutionOwnershipNotAllowed { id, status } => write!(
1707 formatter,
1708 "job execution {id} in {status} cannot acquire process ownership"
1709 ),
1710 Self::Domain(error) => write!(formatter, "invalid repository domain value: {error}"),
1711 Self::Identifier(error) => write!(formatter, "identifier generation failed: {error}"),
1712 Self::Lifecycle(error) => error.fmt(formatter),
1713 Self::RetentionPlanStale => {
1714 formatter.write_str("the purge plan is stale and nothing was deleted")
1715 }
1716 Self::UnsupportedCapability { capability } => {
1717 write!(formatter, "the adapter does not support {capability}")
1718 }
1719 Self::ConcurrentModification => {
1720 formatter.write_str("repository unit of work is based on a stale snapshot")
1721 }
1722 Self::CommitOutcomeUnknown => formatter.write_str(
1723 "PostgreSQL commit outcome is unknown; inspect durable metadata before recovery",
1724 ),
1725 Self::Unavailable => formatter.write_str("repository is unavailable"),
1726 }
1727 }
1728}
1729
1730#[doc(hidden)]
1737pub fn aggregate_partition_parent(
1738 parent: &StepExecution,
1739 expected_version: ExecutionVersion,
1740 aggregate: &PartitionAggregate,
1741 transitioned_at: SystemTime,
1742 failure: Option<FailureSummary>,
1743) -> Result<StepExecution, RepositoryError> {
1744 let transition = if aggregate.status() == BatchStatus::Failed {
1745 LifecycleTransition::failed(
1746 transitioned_at,
1747 failure.ok_or(LifecycleError::FailedTransitionMissingFailure)?,
1748 )
1749 } else {
1750 LifecycleTransition::new(aggregate.status(), transitioned_at)
1751 };
1752 let mut transitioned = parent.clone();
1753 transitioned.transition(expected_version, transition)?;
1754 let metadata = ExecutionMetadata::new(
1755 aggregate.status(),
1756 aggregate.exit_status().clone(),
1757 transitioned.metadata().timestamps(),
1758 aggregate.counts(),
1759 transitioned.metadata().failure(),
1760 )?;
1761 Ok(StepExecution::from_snapshot(
1762 transitioned.id(),
1763 transitioned.job_execution_id(),
1764 transitioned.step_name().clone(),
1765 metadata,
1766 transitioned.version(),
1767 ))
1768}
1769
1770#[doc(hidden)]
1772#[must_use]
1773pub fn map_partition_aggregation(
1774 step_execution_id: StepExecutionId,
1775 error: PartitionAggregationError,
1776) -> RepositoryError {
1777 match error {
1778 PartitionAggregationError::Incomplete { status } => {
1779 RepositoryError::PartitionAggregationIncomplete {
1780 step_execution_id,
1781 status,
1782 }
1783 }
1784 PartitionAggregationError::CountExhausted => {
1785 RepositoryError::Lifecycle(LifecycleError::CountExhausted)
1786 }
1787 PartitionAggregationError::EmptyPlan
1788 | PartitionAggregationError::PlanTooLarge { .. }
1789 | PartitionAggregationError::DuplicateKey => RepositoryError::PartitionStateCorrupt,
1790 }
1791}
1792
1793impl Error for RepositoryError {
1794 fn source(&self) -> Option<&(dyn Error + 'static)> {
1795 match self {
1796 Self::Domain(error) => Some(error),
1797 Self::Identifier(error) => Some(error),
1798 Self::Lifecycle(error) => Some(error),
1799 _ => None,
1800 }
1801 }
1802}
1803
1804impl From<DomainError> for RepositoryError {
1805 fn from(error: DomainError) -> Self {
1806 Self::Domain(error)
1807 }
1808}
1809
1810impl From<IdGenerationError> for RepositoryError {
1811 fn from(error: IdGenerationError) -> Self {
1812 Self::Identifier(error)
1813 }
1814}
1815
1816impl From<LifecycleError> for RepositoryError {
1817 fn from(error: LifecycleError) -> Self {
1818 Self::Lifecycle(error)
1819 }
1820}