Skip to main content

minco_plugin_jobs/
lib.rs

1//! Durable typed work for Minco: typed job contracts, an explicit handler
2//! registry, a bounded versioned envelope, at-least-once dispatch ports,
3//! lease-based execution, retry and permanent-failure semantics, overlap
4//! locks and explicit scheduling contracts.
5//!
6//! Jobs are commands with one registered handler, never domain events. The
7//! durable job row owns execution state, the publication row owns pending
8//! transport delivery, and the queue message is delivery, never truth.
9//! Nothing in this crate schedules itself: dispatch is explicit and bounded,
10//! and disabling the jobs capability creates no queue, worker or schedule.
11//!
12//! Delivery is at least once. Duplicate deliveries are neutralized by an
13//! atomic execution claim; application effects must still be idempotent.
14//! Payload and metadata values never appear in `Debug` output.
15#![forbid(unsafe_code)]
16
17mod envelope;
18mod memory;
19mod policy;
20mod ports;
21mod registry;
22mod services;
23
24pub use envelope::{
25    JOB_ENVELOPE_SCHEMA_VERSION, JobEnvelope, JobOptions, MAX_JOB_ATTEMPTS, MAX_JOB_ENVELOPE_BYTES,
26    MAX_JOB_KEY_BYTES, MAX_JOB_METADATA_ENTRIES, MAX_JOB_METADATA_NAME_BYTES,
27    MAX_JOB_METADATA_VALUE_BYTES, MAX_JOB_NAME_BYTES, MAX_JOB_PARTITION_BYTES,
28    MAX_JOB_PAYLOAD_BYTES, MAX_WORKER_PROFILE_BYTES, semantic_fingerprint, validate_job_name,
29    validate_worker_profile,
30};
31pub use memory::{
32    DispatchAttempt, FailClosedDispatcher, FakeJobDispatcher, MAX_ATTEMPT_HISTORY, MemoryJobStore,
33};
34pub use policy::{
35    BackoffMode, FakeJobClock, JobClock, MAX_BACKOFF_DELAY, RetryPolicy, SystemJobClock,
36};
37pub use ports::{
38    EnqueueOutcome, IngestOutcome, JobAttempt, JobAttemptOutcome, JobClaim, JobDelivery,
39    JobDispatcher, JobError, JobPublication, JobPublicationStore, JobRecord, JobStatus, JobStore,
40    OverlapLockStore, PublicationStatus, validate_worker_claim,
41};
42pub use registry::{
43    Job, JobContext, JobExecutionFailure, JobHandler, JobHandlerRegistry, PayloadUpcaster,
44    ResolvedHandler, TypedJobHandler, failure_codes, typed,
45};
46pub use services::{
47    DEFAULT_EXECUTION_LEASE, DEFAULT_EXECUTION_TIMEOUT, DispatchReport, DurableSubmission,
48    JobExecutionDisposition, JobExecutor, JobRunDisposition, JobSkipReason, JobsPlugin,
49    JobsServices, OVERLAP_RETRY_DELAY, PUBLICATION_RETRY_DELAY, ProfileRoutedDispatcher,
50};
51
52/// Build a pending [`JobRecord`] around an envelope for store submission.
53#[must_use]
54pub const fn pending_record(envelope: JobEnvelope) -> JobRecord {
55    JobRecord {
56        envelope,
57        status: JobStatus::Pending,
58        revision: 1,
59        lease_id: None,
60        lease_expires_at: None,
61        attempt_count: 0,
62        attempts: Vec::new(),
63        failure_code: None,
64        completed_at: None,
65    }
66}
67
68#[cfg(test)]
69mod tests {
70    use super::*;
71    use crate::policy::FakeJobClock;
72    use chrono::TimeDelta;
73    use std::sync::Arc;
74
75    #[derive(serde::Serialize, serde::Deserialize)]
76    struct Confirmation {
77        order_id: String,
78    }
79
80    impl Job for Confirmation {
81        const NAME: &'static str = "orders.send-confirmation";
82        const VERSION: u16 = 1;
83    }
84
85    struct Harness {
86        services: JobsServices,
87        store: Arc<MemoryJobStore>,
88        dispatcher: Arc<FakeJobDispatcher>,
89        clock: Arc<FakeJobClock>,
90    }
91
92    fn services() -> Harness {
93        let registry = Arc::new(JobHandlerRegistry::new());
94        registry
95            .register_typed::<Confirmation, _, _>(|job: Confirmation, _ctx| async move {
96                assert_eq!(job.order_id, "o-1");
97                Ok(())
98            })
99            .expect("register");
100        let (services, store, dispatcher) = JobsServices::memory(registry);
101        let clock = Arc::new(FakeJobClock::starting(chrono::Utc::now()));
102        Harness {
103            services: JobsServices {
104                clock: clock.clone(),
105                ..services
106            },
107            store,
108            dispatcher,
109            clock,
110        }
111    }
112
113    fn envelope() -> JobEnvelope {
114        JobEnvelope::for_job::<Confirmation>(
115            &Confirmation {
116                order_id: "o-1".into(),
117            },
118            "orders-notifications",
119            uuid::Uuid::now_v7(),
120        )
121        .expect("valid envelope")
122    }
123
124    async fn run(
125        harness: &Harness,
126        envelope: &JobEnvelope,
127        worker_execution_id: &str,
128    ) -> Result<JobRunDisposition, JobError> {
129        // Absorb sub-second construction skew: envelopes are minted with
130        // the real clock after the deterministic clock was frozen.
131        harness.clock.advance(TimeDelta::milliseconds(50));
132        harness
133            .services
134            .executor
135            .run(
136                envelope,
137                worker_execution_id,
138                harness.services.clock.as_ref(),
139                harness.services.store.as_ref(),
140                harness.services.publications.as_ref(),
141                harness.services.locks.as_ref(),
142            )
143            .await
144    }
145
146    #[tokio::test]
147    async fn durable_submission_dispatch_and_execution_lifecycle() {
148        let harness = services();
149        let envelope = envelope();
150        match harness
151            .services
152            .submit_durable(envelope.clone())
153            .await
154            .expect("submit")
155        {
156            DurableSubmission::Inserted(job_id) => assert_eq!(job_id, envelope.job_id),
157            DurableSubmission::Duplicate(existing) => {
158                panic!("expected insertion, got duplicate {existing}")
159            }
160        }
161        harness.clock.advance(TimeDelta::milliseconds(50));
162        let report = harness
163            .services
164            .dispatch_due_once("dispatch-1", 10, TimeDelta::minutes(1))
165            .await
166            .expect("dispatch");
167        assert_eq!(report.dispatched, 1);
168        let sent = harness.dispatcher.dispatched();
169        assert_eq!(sent.len(), 1);
170        assert_eq!(sent[0].envelope.job_id, envelope.job_id);
171        let disposition = run(&harness, &sent[0].envelope, "worker-exec-1")
172            .await
173            .expect("execute");
174        assert_eq!(
175            disposition,
176            JobRunDisposition::Executed(JobExecutionDisposition::Succeeded)
177        );
178        let record = harness
179            .store
180            .get(envelope.job_id)
181            .await
182            .expect("get")
183            .expect("present");
184        assert_eq!(record.status, JobStatus::Succeeded);
185        assert_eq!(record.attempt_count, 1);
186    }
187
188    #[tokio::test]
189    async fn altered_transport_copies_cannot_override_the_durable_record() {
190        for mutate in [
191            "payload",
192            "retry-policy",
193            "deadline",
194            "maximum-attempts",
195            "worker-profile",
196            "overlap-key",
197        ] {
198            let harness = services();
199            let envelope = envelope().with(
200                JobOptions::default()
201                    .with_retry(RetryPolicy::fixed(3, 60))
202                    .with_overlap_key("orders.confirm:o-1"),
203            );
204            harness
205                .services
206                .submit_durable(envelope.clone())
207                .await
208                .expect("submit");
209            let mut forged = envelope.clone();
210            match mutate {
211                "payload" => forged.payload = serde_json::json!({ "order_id": "o-forged" }),
212                "retry-policy" => forged.retry = Some(RetryPolicy::fixed(9, 9)),
213                "deadline" => {
214                    forged.deadline = Some(forged.created_at + TimeDelta::days(3));
215                }
216                "maximum-attempts" => forged.maximum_attempts = 99,
217                "worker-profile" => forged.worker_profile = "attacker-profile".into(),
218                "overlap-key" => forged.overlap_key = Some("other-boundary".into()),
219                _ => unreachable!(),
220            }
221            let outcome = run(&harness, &forged, "worker-exec-1").await;
222            let disposition = outcome.expect("a divergent copy fails closed deterministically");
223            assert_eq!(
224                disposition,
225                JobRunDisposition::Executed(JobExecutionDisposition::FailedPermanently {
226                    code: failure_codes::TRANSPORT_INTEGRITY.to_owned()
227                }),
228                "{mutate} mismatch must fail closed"
229            );
230            let record = harness
231                .store
232                .get(envelope.job_id)
233                .await
234                .unwrap()
235                .expect("present");
236            assert_ne!(
237                record.status,
238                JobStatus::Succeeded,
239                "{mutate} never executes"
240            );
241            assert_eq!(
242                record.envelope.payload, envelope.payload,
243                "{mutate} cannot override the durable payload"
244            );
245        }
246    }
247
248    #[tokio::test]
249    async fn missing_durable_row_never_falls_back_to_inline_execution() {
250        let harness = services();
251        let forged = envelope();
252        let disposition = run(&harness, &forged, "worker-exec-1").await.expect("run");
253        assert_eq!(
254            disposition,
255            JobRunDisposition::Skipped(JobSkipReason::Missing),
256            "a durable delivery with no durable row must not execute inline"
257        );
258        assert!(harness.store.records().is_empty());
259    }
260
261    #[tokio::test]
262    async fn stale_claims_cannot_mutate_newer_claims() {
263        let harness = services();
264        let envelope = envelope();
265        harness
266            .services
267            .submit_durable(envelope.clone())
268            .await
269            .expect("submit");
270        let store = harness.services.store.clone();
271        let start = chrono::Utc::now();
272        let stale = store
273            .claim_execution(
274                envelope.job_id,
275                "same-worker-name",
276                start + TimeDelta::minutes(1),
277                start,
278            )
279            .await
280            .expect("stale claim")
281            .expect("claimed");
282        // The lease expires and a newer claimant (same worker name) starts.
283        harness.clock.set(start + TimeDelta::minutes(2));
284        let newer = store
285            .claim_execution(
286                envelope.job_id,
287                "same-worker-name",
288                start + TimeDelta::minutes(30),
289                start + TimeDelta::minutes(2),
290            )
291            .await
292            .expect("newer claim")
293            .expect("reclaimed");
294        assert_ne!(stale.lease_id, newer.lease_id);
295        let error = store
296            .complete(&stale, start + TimeDelta::minutes(3))
297            .await
298            .unwrap_err();
299        assert!(matches!(error, JobError::LeaseFencedOut { .. }));
300        let error = store
301            .schedule_retry_and_publish(
302                &stale,
303                "stale-code",
304                start + TimeDelta::minutes(4),
305                start + TimeDelta::minutes(3),
306            )
307            .await
308            .unwrap_err();
309        assert!(matches!(error, JobError::LeaseFencedOut { .. }));
310        let error = store
311            .fail_permanently(&stale, "stale-code", start + TimeDelta::minutes(3))
312            .await
313            .unwrap_err();
314        assert!(matches!(error, JobError::LeaseFencedOut { .. }));
315        store
316            .complete(&newer, start + TimeDelta::minutes(3))
317            .await
318            .expect("the newer claim still owns the job");
319    }
320
321    #[tokio::test]
322    async fn retry_state_and_next_publication_generation_commit_together() {
323        let registry = Arc::new(JobHandlerRegistry::new());
324        let attempts = Arc::new(std::sync::atomic::AtomicU32::new(0));
325        let counter = attempts.clone();
326        registry
327            .register_typed::<Confirmation, _, _>(move |_: Confirmation, _| {
328                let counter = counter.clone();
329                async move {
330                    if counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 {
331                        Err(JobExecutionFailure::retryable("notification-unavailable"))
332                    } else {
333                        Ok(())
334                    }
335                }
336            })
337            .expect("register");
338        let (base, store, _dispatcher) = JobsServices::memory(registry);
339        let clock = Arc::new(FakeJobClock::starting(chrono::Utc::now()));
340        let services = JobsServices {
341            clock: clock.clone(),
342            ..base
343        };
344        let harness = Harness {
345            services: services.clone(),
346            store: store.clone(),
347            dispatcher: Arc::new(FakeJobDispatcher::new()),
348            clock: clock.clone(),
349        };
350        let envelope = envelope().with(JobOptions::default().with_retry(RetryPolicy::fixed(2, 60)));
351        services
352            .submit_durable(envelope.clone())
353            .await
354            .expect("submit");
355        clock.advance(TimeDelta::milliseconds(50));
356        let delivered = services
357            .dispatch_due_once("dispatch-1", 10, TimeDelta::minutes(1))
358            .await
359            .expect("deliver generation 1");
360        assert_eq!(delivered.dispatched, 1);
361        let first = run(&harness, &envelope, "worker-exec-1")
362            .await
363            .expect("first run");
364        assert!(matches!(
365            first,
366            JobRunDisposition::Executed(JobExecutionDisposition::RetryScheduled { .. })
367        ));
368        // The retry transition and the next publication generation are one
369        // atomic unit: generation 2 exists, pending, at the retry time.
370        let publications = store.publication_records();
371        assert_eq!(publications.len(), 2, "generation 2 was inserted");
372        assert_eq!(
373            publications
374                .iter()
375                .map(|p| p.generation)
376                .collect::<Vec<_>>(),
377            vec![1, 2]
378        );
379        assert_eq!(publications[1].status, PublicationStatus::Pending);
380        assert_ne!(
381            publications[1].publication_id, publications[0].publication_id,
382            "a new retry generation has a new publication identity"
383        );
384        // The next dispatch round claims only the new generation.
385        clock.advance(TimeDelta::milliseconds(50));
386        let report = services
387            .dispatch_due_once("dispatch-2", 10, TimeDelta::minutes(1))
388            .await
389            .expect("dispatch before due time");
390        assert_eq!(report.claimed, 0);
391        clock.advance(TimeDelta::seconds(61));
392        let report = services
393            .dispatch_due_once("dispatch-2", 10, TimeDelta::minutes(1))
394            .await
395            .expect("dispatch at due time");
396        assert_eq!(report.dispatched, 1);
397    }
398
399    #[tokio::test]
400    async fn duplicate_submission_uses_the_semantic_fingerprint() {
401        let harness = services();
402        let options = JobOptions::default().with_dedupe_key("orders.send-confirmation:o-1");
403        harness
404            .services
405            .submit_durable(envelope().with(options.clone()))
406            .await
407            .expect("first");
408        match harness
409            .services
410            .submit_durable(envelope().with(options.clone()))
411            .await
412            .expect("semantically identical resubmission is idempotent")
413        {
414            DurableSubmission::Duplicate(_) => {}
415            DurableSubmission::Inserted(inserted) => {
416                panic!("expected duplicate, got insertion {inserted}")
417            }
418        }
419        let mut conflicting = envelope();
420        conflicting.payload = serde_json::json!({ "order_id": "o-2" });
421        let error = harness
422            .services
423            .submit_durable(conflicting.with(options))
424            .await
425            .expect_err("same key, different semantics fails closed");
426        assert!(matches!(
427            error,
428            JobError::DuplicateSubmissionConflict { .. }
429        ));
430    }
431
432    #[tokio::test]
433    async fn memory_store_reports_duplicate_identity_as_a_duplicate_not_as_missing() {
434        let harness = services();
435        let first = envelope();
436        harness
437            .services
438            .submit_durable(first.clone())
439            .await
440            .expect("first");
441        // Reusing an explicit job identity with different semantics is an
442        // identity collision; the reference store must not report the job
443        // as missing when it demonstrably exists.
444        let mut conflicting = envelope();
445        conflicting.payload = serde_json::json!({ "order_id": "o-other" });
446        conflicting.job_id = first.job_id;
447        let error = harness
448            .store
449            .enqueue_with_intent(pending_record(conflicting))
450            .await
451            .expect_err("identity collision fails closed");
452        assert!(
453            matches!(error, JobError::DuplicateJobIdentity(id) if id == first.job_id),
454            "expected a duplicate-identity error, got {error:?}"
455        );
456        assert!(
457            harness
458                .store
459                .get(first.job_id)
460                .await
461                .expect("get")
462                .is_some(),
463            "the original job still exists"
464        );
465    }
466
467    #[tokio::test]
468    async fn ingest_never_overwrites_an_existing_job_identity() {
469        let harness = services();
470        let first = envelope();
471        harness
472            .services
473            .submit_durable(first.clone())
474            .await
475            .expect("first");
476        // Ingesting a delivery that reuses an existing identity without a
477        // matching dedupe key must fail closed, never replace the record.
478        let mut conflicting = envelope();
479        conflicting.payload = serde_json::json!({ "order_id": "o-other" });
480        conflicting.job_id = first.job_id;
481        conflicting.dedupe_key = None;
482        let error = harness
483            .store
484            .ingest_existing_delivery(pending_record(conflicting))
485            .await
486            .expect_err("identity collision on ingest fails closed");
487        assert!(
488            matches!(error, JobError::DuplicateJobIdentity(id) if id == first.job_id),
489            "expected a duplicate-identity error, got {error:?}"
490        );
491        let surviving = harness
492            .store
493            .get(first.job_id)
494            .await
495            .expect("get")
496            .expect("the original job still exists");
497        assert_eq!(
498            surviving.envelope.payload, first.payload,
499            "the original record was not overwritten"
500        );
501    }
502
503    #[tokio::test]
504    async fn overlap_locks_release_on_every_exit_path() {
505        let registry = Arc::new(JobHandlerRegistry::new());
506        registry
507            .register_typed::<Confirmation, _, _>(|_: Confirmation, _| async {
508                Err(JobExecutionFailure::retryable("always-busy"))
509            })
510            .expect("register");
511        let (base, store, _dispatcher) = JobsServices::memory(registry);
512        let clock = Arc::new(FakeJobClock::starting(chrono::Utc::now()));
513        let services = JobsServices {
514            clock: clock.clone(),
515            ..base
516        };
517        let harness = Harness {
518            services: services.clone(),
519            store: store.clone(),
520            dispatcher: Arc::new(FakeJobDispatcher::new()),
521            clock: clock.clone(),
522        };
523        let options = JobOptions::default()
524            .with_retry(RetryPolicy::fixed(5, 1))
525            .with_overlap_key("orders.confirm:o-1");
526        let envelope = envelope().with(options);
527        services
528            .submit_durable(envelope.clone())
529            .await
530            .expect("submit");
531        let first = run(&harness, &envelope, "worker-exec-1")
532            .await
533            .expect("run");
534        assert!(matches!(
535            first,
536            JobRunDisposition::Executed(JobExecutionDisposition::RetryScheduled { .. })
537        ));
538        // The overlap lock was released before the retry, so the next
539        // attempt is not blocked until TTL.
540        clock.advance(TimeDelta::seconds(2));
541        let second = run(&harness, &envelope, "worker-exec-2")
542            .await
543            .expect("run");
544        assert!(
545            matches!(
546                second,
547                JobRunDisposition::Executed(JobExecutionDisposition::RetryScheduled { .. })
548            ),
549            "the freed overlap boundary admits the retry: {second:?}"
550        );
551    }
552
553    #[tokio::test]
554    async fn profile_routing_fails_closed_for_unknown_profiles() {
555        let mut routed = ProfileRoutedDispatcher::new();
556        let sink = Arc::new(FakeJobDispatcher::new());
557        routed
558            .route("orders-notifications", sink.clone())
559            .expect("route");
560        let envelope = envelope();
561        let delivery = JobDelivery {
562            envelope: envelope.clone(),
563            publication_id: uuid::Uuid::now_v7(),
564        };
565        routed
566            .dispatch(&delivery, chrono::Utc::now())
567            .await
568            .expect("known profile dispatches");
569        let mut foreign = envelope;
570        foreign.worker_profile = "other-profile".into();
571        let foreign_delivery = JobDelivery {
572            envelope: foreign,
573            publication_id: uuid::Uuid::now_v7(),
574        };
575        let error = routed
576            .dispatch(&foreign_delivery, chrono::Utc::now())
577            .await
578            .expect_err("unknown profile fails before provider contact");
579        assert!(matches!(error, JobError::UnknownWorkerProfile(_)));
580        assert_eq!(sink.dispatched().len(), 1);
581    }
582
583    #[tokio::test]
584    async fn rejected_duplicate_route_registration_preserves_the_original_route() {
585        let mut routed = ProfileRoutedDispatcher::new();
586        let original = Arc::new(FakeJobDispatcher::new());
587        routed
588            .route("orders-notifications", original.clone())
589            .expect("route");
590        // A rejected duplicate registration must not replace the bound
591        // dispatcher: the routing table stays exactly as it was.
592        let replacement = Arc::new(FakeJobDispatcher::new());
593        let error = routed
594            .route("orders-notifications", replacement.clone())
595            .expect_err("duplicate registration fails closed");
596        assert!(matches!(error, JobError::InvalidJob(_)));
597        let envelope = envelope();
598        let delivery = JobDelivery {
599            envelope,
600            publication_id: uuid::Uuid::now_v7(),
601        };
602        routed
603            .dispatch(&delivery, chrono::Utc::now())
604            .await
605            .expect("dispatch still succeeds");
606        assert_eq!(
607            original.dispatched().len(),
608            1,
609            "the delivery reaches the original dispatcher"
610        );
611        assert!(
612            replacement.dispatched().is_empty(),
613            "the rejected replacement never receives a delivery"
614        );
615    }
616
617    #[tokio::test]
618    async fn scheduler_ingestion_creates_no_pending_publication() {
619        let harness = services();
620        let mut occurrence = envelope();
621        occurrence.dedupe_key = Some("orders-nightly:2026-08-22T13:00:00Z".into());
622        match harness
623            .store
624            .ingest_existing_delivery(pending_record(occurrence.clone()))
625            .await
626            .expect("ingest")
627        {
628            IngestOutcome::Ingested(job_id) => assert_eq!(job_id, occurrence.job_id),
629            IngestOutcome::Duplicate(existing) => panic!("first occurrence, got {existing}"),
630        }
631        let publications = harness.store.publication_records();
632        assert_eq!(publications.len(), 1);
633        assert_eq!(
634            publications[0].status,
635            PublicationStatus::Published,
636            "ingestion records the existing delivery, never a pending one"
637        );
638        match harness
639            .store
640            .ingest_existing_delivery(pending_record(occurrence.clone()))
641            .await
642            .expect("re-ingest")
643        {
644            IngestOutcome::Duplicate(existing) => assert_eq!(existing, occurrence.job_id),
645            IngestOutcome::Ingested(job_id) => panic!("duplicate occurrence, got {job_id}"),
646        }
647        assert_eq!(
648            harness.store.publication_records().len(),
649            1,
650            "no orphan publication appears"
651        );
652    }
653
654    #[tokio::test]
655    async fn queued_and_inline_modes_are_explicit() {
656        let harness = services();
657        harness
658            .services
659            .submit_inline(envelope())
660            .await
661            .expect("inline");
662        harness
663            .services
664            .submit_queued(envelope())
665            .await
666            .expect("queued");
667        assert_eq!(harness.dispatcher.dispatched().len(), 1);
668    }
669
670    #[tokio::test]
671    async fn deadline_and_unknown_job_behaviour() {
672        let harness = services();
673        let now = harness.clock.now();
674        let expiring =
675            envelope().with(JobOptions::default().with_deadline(now + TimeDelta::seconds(10)));
676        harness
677            .services
678            .submit_durable(expiring.clone())
679            .await
680            .expect("submit");
681        let disposition = run(&harness, &expiring, "worker-exec-1")
682            .await
683            .expect("run");
684        assert_eq!(
685            disposition,
686            JobRunDisposition::Executed(JobExecutionDisposition::Succeeded),
687            "a live deadline admits execution"
688        );
689
690        let registry = Arc::new(JobHandlerRegistry::new());
691        let (base, store, _dispatcher) = JobsServices::memory(registry);
692        let unknown = JobEnvelope::for_parts(
693            "orders.missing",
694            1,
695            serde_json::json!({ "order_id": "o-1" }),
696            "orders-notifications",
697            uuid::Uuid::now_v7(),
698        )
699        .unwrap();
700        base.submit_durable(unknown.clone()).await.unwrap();
701        let disposition = base
702            .executor
703            .run(
704                &unknown,
705                "worker-exec-1",
706                base.clock.as_ref(),
707                base.store.as_ref(),
708                base.publications.as_ref(),
709                base.locks.as_ref(),
710            )
711            .await
712            .unwrap();
713        assert_eq!(
714            disposition,
715            JobRunDisposition::Executed(JobExecutionDisposition::FailedPermanently {
716                code: failure_codes::UNKNOWN_JOB.to_owned()
717            })
718        );
719        let _ = store;
720    }
721
722    #[tokio::test]
723    async fn attempt_history_stays_ordered_and_bounded() {
724        let registry = Arc::new(JobHandlerRegistry::new());
725        registry
726            .register_typed::<Confirmation, _, _>(|_: Confirmation, _| async {
727                Err(JobExecutionFailure::retryable("always-busy"))
728            })
729            .expect("register");
730        let (base, store, _dispatcher) = JobsServices::memory(registry);
731        let clock = Arc::new(FakeJobClock::starting(chrono::Utc::now()));
732        let services = JobsServices {
733            clock: clock.clone(),
734            ..base
735        };
736        let harness = Harness {
737            services: services.clone(),
738            store: store.clone(),
739            dispatcher: Arc::new(FakeJobDispatcher::new()),
740            clock: clock.clone(),
741        };
742        let envelope = envelope().with(JobOptions::default().with_retry(RetryPolicy::fixed(60, 1)));
743        services
744            .submit_durable(envelope.clone())
745            .await
746            .expect("submit");
747        for round in 0..30 {
748            run(&harness, &envelope, &format!("worker-exec-{round}"))
749                .await
750                .expect("run");
751            clock.advance(TimeDelta::seconds(2));
752        }
753        let record = services.store.get(envelope.job_id).await.unwrap().unwrap();
754        assert_eq!(record.attempts.len(), MAX_ATTEMPT_HISTORY);
755        assert!(
756            record.attempts.first().expect("oldest").attempt
757                < record.attempts.last().expect("newest").attempt
758        );
759        assert_eq!(record.attempt_count, 30);
760    }
761}