1use std::{fmt, str::FromStr};
12
13use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema};
14use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
15use thiserror::Error;
16
17use crate::decimal::{UnsignedDecimalError, parse_bounded_u64};
18use crate::{AttemptId, RunId, TenantId, Timestamp};
19
20const MAX_DATABASE_ORDINAL: u64 = i64::MAX as u64;
21const POSITIVE_I64_PATTERN: &str = "^[1-9][0-9]{0,18}$";
22
23#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
30pub struct FencingEpoch(u64);
31
32impl FencingEpoch {
33 pub const FIRST: Self = Self(1);
35
36 pub const MAX: Self = Self(MAX_DATABASE_ORDINAL);
38
39 pub const fn new(value: u64) -> Result<Self, FencingEpochError> {
46 if value == 0 {
47 return Err(FencingEpochError::Zero);
48 }
49 if value > MAX_DATABASE_ORDINAL {
50 return Err(FencingEpochError::AboveMaximum);
51 }
52 Ok(Self(value))
53 }
54
55 #[must_use]
57 pub const fn get(self) -> u64 {
58 self.0
59 }
60
61 #[must_use]
63 pub const fn checked_next(self) -> Option<Self> {
64 if self.0 == MAX_DATABASE_ORDINAL {
65 None
66 } else {
67 Some(Self(self.0 + 1))
68 }
69 }
70}
71
72impl fmt::Display for FencingEpoch {
73 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
74 self.0.fmt(formatter)
75 }
76}
77
78impl FromStr for FencingEpoch {
79 type Err = FencingEpochError;
80
81 fn from_str(value: &str) -> Result<Self, Self::Err> {
82 let value = parse_bounded_u64(value, MAX_DATABASE_ORDINAL)
83 .map_err(FencingEpochError::from_decimal_error)?;
84 Self::new(value)
85 }
86}
87
88impl Serialize for FencingEpoch {
89 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
90 where
91 S: Serializer,
92 {
93 serializer.collect_str(self)
94 }
95}
96
97impl<'de> Deserialize<'de> for FencingEpoch {
98 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
99 where
100 D: Deserializer<'de>,
101 {
102 deserializer.deserialize_str(FencingEpochVisitor)
103 }
104}
105
106impl JsonSchema for FencingEpoch {
107 fn schema_name() -> std::borrow::Cow<'static, str> {
108 "FencingEpoch".into()
109 }
110
111 fn schema_id() -> std::borrow::Cow<'static, str> {
112 concat!(module_path!(), "::FencingEpoch").into()
113 }
114
115 fn json_schema(_: &mut SchemaGenerator) -> Schema {
116 json_schema!({
117 "type": "string",
118 "minLength": 1,
119 "maxLength": 19,
120 "pattern": POSITIVE_I64_PATTERN
121 })
122 }
123
124 fn inline_schema() -> bool {
125 true
126 }
127}
128
129struct FencingEpochVisitor;
130
131impl de::Visitor<'_> for FencingEpochVisitor {
132 type Value = FencingEpoch;
133
134 fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
135 formatter.write_str("a canonical positive decimal PostgreSQL BIGINT fencing epoch")
136 }
137
138 fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
139 where
140 E: de::Error,
141 {
142 value.parse().map_err(E::custom)
143 }
144}
145
146#[derive(Clone, Debug, Eq, Error, PartialEq)]
148#[non_exhaustive]
149pub enum FencingEpochError {
150 #[error("fencing epoch must be positive")]
152 Zero,
153
154 #[error("fencing epoch exceeds the PostgreSQL BIGINT maximum")]
156 AboveMaximum,
157
158 #[error("fencing epoch must contain only unsigned ASCII decimal digits")]
160 InvalidFormat,
161
162 #[error("fencing epoch must use canonical decimal text")]
164 NonCanonical,
165}
166
167impl FencingEpochError {
168 const fn from_decimal_error(error: UnsignedDecimalError) -> Self {
169 match error {
170 UnsignedDecimalError::Empty | UnsignedDecimalError::InvalidCharacter { .. } => {
171 Self::InvalidFormat
172 }
173 UnsignedDecimalError::LeadingZero => Self::NonCanonical,
174 UnsignedDecimalError::TooLong { .. } | UnsignedDecimalError::Overflow => {
175 Self::AboveMaximum
176 }
177 }
178 }
179}
180
181#[derive(Clone, Debug, Deserialize, Eq, Hash, JsonSchema, PartialEq, Serialize)]
188#[serde(deny_unknown_fields)]
189pub struct RunFence {
190 tenant_id: TenantId,
191 run_id: RunId,
192 attempt_id: AttemptId,
193 epoch: FencingEpoch,
194}
195
196impl RunFence {
197 #[must_use]
199 pub const fn new(
200 tenant_id: TenantId,
201 run_id: RunId,
202 attempt_id: AttemptId,
203 epoch: FencingEpoch,
204 ) -> Self {
205 Self {
206 tenant_id,
207 run_id,
208 attempt_id,
209 epoch,
210 }
211 }
212
213 #[must_use]
215 pub const fn tenant_id(&self) -> &TenantId {
216 &self.tenant_id
217 }
218
219 #[must_use]
221 pub const fn run_id(&self) -> RunId {
222 self.run_id
223 }
224
225 #[must_use]
227 pub const fn attempt_id(&self) -> AttemptId {
228 self.attempt_id
229 }
230
231 #[must_use]
233 pub const fn epoch(&self) -> FencingEpoch {
234 self.epoch
235 }
236}
237
238#[derive(Clone, Eq, JsonSchema, PartialEq, Serialize)]
246#[serde(deny_unknown_fields)]
247pub struct RunLease {
248 fence: RunFence,
249 acquired_at: Timestamp,
250 renewed_at: Timestamp,
251 expires_at: Timestamp,
252}
253
254impl RunLease {
255 pub fn restore(
266 fence: RunFence,
267 acquired_at: Timestamp,
268 renewed_at: Timestamp,
269 expires_at: Timestamp,
270 ) -> Result<Self, RunLeaseError> {
271 Self::from_observations(fence, acquired_at, renewed_at, expires_at)
272 }
273
274 pub fn new(
281 fence: RunFence,
282 acquired_at: Timestamp,
283 expires_at: Timestamp,
284 ) -> Result<Self, RunLeaseError> {
285 Self::from_observations(fence, acquired_at, acquired_at, expires_at)
286 }
287
288 pub fn first(
295 tenant_id: TenantId,
296 run_id: RunId,
297 attempt_id: AttemptId,
298 acquired_at: Timestamp,
299 expires_at: Timestamp,
300 ) -> Result<Self, RunLeaseError> {
301 Self::new(
302 RunFence::new(tenant_id, run_id, attempt_id, FencingEpoch::FIRST),
303 acquired_at,
304 expires_at,
305 )
306 }
307
308 #[must_use]
310 pub const fn fence(&self) -> &RunFence {
311 &self.fence
312 }
313
314 #[must_use]
316 pub const fn acquired_at(&self) -> Timestamp {
317 self.acquired_at
318 }
319
320 #[must_use]
322 pub const fn renewed_at(&self) -> Timestamp {
323 self.renewed_at
324 }
325
326 #[must_use]
328 pub const fn expires_at(&self) -> Timestamp {
329 self.expires_at
330 }
331
332 pub fn validate_write(
345 &self,
346 fence: &RunFence,
347 observed_at: Timestamp,
348 ) -> Result<(), RunLeaseValidationError> {
349 self.validate_fence(fence)?;
350 if observed_at < self.acquired_at {
351 return Err(RunLeaseValidationError::ObservationBeforeAcquisition {
352 acquired_at: self.acquired_at,
353 observed_at,
354 });
355 }
356 if observed_at < self.renewed_at {
357 return Err(RunLeaseValidationError::ObservationBeforeRenewal {
358 renewed_at: self.renewed_at,
359 observed_at,
360 });
361 }
362 if observed_at >= self.expires_at {
363 return Err(RunLeaseValidationError::Expired {
364 expires_at: self.expires_at,
365 observed_at,
366 });
367 }
368 Ok(())
369 }
370
371 pub fn renewed(
382 &self,
383 fence: &RunFence,
384 renewed_at: Timestamp,
385 expires_at: Timestamp,
386 ) -> Result<Self, RunLeaseError> {
387 self.validate_fence(fence).map_err(RunLeaseError::fence)?;
388 if renewed_at < self.renewed_at {
389 return Err(RunLeaseError::RenewalClockRegression {
390 previous: self.renewed_at,
391 actual: renewed_at,
392 });
393 }
394 if renewed_at >= self.expires_at {
395 return Err(RunLeaseError::RenewalTooLate {
396 expires_at: self.expires_at,
397 renewed_at,
398 });
399 }
400 if expires_at <= self.expires_at {
401 return Err(RunLeaseError::ExpiryNotExtended {
402 previous: self.expires_at,
403 actual: expires_at,
404 });
405 }
406 Self::from_observations(self.fence.clone(), self.acquired_at, renewed_at, expires_at)
407 }
408
409 pub fn superseded(
420 &self,
421 attempt_id: AttemptId,
422 acquired_at: Timestamp,
423 expires_at: Timestamp,
424 ) -> Result<Self, RunLeaseError> {
425 if attempt_id == self.fence.attempt_id {
426 return Err(RunLeaseError::AttemptReused { attempt_id });
427 }
428 if acquired_at < self.renewed_at {
429 return Err(RunLeaseError::SupersessionClockRegression {
430 previous: self.renewed_at,
431 actual: acquired_at,
432 });
433 }
434 let epoch = self
435 .fence
436 .epoch
437 .checked_next()
438 .ok_or(RunLeaseError::EpochOverflow)?;
439 let fence = RunFence::new(
440 self.fence.tenant_id.clone(),
441 self.fence.run_id,
442 attempt_id,
443 epoch,
444 );
445 Self::new(fence, acquired_at, expires_at)
446 }
447
448 fn from_observations(
449 fence: RunFence,
450 acquired_at: Timestamp,
451 renewed_at: Timestamp,
452 expires_at: Timestamp,
453 ) -> Result<Self, RunLeaseError> {
454 if renewed_at < acquired_at {
455 return Err(RunLeaseError::RenewalBeforeAcquisition {
456 acquired_at,
457 renewed_at,
458 });
459 }
460 if expires_at <= renewed_at {
461 return Err(RunLeaseError::ExpiryNotAfterObservation {
462 observed_at: renewed_at,
463 expires_at,
464 });
465 }
466 Ok(Self {
467 fence,
468 acquired_at,
469 renewed_at,
470 expires_at,
471 })
472 }
473
474 fn validate_fence(&self, fence: &RunFence) -> Result<(), RunLeaseValidationError> {
475 if fence.tenant_id != self.fence.tenant_id {
476 return Err(RunLeaseValidationError::TenantMismatch);
477 }
478 if fence.run_id != self.fence.run_id {
479 return Err(RunLeaseValidationError::RunMismatch);
480 }
481 if fence.attempt_id != self.fence.attempt_id {
482 return Err(RunLeaseValidationError::AttemptMismatch {
483 expected: self.fence.attempt_id,
484 actual: fence.attempt_id,
485 });
486 }
487 if fence.epoch != self.fence.epoch {
488 return Err(RunLeaseValidationError::EpochMismatch {
489 expected: self.fence.epoch,
490 actual: fence.epoch,
491 });
492 }
493 Ok(())
494 }
495}
496
497impl fmt::Debug for RunLease {
498 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
499 formatter
500 .debug_struct("RunLease")
501 .field("fence", &self.fence)
502 .field("acquired_at", &self.acquired_at)
503 .field("renewed_at", &self.renewed_at)
504 .field("expires_at", &self.expires_at)
505 .finish_non_exhaustive()
506 }
507}
508
509impl<'de> Deserialize<'de> for RunLease {
510 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
511 where
512 D: Deserializer<'de>,
513 {
514 #[derive(Deserialize)]
515 #[serde(deny_unknown_fields)]
516 struct Wire {
517 fence: RunFence,
518 acquired_at: Timestamp,
519 renewed_at: Timestamp,
520 expires_at: Timestamp,
521 }
522
523 let wire = Wire::deserialize(deserializer)?;
524 Self::from_observations(
525 wire.fence,
526 wire.acquired_at,
527 wire.renewed_at,
528 wire.expires_at,
529 )
530 .map_err(de::Error::custom)
531 }
532}
533
534#[derive(Clone, Debug, Eq, Error, PartialEq)]
536#[non_exhaustive]
537pub enum RunLeaseError {
538 #[error("lease renewal {renewed_at} precedes acquisition {acquired_at}")]
540 RenewalBeforeAcquisition {
541 acquired_at: Timestamp,
543 renewed_at: Timestamp,
545 },
546
547 #[error("lease expiry {expires_at} must be later than observation {observed_at}")]
549 ExpiryNotAfterObservation {
550 observed_at: Timestamp,
552 expires_at: Timestamp,
554 },
555
556 #[error("lease renewal fencing token is stale or belongs to another run")]
558 Fence {
559 #[source]
561 source: RunLeaseValidationError,
562 },
563
564 #[error("lease renewal {actual} precedes previous observation {previous}")]
566 RenewalClockRegression {
567 previous: Timestamp,
569 actual: Timestamp,
571 },
572
573 #[error("lease renewal {renewed_at} is not before expiry {expires_at}")]
575 RenewalTooLate {
576 expires_at: Timestamp,
578 renewed_at: Timestamp,
580 },
581
582 #[error("renewed expiry {actual} must be later than previous expiry {previous}")]
584 ExpiryNotExtended {
585 previous: Timestamp,
587 actual: Timestamp,
589 },
590
591 #[error("successor lease cannot reuse attempt {attempt_id}")]
593 AttemptReused {
594 attempt_id: AttemptId,
596 },
597
598 #[error("lease supersession {actual} precedes previous observation {previous}")]
600 SupersessionClockRegression {
601 previous: Timestamp,
603 actual: Timestamp,
605 },
606
607 #[error("fencing epoch overflowed")]
609 EpochOverflow,
610}
611
612impl RunLeaseError {
613 const fn fence(source: RunLeaseValidationError) -> Self {
614 Self::Fence { source }
615 }
616}
617
618#[derive(Clone, Debug, Eq, Error, PartialEq)]
620#[non_exhaustive]
621pub enum RunLeaseValidationError {
622 #[error("fencing token tenant does not match the current lease")]
624 TenantMismatch,
625
626 #[error("fencing token run does not match the current lease")]
628 RunMismatch,
629
630 #[error("fencing token attempt {actual} does not match current attempt {expected}")]
632 AttemptMismatch {
633 expected: AttemptId,
635 actual: AttemptId,
637 },
638
639 #[error("fencing epoch {actual} does not match current epoch {expected}")]
641 EpochMismatch {
642 expected: FencingEpoch,
644 actual: FencingEpoch,
646 },
647
648 #[error("write observation {observed_at} precedes lease acquisition {acquired_at}")]
650 ObservationBeforeAcquisition {
651 acquired_at: Timestamp,
653 observed_at: Timestamp,
655 },
656
657 #[error("write observation {observed_at} precedes latest lease renewal {renewed_at}")]
659 ObservationBeforeRenewal {
660 renewed_at: Timestamp,
662 observed_at: Timestamp,
664 },
665
666 #[error("lease expired at {expires_at}; write observed at {observed_at}")]
668 Expired {
669 expires_at: Timestamp,
671 observed_at: Timestamp,
673 },
674}
675
676#[cfg(test)]
677mod tests {
678 use super::*;
679 use proptest::prelude::*;
680 use serde_json::{Value, from_value, json, to_value};
681
682 fn at(offset_micros: i64) -> Timestamp {
683 let base = "2030-01-01T00:00:00.000000Z".parse::<Timestamp>().unwrap();
684 Timestamp::from_unix_micros(base.unix_micros() + offset_micros).unwrap()
685 }
686
687 fn tenant() -> TenantId {
688 TenantId::try_from("tenant-a").unwrap()
689 }
690
691 fn run() -> RunId {
692 "01912345-6789-7abc-8def-0123456789ab".parse().unwrap()
693 }
694
695 fn attempt(suffix: &str) -> AttemptId {
696 format!("01912345-6789-7abc-8def-0123456789{suffix}")
697 .parse()
698 .unwrap()
699 }
700
701 fn lease() -> RunLease {
702 RunLease::first(tenant(), run(), attempt("ac"), at(10), at(20)).unwrap()
703 }
704
705 #[test]
706 fn fencing_epochs_are_positive_canonical_database_ordinals() {
707 assert_eq!(FencingEpoch::new(1), Ok(FencingEpoch::FIRST));
708 assert_eq!(FencingEpoch::new(0), Err(FencingEpochError::Zero));
709 assert_eq!(
710 FencingEpoch::new(MAX_DATABASE_ORDINAL + 1),
711 Err(FencingEpochError::AboveMaximum)
712 );
713 assert_eq!(FencingEpoch::MAX.checked_next(), None);
714 assert_eq!(FencingEpoch::FIRST.checked_next().unwrap().get(), 2);
715
716 assert_eq!(
717 to_value(FencingEpoch::MAX).unwrap(),
718 json!(i64::MAX.to_string())
719 );
720 assert_eq!(
721 from_value::<FencingEpoch>(json!("1")).unwrap(),
722 FencingEpoch::FIRST
723 );
724 for invalid in [json!("0"), json!("01"), json!("x"), json!(1), Value::Null] {
725 assert!(from_value::<FencingEpoch>(invalid).is_err());
726 }
727 }
728
729 #[test]
730 fn fencing_epoch_schema_matches_runtime_shape() {
731 let schema = to_value(schemars::schema_for!(FencingEpoch)).unwrap();
732 assert_eq!(schema["type"], "string");
733 assert_eq!(schema["minLength"], 1);
734 assert_eq!(schema["maxLength"], 19);
735 assert_eq!(schema["pattern"], POSITIVE_I64_PATTERN);
736 }
737
738 #[test]
739 fn lease_expiry_is_exclusive() {
740 let lease = lease();
741 assert_eq!(lease.fence().epoch(), FencingEpoch::FIRST);
742 assert_eq!(lease.acquired_at(), at(10));
743 assert_eq!(lease.renewed_at(), at(10));
744 assert_eq!(lease.expires_at(), at(20));
745 assert_eq!(lease.validate_write(lease.fence(), at(10)), Ok(()));
746 assert_eq!(lease.validate_write(lease.fence(), at(19)), Ok(()));
747 assert_eq!(
748 lease.validate_write(lease.fence(), at(20)),
749 Err(RunLeaseValidationError::Expired {
750 expires_at: at(20),
751 observed_at: at(20),
752 })
753 );
754 assert!(matches!(
755 lease.validate_write(lease.fence(), at(9)),
756 Err(RunLeaseValidationError::ObservationBeforeAcquisition { .. })
757 ));
758 }
759
760 #[test]
761 fn renewal_keeps_the_fence_and_strictly_extends_expiry() {
762 let lease = lease();
763 let renewed = lease.renewed(lease.fence(), at(19), at(30)).unwrap();
764 assert_eq!(renewed.fence(), lease.fence());
765 assert_eq!(renewed.acquired_at(), at(10));
766 assert_eq!(renewed.renewed_at(), at(19));
767 assert_eq!(renewed.expires_at(), at(30));
768
769 assert!(matches!(
770 lease.renewed(lease.fence(), at(20), at(30)),
771 Err(RunLeaseError::RenewalTooLate { .. })
772 ));
773 assert!(matches!(
774 lease.renewed(lease.fence(), at(19), at(20)),
775 Err(RunLeaseError::ExpiryNotExtended { .. })
776 ));
777 let twice = renewed.renewed(renewed.fence(), at(18), at(40));
778 assert!(matches!(
779 twice,
780 Err(RunLeaseError::RenewalClockRegression { .. })
781 ));
782 }
783
784 #[test]
785 fn storage_restore_preserves_renewal_observations_and_revalidates_timing() {
786 let original = lease().renewed(lease().fence(), at(19), at(30)).unwrap();
787 let restored = RunLease::restore(
788 original.fence().clone(),
789 original.acquired_at(),
790 original.renewed_at(),
791 original.expires_at(),
792 )
793 .unwrap();
794 assert_eq!(restored, original);
795 assert_eq!(
796 restored.validate_write(restored.fence(), at(18)),
797 Err(RunLeaseValidationError::ObservationBeforeRenewal {
798 renewed_at: at(19),
799 observed_at: at(18),
800 })
801 );
802 assert!(matches!(
803 RunLease::restore(
804 original.fence().clone(),
805 original.acquired_at(),
806 at(9),
807 original.expires_at(),
808 ),
809 Err(RunLeaseError::RenewalBeforeAcquisition { .. })
810 ));
811 }
812
813 #[test]
814 fn supersession_fences_the_old_attempt_even_before_expiry() {
815 let old = lease();
816 let new = old.superseded(attempt("ad"), at(15), at(25)).unwrap();
817 assert_eq!(new.fence().epoch().get(), 2);
818 assert_eq!(new.fence().attempt_id(), attempt("ad"));
819 assert_eq!(
820 new.validate_write(old.fence(), at(16)),
821 Err(RunLeaseValidationError::AttemptMismatch {
822 expected: attempt("ad"),
823 actual: attempt("ac"),
824 })
825 );
826 assert_eq!(new.validate_write(new.fence(), at(16)), Ok(()));
827 assert!(matches!(
828 old.superseded(attempt("ac"), at(15), at(25)),
829 Err(RunLeaseError::AttemptReused { .. })
830 ));
831 assert!(matches!(
832 old.superseded(attempt("ad"), at(9), at(25)),
833 Err(RunLeaseError::SupersessionClockRegression { .. })
834 ));
835 }
836
837 #[test]
838 fn every_token_component_is_bound() {
839 let lease = lease();
840 let other_tenant = RunFence::new(
841 TenantId::try_from("tenant-b").unwrap(),
842 run(),
843 attempt("ac"),
844 FencingEpoch::FIRST,
845 );
846 assert_eq!(
847 lease.validate_write(&other_tenant, at(15)),
848 Err(RunLeaseValidationError::TenantMismatch)
849 );
850
851 let other_run = RunFence::new(
852 tenant(),
853 "01912345-6789-7abc-8def-0123456789bb".parse().unwrap(),
854 attempt("ac"),
855 FencingEpoch::FIRST,
856 );
857 assert_eq!(
858 lease.validate_write(&other_run, at(15)),
859 Err(RunLeaseValidationError::RunMismatch)
860 );
861
862 let other_epoch = RunFence::new(
863 tenant(),
864 run(),
865 attempt("ac"),
866 FencingEpoch::new(2).unwrap(),
867 );
868 assert!(matches!(
869 lease.validate_write(&other_epoch, at(15)),
870 Err(RunLeaseValidationError::EpochMismatch { .. })
871 ));
872 }
873
874 #[test]
875 fn snapshot_deserialization_revalidates_timing_and_shape() {
876 let encoded = to_value(lease()).unwrap();
877 assert_eq!(from_value::<RunLease>(encoded.clone()).unwrap(), lease());
878
879 let mut expiry = encoded.clone();
880 expiry["expires_at"] = expiry["renewed_at"].clone();
881 assert!(from_value::<RunLease>(expiry).is_err());
882
883 let mut renewal = encoded.clone();
884 renewal["renewed_at"] = json!(at(9).to_string());
885 assert!(from_value::<RunLease>(renewal).is_err());
886
887 let mut extra = encoded;
888 extra["owner"] = json!("untrusted-worker");
889 assert!(from_value::<RunLease>(extra).is_err());
890 }
891
892 #[test]
893 fn successor_rejects_epoch_exhaustion() {
894 let fence = RunFence::new(tenant(), run(), attempt("ac"), FencingEpoch::MAX);
895 let lease = RunLease::new(fence, at(10), at(20)).unwrap();
896 assert_eq!(
897 lease.superseded(attempt("ad"), at(20), at(30)),
898 Err(RunLeaseError::EpochOverflow)
899 );
900 }
901
902 proptest! {
903 #[test]
904 fn accepted_renewals_preserve_token_and_monotonically_extend(
905 renewal_offsets in proptest::collection::vec(1_i64..10_000, 1..64)
906 ) {
907 let mut lease = RunLease::first(
908 tenant(), run(), attempt("ac"), at(0), at(10_000)
909 ).unwrap();
910 let fence = lease.fence().clone();
911 let mut observed = 0_i64;
912 for offset in renewal_offsets {
913 observed = (observed + offset).min(9_999);
914 let new_expiry = lease.expires_at().unix_micros().checked_add(10_000).unwrap();
915 let renewed = lease.renewed(
916 &fence,
917 Timestamp::from_unix_micros(at(0).unix_micros() + observed).unwrap(),
918 Timestamp::from_unix_micros(new_expiry).unwrap(),
919 ).unwrap();
920 prop_assert_eq!(renewed.fence(), &fence);
921 prop_assert!(renewed.expires_at() > lease.expires_at());
922 lease = renewed;
923 }
924 }
925 }
926}