1use aequora_types::{AuthorityEpoch, AuthorityId, RegionId, Sequence, TenantId};
8use serde::{Deserialize, Serialize};
9use std::{
10 collections::{BTreeMap, BTreeSet},
11 fmt,
12 str::FromStr,
13 time::Duration,
14};
15use thiserror::Error;
16use uuid::Uuid;
17
18#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
20#[serde(transparent)]
21pub struct ReplicaId(Uuid);
22
23impl ReplicaId {
24 #[must_use]
26 pub fn new() -> Self {
27 Self(Uuid::now_v7())
28 }
29
30 #[must_use]
32 pub const fn from_uuid(value: Uuid) -> Self {
33 Self(value)
34 }
35
36 #[must_use]
38 pub const fn as_uuid(self) -> Uuid {
39 self.0
40 }
41}
42
43impl Default for ReplicaId {
44 fn default() -> Self {
45 Self::new()
46 }
47}
48
49impl fmt::Display for ReplicaId {
50 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
51 self.0.fmt(f)
52 }
53}
54
55impl FromStr for ReplicaId {
56 type Err = uuid::Error;
57
58 fn from_str(value: &str) -> Result<Self, Self::Err> {
59 Uuid::parse_str(value).map(Self)
60 }
61}
62
63#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
65#[serde(transparent)]
66pub struct AuthorityShardId(pub u32);
67
68#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
70#[serde(transparent)]
71pub struct ProjectionId(pub u32);
72
73#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
75#[serde(transparent)]
76pub struct ProjectionSchemaVersion(pub u32);
77
78#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
80pub enum RegionalRole {
81 AuthorityWriter,
83 ReadReplica,
85 EdgeOnly,
87 RecoveryStandby,
89}
90
91#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
93pub enum RegionMode {
94 Healthy,
95 ReadOnlyStale,
96 Disconnected,
97 PromotionCandidate,
98}
99
100#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)]
102pub enum RegionHealth {
103 Healthy,
104 Degraded,
105 Unavailable,
106}
107
108#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
110pub struct ReplicaWatermark {
111 pub authority_id: AuthorityId,
112 pub authority_epoch: AuthorityEpoch,
113 pub sequence: Sequence,
114}
115
116impl ReplicaWatermark {
117 #[must_use]
119 pub fn covers(
120 self,
121 authority_id: AuthorityId,
122 authority_epoch: AuthorityEpoch,
123 sequence: Sequence,
124 ) -> bool {
125 self.authority_id == authority_id
126 && self.authority_epoch == authority_epoch
127 && self.sequence.0 >= sequence.0
128 }
129}
130
131#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
133pub struct ProjectionWatermark {
134 pub projection_id: ProjectionId,
135 pub schema_version: ProjectionSchemaVersion,
136 pub authority_id: AuthorityId,
137 pub authority_epoch: AuthorityEpoch,
138 pub sequence: Sequence,
139}
140
141#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
143pub enum ReadConsistency {
144 Eventual,
145 AtLeast(Sequence),
146 Session,
147 Authority,
148}
149
150#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
152pub struct SessionWatermark {
153 pub authority_id: AuthorityId,
154 pub authority_epoch: AuthorityEpoch,
155 pub min_sequence: Sequence,
156}
157
158impl SessionWatermark {
159 #[must_use]
161 pub const fn new(
162 authority_id: AuthorityId,
163 authority_epoch: AuthorityEpoch,
164 min_sequence: Sequence,
165 ) -> Self {
166 Self {
167 authority_id,
168 authority_epoch,
169 min_sequence,
170 }
171 }
172
173 pub fn observe(
180 &mut self,
181 authority_id: AuthorityId,
182 authority_epoch: AuthorityEpoch,
183 sequence: Sequence,
184 ) -> Result<(), RegionError> {
185 if authority_id != self.authority_id {
186 return Err(RegionError::AuthorityMismatch);
187 }
188 if authority_epoch < self.authority_epoch {
189 return Err(RegionError::EpochRollback {
190 trusted: self.authority_epoch,
191 received: authority_epoch,
192 });
193 }
194 if authority_epoch > self.authority_epoch {
195 self.authority_epoch = authority_epoch;
196 self.min_sequence = sequence;
197 } else if sequence > self.min_sequence {
198 self.min_sequence = sequence;
199 }
200 Ok(())
201 }
202}
203
204#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
206pub enum StalenessBudget {
207 None,
209 MaxSequenceLag(u64),
210 MaxDuration(Duration),
211}
212
213#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
215pub enum ReadFallbackPolicy {
216 NeverDowngrade,
217 AllowEventual,
218 AllowBoundedStale,
219}
220
221#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
223pub struct EndpointReadPolicy {
224 pub consistency: ReadConsistency,
225 pub staleness_budget: StalenessBudget,
226 pub fallback: ReadFallbackPolicy,
227 pub allow_authority_fallback: bool,
228 pub authorization_sensitive: bool,
229}
230
231impl EndpointReadPolicy {
232 #[must_use]
234 pub const fn session() -> Self {
235 Self {
236 consistency: ReadConsistency::Session,
237 staleness_budget: StalenessBudget::None,
238 fallback: ReadFallbackPolicy::NeverDowngrade,
239 allow_authority_fallback: true,
240 authorization_sensitive: false,
241 }
242 }
243
244 #[must_use]
246 pub const fn authority() -> Self {
247 Self {
248 consistency: ReadConsistency::Authority,
249 staleness_budget: StalenessBudget::None,
250 fallback: ReadFallbackPolicy::NeverDowngrade,
251 allow_authority_fallback: true,
252 authorization_sensitive: true,
253 }
254 }
255}
256
257#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
259pub struct AuthorityLocation {
260 pub region_id: RegionId,
261 pub endpoint: String,
262 pub authority_id: AuthorityId,
263 pub authority_epoch: AuthorityEpoch,
264 pub shard_id: Option<AuthorityShardId>,
265 pub latest_sequence: Sequence,
266}
267
268impl AuthorityLocation {
269 pub fn validate(&self) -> Result<(), RegionError> {
275 let length = self.endpoint.len();
276 if length == 0 || length > 2_048 {
277 return Err(RegionError::InvalidEndpoint);
278 }
279 Ok(())
280 }
281}
282
283#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
285pub struct ReplicaObservation {
286 pub replica_id: ReplicaId,
287 pub region_id: RegionId,
288 pub role: RegionalRole,
289 pub watermark: ReplicaWatermark,
290 pub projection: Option<ProjectionWatermark>,
291 pub authorization_sequence: Option<Sequence>,
292 pub observed_authority_sequence: Sequence,
293 pub approximate_time_lag: Duration,
294 pub health: RegionHealth,
295 pub mode: RegionMode,
296 pub priority: u16,
297 pub load_basis_points: u16,
299 pub maintenance: bool,
300}
301
302impl ReplicaObservation {
303 #[must_use]
305 pub const fn sequence_lag(&self) -> u64 {
306 self.observed_authority_sequence
307 .0
308 .saturating_sub(self.watermark.sequence.0)
309 }
310
311 fn validate(&self) -> Result<(), RegionError> {
312 if self.load_basis_points > 10_000 {
313 return Err(RegionError::InvalidLoad);
314 }
315 Ok(())
316 }
317}
318
319#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
321pub struct RegionalReadRequest {
322 pub authority_id: AuthorityId,
323 pub authority_epoch: AuthorityEpoch,
324 pub policy: EndpointReadPolicy,
325 pub session: Option<SessionWatermark>,
326 pub region_hint: Option<RegionId>,
327 pub required_projection: Option<(ProjectionId, ProjectionSchemaVersion)>,
328 pub minimum_authorization_sequence: Option<Sequence>,
329}
330
331impl RegionalReadRequest {
332 fn required_sequence(self) -> Result<Option<Sequence>, RegionError> {
333 match self.policy.consistency {
334 ReadConsistency::Eventual | ReadConsistency::Authority => Ok(None),
335 ReadConsistency::AtLeast(sequence) => Ok(Some(sequence)),
336 ReadConsistency::Session => {
337 let session = self.session.ok_or(RegionError::MissingSessionWatermark)?;
338 if session.authority_id != self.authority_id {
339 return Err(RegionError::AuthorityMismatch);
340 }
341 if session.authority_epoch != self.authority_epoch {
342 return Err(RegionError::WrongEpoch {
343 available: session.authority_epoch,
344 required: self.authority_epoch,
345 });
346 }
347 Ok(Some(session.min_sequence))
348 }
349 }
350 }
351}
352
353#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
355pub enum ReadTarget {
356 RegionalReplica(ReplicaId),
357 Authority,
358}
359
360#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
362pub struct ReplicaWait {
363 pub replica_id: ReplicaId,
364 pub required_sequence: Sequence,
365 pub timeout: Duration,
366}
367
368#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
370pub struct ReadRouteDecision {
371 pub target: ReadTarget,
372 pub effective_consistency: ReadConsistency,
373 pub downgraded: bool,
374 pub wait_before_fallback: Option<ReplicaWait>,
375}
376
377#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
379pub struct ReadResponseMetadata {
380 pub authority_id: AuthorityId,
381 pub authority_epoch: AuthorityEpoch,
382 pub served_sequence: Sequence,
383 pub region: RegionId,
384 pub projection_version: Option<ProjectionSchemaVersion>,
385}
386
387#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
389pub struct RegionalCacheMetadata {
390 pub authority_id: AuthorityId,
391 pub authority_epoch: AuthorityEpoch,
392 pub served_sequence: Sequence,
393 pub projection_version: Option<ProjectionSchemaVersion>,
394}
395
396impl RegionalCacheMetadata {
397 pub fn satisfies(self, request: RegionalReadRequest) -> Result<bool, RegionError> {
403 if self.authority_id != request.authority_id
404 || self.authority_epoch != request.authority_epoch
405 {
406 return Ok(false);
407 }
408 if let Some((_, version)) = request.required_projection {
409 if self.projection_version != Some(version) {
410 return Ok(false);
411 }
412 }
413 Ok(request
414 .required_sequence()?
415 .is_none_or(|minimum| self.served_sequence >= minimum))
416 }
417}
418
419#[derive(Clone, Copy, Debug, Default)]
421pub struct ReplicaReadGuard;
422
423impl ReplicaReadGuard {
424 pub fn validate(
430 observation: &ReplicaObservation,
431 request: RegionalReadRequest,
432 ) -> Result<ReadResponseMetadata, RegionError> {
433 validate_replica_base(observation, request)?;
434 if matches!(
435 observation.mode,
436 RegionMode::ReadOnlyStale | RegionMode::Disconnected
437 ) && request.policy.consistency != ReadConsistency::Eventual
438 {
439 return Err(RegionError::ReplicaUnavailable);
440 }
441 let required = request.required_sequence()?;
442 if let Some(required) = required {
443 if observation.watermark.sequence < required {
444 return Err(RegionError::ReplicaTooStale {
445 available: observation.watermark.sequence,
446 required,
447 });
448 }
449 }
450 if request.policy.authorization_sensitive
451 || request.minimum_authorization_sequence.is_some()
452 {
453 let required_security = request
454 .minimum_authorization_sequence
455 .or(required)
456 .unwrap_or(observation.observed_authority_sequence);
457 if observation.authorization_sequence < Some(required_security) {
458 return Err(RegionError::AuthorizationTooStale {
459 available: observation.authorization_sequence,
460 required: required_security,
461 });
462 }
463 }
464 if !within_budget(observation, request.policy.staleness_budget) {
465 return Err(RegionError::StalenessBudgetExceeded);
466 }
467 Ok(ReadResponseMetadata {
468 authority_id: observation.watermark.authority_id,
469 authority_epoch: observation.watermark.authority_epoch,
470 served_sequence: observation.watermark.sequence,
471 region: observation.region_id,
472 projection_version: observation.projection.map(|value| value.schema_version),
473 })
474 }
475}
476
477fn validate_replica_base(
478 observation: &ReplicaObservation,
479 request: RegionalReadRequest,
480) -> Result<(), RegionError> {
481 observation.validate()?;
482 if observation.role != RegionalRole::ReadReplica {
483 return Err(RegionError::NotReadReplica);
484 }
485 if observation.maintenance || observation.health == RegionHealth::Unavailable {
486 return Err(RegionError::ReplicaUnavailable);
487 }
488 if matches!(observation.mode, RegionMode::PromotionCandidate) {
489 return Err(RegionError::ReplicaUnavailable);
490 }
491 if observation.watermark.authority_id != request.authority_id {
492 return Err(RegionError::AuthorityMismatch);
493 }
494 if observation.watermark.authority_epoch != request.authority_epoch {
495 return Err(RegionError::WrongEpoch {
496 available: observation.watermark.authority_epoch,
497 required: request.authority_epoch,
498 });
499 }
500 if let Some((projection_id, schema_version)) = request.required_projection {
501 let projection = observation
502 .projection
503 .ok_or(RegionError::ProjectionUnavailable)?;
504 if projection.projection_id != projection_id
505 || projection.schema_version != schema_version
506 || projection.authority_id != request.authority_id
507 || projection.authority_epoch != request.authority_epoch
508 || projection.sequence > observation.watermark.sequence
509 {
510 return Err(RegionError::ProjectionIncompatible);
511 }
512 }
513 Ok(())
514}
515
516fn within_budget(observation: &ReplicaObservation, budget: StalenessBudget) -> bool {
517 match budget {
518 StalenessBudget::None => true,
519 StalenessBudget::MaxSequenceLag(maximum) => observation.sequence_lag() <= maximum,
520 StalenessBudget::MaxDuration(maximum) => observation.approximate_time_lag <= maximum,
521 }
522}
523
524#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
526pub struct RegionalRouterConfig {
527 pub local_region: RegionId,
528 pub replica_wait: Duration,
529 pub allow_authority_fallback: bool,
530}
531
532impl RegionalRouterConfig {
533 pub fn validate(self) -> Result<(), RegionError> {
539 if self.replica_wait > Duration::from_secs(30) {
540 return Err(RegionError::UnboundedReplicaWait);
541 }
542 Ok(())
543 }
544}
545
546#[derive(Clone, Debug)]
548pub struct RegionalRouter {
549 authority: AuthorityLocation,
550 authority_available: bool,
551 config: RegionalRouterConfig,
552 replicas: BTreeMap<ReplicaId, ReplicaObservation>,
553}
554
555impl RegionalRouter {
556 pub fn new(
562 authority: AuthorityLocation,
563 authority_available: bool,
564 config: RegionalRouterConfig,
565 ) -> Result<Self, RegionError> {
566 authority.validate()?;
567 config.validate()?;
568 Ok(Self {
569 authority,
570 authority_available,
571 config,
572 replicas: BTreeMap::new(),
573 })
574 }
575
576 pub fn observe(&mut self, observation: ReplicaObservation) -> Result<(), RegionError> {
582 observation.validate()?;
583 self.replicas.insert(observation.replica_id, observation);
584 Ok(())
585 }
586
587 pub fn remove(&mut self, replica_id: ReplicaId) -> Option<ReplicaObservation> {
589 self.replicas.remove(&replica_id)
590 }
591
592 pub fn update_authority(
599 &mut self,
600 authority: AuthorityLocation,
601 available: bool,
602 ) -> Result<(), RegionError> {
603 authority.validate()?;
604 if authority.authority_id != self.authority.authority_id {
605 return Err(RegionError::AuthorityMismatch);
606 }
607 if authority.authority_epoch < self.authority.authority_epoch {
608 return Err(RegionError::EpochRollback {
609 trusted: self.authority.authority_epoch,
610 received: authority.authority_epoch,
611 });
612 }
613 self.authority = authority;
614 self.authority_available = available;
615 self.replicas.retain(|_, replica| {
616 replica.watermark.authority_id == self.authority.authority_id
617 && replica.watermark.authority_epoch == self.authority.authority_epoch
618 });
619 Ok(())
620 }
621
622 pub fn route(&self, request: RegionalReadRequest) -> Result<ReadRouteDecision, RegionError> {
628 if request.authority_id != self.authority.authority_id {
629 return Err(RegionError::AuthorityMismatch);
630 }
631 if request.authority_epoch != self.authority.authority_epoch {
632 return Err(RegionError::WrongEpoch {
633 available: self.authority.authority_epoch,
634 required: request.authority_epoch,
635 });
636 }
637 let required = request.required_sequence()?;
638 if request.policy.consistency == ReadConsistency::Authority {
639 return self.authority_decision(ReadConsistency::Authority, false, None);
640 }
641
642 let mut exact = self
643 .replicas
644 .values()
645 .filter(|replica| ReplicaReadGuard::validate(replica, request).is_ok())
646 .collect::<Vec<_>>();
647 sort_replicas(&mut exact, request.region_hint, self.config.local_region);
648 if let Some(replica) = exact.first() {
649 return Ok(ReadRouteDecision {
650 target: ReadTarget::RegionalReplica(replica.replica_id),
651 effective_consistency: request.policy.consistency,
652 downgraded: false,
653 wait_before_fallback: None,
654 });
655 }
656
657 let wait_before_fallback = required.and_then(|required_sequence| {
658 self.stale_wait_candidate(request)
659 .map(|replica_id| ReplicaWait {
660 replica_id,
661 required_sequence,
662 timeout: self.config.replica_wait,
663 })
664 });
665 if request.policy.allow_authority_fallback
666 && self.config.allow_authority_fallback
667 && self.authority_available
668 {
669 return self.authority_decision(
670 request.policy.consistency,
671 false,
672 wait_before_fallback,
673 );
674 }
675
676 if request.policy.fallback != ReadFallbackPolicy::NeverDowngrade {
677 let mut downgraded = self
678 .replicas
679 .values()
680 .filter(|replica| validate_replica_base(replica, request).is_ok())
681 .filter(|replica| {
682 request.policy.fallback == ReadFallbackPolicy::AllowEventual
683 || (request.policy.fallback == ReadFallbackPolicy::AllowBoundedStale
684 && request.policy.staleness_budget != StalenessBudget::None
685 && within_budget(replica, request.policy.staleness_budget))
686 })
687 .collect::<Vec<_>>();
688 sort_replicas(
689 &mut downgraded,
690 request.region_hint,
691 self.config.local_region,
692 );
693 if let Some(replica) = downgraded.first() {
694 return Ok(ReadRouteDecision {
695 target: ReadTarget::RegionalReplica(replica.replica_id),
696 effective_consistency: ReadConsistency::Eventual,
697 downgraded: true,
698 wait_before_fallback: None,
699 });
700 }
701 }
702
703 if let Some(required) = required {
704 let available = self
705 .replicas
706 .values()
707 .filter(|replica| {
708 replica.watermark.authority_id == request.authority_id
709 && replica.watermark.authority_epoch == request.authority_epoch
710 })
711 .map(|replica| replica.watermark.sequence)
712 .max()
713 .unwrap_or(Sequence(0));
714 return Err(RegionError::ReplicaTooStale {
715 available,
716 required,
717 });
718 }
719 Err(if self.authority_available {
720 RegionError::NoEligibleReplica
721 } else {
722 RegionError::AuthorityUnavailable
723 })
724 }
725
726 fn authority_decision(
727 &self,
728 effective_consistency: ReadConsistency,
729 downgraded: bool,
730 wait_before_fallback: Option<ReplicaWait>,
731 ) -> Result<ReadRouteDecision, RegionError> {
732 if !self.authority_available {
733 return Err(RegionError::AuthorityUnavailable);
734 }
735 Ok(ReadRouteDecision {
736 target: ReadTarget::Authority,
737 effective_consistency,
738 downgraded,
739 wait_before_fallback,
740 })
741 }
742
743 fn stale_wait_candidate(&self, request: RegionalReadRequest) -> Option<ReplicaId> {
744 let mut candidates = self
745 .replicas
746 .values()
747 .filter(|replica| validate_replica_base(replica, request).is_ok())
748 .collect::<Vec<_>>();
749 sort_replicas(
750 &mut candidates,
751 request.region_hint,
752 self.config.local_region,
753 );
754 candidates.first().map(|replica| replica.replica_id)
755 }
756}
757
758fn sort_replicas(
759 replicas: &mut Vec<&ReplicaObservation>,
760 region_hint: Option<RegionId>,
761 local_region: RegionId,
762) {
763 replicas.sort_by_key(|replica| {
764 (
765 replica.health,
766 u8::from(region_hint.or(Some(local_region)) != Some(replica.region_id)),
767 replica.priority,
768 replica.load_basis_points,
769 replica.sequence_lag(),
770 replica.replica_id,
771 )
772 });
773}
774
775#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)]
777pub enum RegionalPlacementKind {
778 Authority,
779 Replica,
780 Projection,
781 Snapshot,
782 Blob,
783 Cache,
784 Backup,
785 EncryptionKey,
786}
787
788#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
790pub struct ResidencyPolicy {
791 pub allowed_regions: BTreeSet<RegionId>,
792 pub authority_region: RegionId,
793 pub replica_regions: BTreeSet<RegionId>,
794 pub key_regions: BTreeSet<RegionId>,
795}
796
797impl ResidencyPolicy {
798 pub fn validate(&self) -> Result<(), RegionError> {
804 if self.allowed_regions.is_empty()
805 || !self.allowed_regions.contains(&self.authority_region)
806 || !self.replica_regions.is_subset(&self.allowed_regions)
807 || !self.key_regions.is_subset(&self.allowed_regions)
808 {
809 return Err(RegionError::ResidencyViolation);
810 }
811 Ok(())
812 }
813
814 pub fn verify_placement(
820 &self,
821 kind: RegionalPlacementKind,
822 region: RegionId,
823 ) -> Result<(), RegionError> {
824 self.validate()?;
825 let allowed = self.allowed_regions.contains(®ion)
826 && match kind {
827 RegionalPlacementKind::Authority => region == self.authority_region,
828 RegionalPlacementKind::Replica | RegionalPlacementKind::Projection => {
829 self.replica_regions.contains(®ion)
830 }
831 RegionalPlacementKind::EncryptionKey => self.key_regions.contains(®ion),
832 RegionalPlacementKind::Snapshot
833 | RegionalPlacementKind::Blob
834 | RegionalPlacementKind::Cache
835 | RegionalPlacementKind::Backup => true,
836 };
837 if allowed {
838 Ok(())
839 } else {
840 Err(RegionError::ResidencyViolation)
841 }
842 }
843}
844
845#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)]
847#[serde(transparent)]
848pub struct DirectoryGeneration(pub u64);
849
850#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
852pub struct TenantAuthorityAssignment {
853 pub tenant_id: TenantId,
854 pub shard_id: AuthorityShardId,
855 pub authority_id: AuthorityId,
856 pub authority_epoch: AuthorityEpoch,
857 pub authority_region: RegionId,
858 pub directory_generation: DirectoryGeneration,
859 pub residency: ResidencyPolicy,
860}
861
862#[derive(Clone, Copy, Debug, Eq, PartialEq)]
864pub enum AssignmentTrust {
865 Verified,
866 Unverified,
867}
868
869#[derive(Clone, Debug, Default)]
871pub struct TenantAuthorityDirectory {
872 assignments: BTreeMap<TenantId, TenantAuthorityAssignment>,
873}
874
875impl TenantAuthorityDirectory {
876 pub fn install(
882 &mut self,
883 assignment: TenantAuthorityAssignment,
884 trust: AssignmentTrust,
885 ) -> Result<(), RegionError> {
886 if trust != AssignmentTrust::Verified {
887 return Err(RegionError::UntrustedDirectoryAssignment);
888 }
889 assignment.residency.validate()?;
890 assignment.residency.verify_placement(
891 RegionalPlacementKind::Authority,
892 assignment.authority_region,
893 )?;
894 if let Some(current) = self.assignments.get(&assignment.tenant_id) {
895 if assignment.directory_generation <= current.directory_generation {
896 return Err(RegionError::DirectoryRollback);
897 }
898 if assignment.authority_id == current.authority_id
899 && assignment.authority_epoch < current.authority_epoch
900 {
901 return Err(RegionError::EpochRollback {
902 trusted: current.authority_epoch,
903 received: assignment.authority_epoch,
904 });
905 }
906 }
907 self.assignments.insert(assignment.tenant_id, assignment);
908 Ok(())
909 }
910
911 #[must_use]
913 pub fn resolve(&self, tenant_id: TenantId) -> Option<&TenantAuthorityAssignment> {
914 self.assignments.get(&tenant_id)
915 }
916
917 pub fn route_write(
923 &self,
924 tenant_id: TenantId,
925 location: &AuthorityLocation,
926 role: RegionalRole,
927 ) -> Result<RegionId, RegionError> {
928 let assignment = self
929 .resolve(tenant_id)
930 .ok_or(RegionError::TenantAssignmentMissing)?;
931 if role != RegionalRole::AuthorityWriter {
932 return Err(RegionError::NotAuthorityWriter);
933 }
934 if location.authority_id != assignment.authority_id
935 || location.authority_epoch != assignment.authority_epoch
936 || location.region_id != assignment.authority_region
937 {
938 return Err(RegionError::StaleControlPlane);
939 }
940 assignment
941 .residency
942 .verify_placement(RegionalPlacementKind::Authority, location.region_id)?;
943 Ok(location.region_id)
944 }
945}
946
947#[derive(Clone, Copy, Debug, Eq, PartialEq)]
949pub enum ProjectionApplyDecision {
950 Apply,
951 Duplicate,
952}
953
954pub fn validate_projection_apply(
960 current: ProjectionWatermark,
961 incoming_epoch: AuthorityEpoch,
962 incoming_sequence: Sequence,
963) -> Result<ProjectionApplyDecision, RegionError> {
964 if incoming_epoch != current.authority_epoch {
965 return Err(RegionError::WrongEpoch {
966 available: current.authority_epoch,
967 required: incoming_epoch,
968 });
969 }
970 if incoming_sequence <= current.sequence {
971 return Ok(ProjectionApplyDecision::Duplicate);
972 }
973 if incoming_sequence.0 != current.sequence.0.saturating_add(1) {
974 return Err(RegionError::ProjectionGap {
975 current: current.sequence,
976 incoming: incoming_sequence,
977 });
978 }
979 Ok(ProjectionApplyDecision::Apply)
980}
981
982#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
984pub enum RegionalArtifactKind {
985 SnapshotManifest,
986 SnapshotChunk,
987 Blob,
988}
989
990#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
992pub struct RegionalArtifactEvidence {
993 pub kind: RegionalArtifactKind,
994 pub authority_id: AuthorityId,
995 pub authority_epoch: AuthorityEpoch,
996 pub origin_region: RegionId,
997 pub delivery_region: RegionId,
998 pub authority_signature_verified: bool,
999 pub content_digest_verified: bool,
1000}
1001
1002impl RegionalArtifactEvidence {
1003 pub fn verify(
1009 self,
1010 authority_id: AuthorityId,
1011 authority_epoch: AuthorityEpoch,
1012 residency: &ResidencyPolicy,
1013 ) -> Result<(), RegionError> {
1014 if self.authority_id != authority_id {
1015 return Err(RegionError::AuthorityMismatch);
1016 }
1017 if self.authority_epoch != authority_epoch {
1018 return Err(RegionError::WrongEpoch {
1019 available: self.authority_epoch,
1020 required: authority_epoch,
1021 });
1022 }
1023 if !self.authority_signature_verified || !self.content_digest_verified {
1024 return Err(RegionError::ArtifactIntegrityUnverified);
1025 }
1026 let placement = match self.kind {
1027 RegionalArtifactKind::SnapshotManifest | RegionalArtifactKind::SnapshotChunk => {
1028 RegionalPlacementKind::Snapshot
1029 }
1030 RegionalArtifactKind::Blob => RegionalPlacementKind::Blob,
1031 };
1032 residency.verify_placement(placement, self.origin_region)?;
1033 residency.verify_placement(placement, self.delivery_region)
1034 }
1035}
1036
1037#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)]
1039#[serde(transparent)]
1040pub struct RegionalCopyId(Uuid);
1041
1042impl RegionalCopyId {
1043 #[must_use]
1044 pub fn new() -> Self {
1045 Self(Uuid::now_v7())
1046 }
1047}
1048
1049impl Default for RegionalCopyId {
1050 fn default() -> Self {
1051 Self::new()
1052 }
1053}
1054
1055#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
1057pub enum RegionalCopyKind {
1058 Replica,
1059 Projection,
1060 Snapshot,
1061 Blob,
1062 Cache,
1063 SearchIndex,
1064 Backup,
1065}
1066
1067#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
1069pub enum RegionalPurgeState {
1070 Registered,
1071 Purged,
1072 Verified,
1073 PolicyExempt,
1074}
1075
1076#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
1078pub struct RegionalCopyRecord {
1079 pub copy_id: RegionalCopyId,
1080 pub region_id: RegionId,
1081 pub kind: RegionalCopyKind,
1082 pub required: bool,
1083 pub state: RegionalPurgeState,
1084}
1085
1086#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1088pub struct RegionalGovernanceReport {
1089 pub registered: usize,
1090 pub required: usize,
1091 pub verified: usize,
1092 pub complete: bool,
1093}
1094
1095#[derive(Clone, Debug, Default)]
1097pub struct RegionalGovernanceRegistry {
1098 copies: BTreeMap<RegionalCopyId, RegionalCopyRecord>,
1099}
1100
1101impl RegionalGovernanceRegistry {
1102 pub fn register(&mut self, record: RegionalCopyRecord) {
1104 self.copies.insert(record.copy_id, record);
1105 }
1106
1107 pub fn transition(
1113 &mut self,
1114 copy_id: RegionalCopyId,
1115 next: RegionalPurgeState,
1116 ) -> Result<(), RegionError> {
1117 let record = self
1118 .copies
1119 .get_mut(©_id)
1120 .ok_or(RegionError::RegionalCopyMissing)?;
1121 let allowed = matches!(
1122 (record.state, next),
1123 (
1124 RegionalPurgeState::Registered,
1125 RegionalPurgeState::Purged | RegionalPurgeState::PolicyExempt
1126 ) | (RegionalPurgeState::Purged, RegionalPurgeState::Verified)
1127 ) || record.state == next;
1128 if !allowed {
1129 return Err(RegionError::InvalidPurgeTransition);
1130 }
1131 record.state = next;
1132 Ok(())
1133 }
1134
1135 #[must_use]
1137 pub fn report(&self) -> RegionalGovernanceReport {
1138 let required = self
1139 .copies
1140 .values()
1141 .filter(|record| record.required)
1142 .count();
1143 let verified = self
1144 .copies
1145 .values()
1146 .filter(|record| {
1147 record.required
1148 && matches!(
1149 record.state,
1150 RegionalPurgeState::Verified | RegionalPurgeState::PolicyExempt
1151 )
1152 })
1153 .count();
1154 RegionalGovernanceReport {
1155 registered: self.copies.len(),
1156 required,
1157 verified,
1158 complete: required == verified,
1159 }
1160 }
1161}
1162
1163#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
1165pub enum RegionalDeploymentProfile {
1166 SingleRegion,
1167 DualRegion,
1168 GlobalRead,
1169 TenantShardedGlobal,
1170}
1171
1172#[derive(Clone, Copy, Debug, Eq, PartialEq)]
1174pub enum RegionalEventKind {
1175 ReadRoutedReplica,
1176 ReadFallbackAuthority,
1177 ReadDowngraded,
1178 ReplicaTooStale,
1179 WrongEpochReplica,
1180 RegionDegraded,
1181 AuthorityRegionChanged,
1182 ResidencyViolation,
1183 ProjectionStalled,
1184 GovernanceIncomplete,
1185}
1186
1187#[derive(Clone, Copy, Debug, Error, Eq, PartialEq)]
1189pub enum RegionError {
1190 #[error("authority identity does not match the trusted routing domain")]
1191 AuthorityMismatch,
1192 #[error("received authority epoch {received:?} is below trusted epoch {trusted:?}")]
1193 EpochRollback {
1194 trusted: AuthorityEpoch,
1195 received: AuthorityEpoch,
1196 },
1197 #[error("replica epoch {available:?} does not match required epoch {required:?}")]
1198 WrongEpoch {
1199 available: AuthorityEpoch,
1200 required: AuthorityEpoch,
1201 },
1202 #[error("session consistency requires a session watermark")]
1203 MissingSessionWatermark,
1204 #[error("replica is visible through {available:?}, below required {required:?}")]
1205 ReplicaTooStale {
1206 available: Sequence,
1207 required: Sequence,
1208 },
1209 #[error("authorization watermark {available:?} is below required {required:?}")]
1210 AuthorizationTooStale {
1211 available: Option<Sequence>,
1212 required: Sequence,
1213 },
1214 #[error("replica staleness exceeds the endpoint budget")]
1215 StalenessBudgetExceeded,
1216 #[error("regional node is not a read replica")]
1217 NotReadReplica,
1218 #[error("regional node is not the active authority writer")]
1219 NotAuthorityWriter,
1220 #[error("replica is unavailable, in maintenance, or a promotion candidate")]
1221 ReplicaUnavailable,
1222 #[error("no regional replica satisfies the request")]
1223 NoEligibleReplica,
1224 #[error("authority writer is unavailable")]
1225 AuthorityUnavailable,
1226 #[error("projection is unavailable")]
1227 ProjectionUnavailable,
1228 #[error("projection timeline or schema is incompatible")]
1229 ProjectionIncompatible,
1230 #[error("projection apply has a gap from {current:?} to {incoming:?}")]
1231 ProjectionGap {
1232 current: Sequence,
1233 incoming: Sequence,
1234 },
1235 #[error("replica load must be between zero and 10000 basis points")]
1236 InvalidLoad,
1237 #[error("replica wait exceeds the bounded maximum")]
1238 UnboundedReplicaWait,
1239 #[error("writer endpoint metadata is empty or too large")]
1240 InvalidEndpoint,
1241 #[error("regional placement violates tenant residency policy")]
1242 ResidencyViolation,
1243 #[error("tenant authority assignment has not been trust verified")]
1244 UntrustedDirectoryAssignment,
1245 #[error("tenant directory generation did not advance")]
1246 DirectoryRollback,
1247 #[error("tenant has no trusted authority assignment")]
1248 TenantAssignmentMissing,
1249 #[error("routing control plane is stale relative to the trusted tenant assignment")]
1250 StaleControlPlane,
1251 #[error("regional artifact authority or integrity evidence is incomplete")]
1252 ArtifactIntegrityUnverified,
1253 #[error("regional governed copy is not registered")]
1254 RegionalCopyMissing,
1255 #[error("regional purge transition skipped verification or rolled back")]
1256 InvalidPurgeTransition,
1257}
1258
1259#[cfg(test)]
1260mod tests {
1261 use super::*;
1262
1263 fn authority(epoch: AuthorityEpoch, latest: u64) -> AuthorityLocation {
1264 AuthorityLocation {
1265 region_id: RegionId::new(1),
1266 endpoint: "https://writer.internal".to_owned(),
1267 authority_id: AuthorityId::LOCAL_DEVELOPMENT,
1268 authority_epoch: epoch,
1269 shard_id: Some(AuthorityShardId(7)),
1270 latest_sequence: Sequence(latest),
1271 }
1272 }
1273
1274 fn replica(region: u16, epoch: AuthorityEpoch, sequence: u64) -> ReplicaObservation {
1275 ReplicaObservation {
1276 replica_id: ReplicaId::new(),
1277 region_id: RegionId::new(region),
1278 role: RegionalRole::ReadReplica,
1279 watermark: ReplicaWatermark {
1280 authority_id: AuthorityId::LOCAL_DEVELOPMENT,
1281 authority_epoch: epoch,
1282 sequence: Sequence(sequence),
1283 },
1284 projection: None,
1285 authorization_sequence: Some(Sequence(sequence)),
1286 observed_authority_sequence: Sequence(10),
1287 approximate_time_lag: Duration::from_millis(50),
1288 health: RegionHealth::Healthy,
1289 mode: RegionMode::Healthy,
1290 priority: 1,
1291 load_basis_points: 100,
1292 maintenance: false,
1293 }
1294 }
1295
1296 fn request(
1297 consistency: ReadConsistency,
1298 session: Option<SessionWatermark>,
1299 ) -> RegionalReadRequest {
1300 RegionalReadRequest {
1301 authority_id: AuthorityId::LOCAL_DEVELOPMENT,
1302 authority_epoch: AuthorityEpoch::INITIAL,
1303 policy: EndpointReadPolicy {
1304 consistency,
1305 staleness_budget: StalenessBudget::None,
1306 fallback: ReadFallbackPolicy::NeverDowngrade,
1307 allow_authority_fallback: true,
1308 authorization_sensitive: false,
1309 },
1310 session,
1311 region_hint: Some(RegionId::new(2)),
1312 required_projection: None,
1313 minimum_authorization_sequence: None,
1314 }
1315 }
1316
1317 fn router(authority_available: bool) -> RegionalRouter {
1318 RegionalRouter::new(
1319 authority(AuthorityEpoch::INITIAL, 10),
1320 authority_available,
1321 RegionalRouterConfig {
1322 local_region: RegionId::new(2),
1323 replica_wait: Duration::from_millis(200),
1324 allow_authority_fallback: true,
1325 },
1326 )
1327 .unwrap_or_else(|error| panic!("{error}"))
1328 }
1329
1330 #[test]
1331 fn read_your_writes_falls_back_instead_of_returning_old_state() {
1332 let mut router = router(true);
1333 let stale = replica(2, AuthorityEpoch::INITIAL, 8);
1334 let stale_id = stale.replica_id;
1335 router
1336 .observe(stale)
1337 .unwrap_or_else(|error| panic!("{error}"));
1338 let session = SessionWatermark::new(
1339 AuthorityId::LOCAL_DEVELOPMENT,
1340 AuthorityEpoch::INITIAL,
1341 Sequence(10),
1342 );
1343
1344 let decision = router
1345 .route(request(ReadConsistency::Session, Some(session)))
1346 .unwrap_or_else(|error| panic!("{error}"));
1347 assert_eq!(decision.target, ReadTarget::Authority);
1348 assert_eq!(
1349 decision.wait_before_fallback.map(|wait| wait.replica_id),
1350 Some(stale_id)
1351 );
1352 assert!(!decision.downgraded);
1353 }
1354
1355 #[test]
1356 fn caught_up_replica_serves_at_least_read() {
1357 let mut router = router(true);
1358 let caught_up = replica(2, AuthorityEpoch::INITIAL, 10);
1359 let replica_id = caught_up.replica_id;
1360 router
1361 .observe(caught_up)
1362 .unwrap_or_else(|error| panic!("{error}"));
1363 let decision = router
1364 .route(request(ReadConsistency::AtLeast(Sequence(10)), None))
1365 .unwrap_or_else(|error| panic!("{error}"));
1366 assert_eq!(decision.target, ReadTarget::RegionalReplica(replica_id));
1367 }
1368
1369 #[test]
1370 fn wrong_epoch_replica_is_never_routed_as_current() {
1371 let next = AuthorityEpoch::INITIAL
1372 .checked_next()
1373 .unwrap_or(AuthorityEpoch::INITIAL);
1374 let mut router = router(false);
1375 router
1376 .observe(replica(2, next, 10))
1377 .unwrap_or_else(|error| panic!("{error}"));
1378 assert!(matches!(
1379 router.route(request(ReadConsistency::AtLeast(Sequence(10)), None)),
1380 Err(RegionError::ReplicaTooStale { .. })
1381 ));
1382 }
1383
1384 #[test]
1385 fn explicit_downgrade_is_visible_and_policy_bound() {
1386 let mut router = router(false);
1387 let stale = replica(2, AuthorityEpoch::INITIAL, 8);
1388 let id = stale.replica_id;
1389 router
1390 .observe(stale)
1391 .unwrap_or_else(|error| panic!("{error}"));
1392 let mut read = request(ReadConsistency::AtLeast(Sequence(10)), None);
1393 read.policy.fallback = ReadFallbackPolicy::AllowEventual;
1394 let decision = router.route(read).unwrap_or_else(|error| panic!("{error}"));
1395 assert_eq!(decision.target, ReadTarget::RegionalReplica(id));
1396 assert!(decision.downgraded);
1397 assert_eq!(decision.effective_consistency, ReadConsistency::Eventual);
1398 }
1399
1400 #[test]
1401 fn epoch_transition_evicts_old_replica_and_cache() {
1402 let next = AuthorityEpoch::INITIAL
1403 .checked_next()
1404 .unwrap_or(AuthorityEpoch::INITIAL);
1405 let mut router = router(true);
1406 let old = replica(2, AuthorityEpoch::INITIAL, 10);
1407 router
1408 .observe(old)
1409 .unwrap_or_else(|error| panic!("{error}"));
1410 router
1411 .update_authority(authority(next, 1), true)
1412 .unwrap_or_else(|error| panic!("{error}"));
1413
1414 let mut read = request(ReadConsistency::Eventual, None);
1415 read.authority_epoch = next;
1416 assert_eq!(
1417 router
1418 .route(read)
1419 .unwrap_or_else(|error| panic!("{error}"))
1420 .target,
1421 ReadTarget::Authority
1422 );
1423 let cache = RegionalCacheMetadata {
1424 authority_id: AuthorityId::LOCAL_DEVELOPMENT,
1425 authority_epoch: AuthorityEpoch::INITIAL,
1426 served_sequence: Sequence(10),
1427 projection_version: None,
1428 };
1429 assert!(!cache.satisfies(read).unwrap_or(false));
1430 }
1431
1432 #[test]
1433 fn writes_route_only_to_trusted_resident_authority() {
1434 let tenant = TenantId::new();
1435 let allowed = BTreeSet::from([RegionId::new(1), RegionId::new(2)]);
1436 let residency = ResidencyPolicy {
1437 allowed_regions: allowed,
1438 authority_region: RegionId::new(1),
1439 replica_regions: BTreeSet::from([RegionId::new(2)]),
1440 key_regions: BTreeSet::from([RegionId::new(1)]),
1441 };
1442 let mut directory = TenantAuthorityDirectory::default();
1443 directory
1444 .install(
1445 TenantAuthorityAssignment {
1446 tenant_id: tenant,
1447 shard_id: AuthorityShardId(7),
1448 authority_id: AuthorityId::LOCAL_DEVELOPMENT,
1449 authority_epoch: AuthorityEpoch::INITIAL,
1450 authority_region: RegionId::new(1),
1451 directory_generation: DirectoryGeneration(1),
1452 residency,
1453 },
1454 AssignmentTrust::Verified,
1455 )
1456 .unwrap_or_else(|error| panic!("{error}"));
1457 assert_eq!(
1458 directory
1459 .route_write(
1460 tenant,
1461 &authority(AuthorityEpoch::INITIAL, 10),
1462 RegionalRole::AuthorityWriter
1463 )
1464 .unwrap_or_else(|error| panic!("{error}")),
1465 RegionId::new(1)
1466 );
1467 assert_eq!(
1468 directory.route_write(
1469 tenant,
1470 &authority(AuthorityEpoch::INITIAL, 10),
1471 RegionalRole::ReadReplica,
1472 ),
1473 Err(RegionError::NotAuthorityWriter)
1474 );
1475 }
1476
1477 #[test]
1478 fn governance_requires_every_registered_required_copy() {
1479 let copy = RegionalCopyId::new();
1480 let mut registry = RegionalGovernanceRegistry::default();
1481 registry.register(RegionalCopyRecord {
1482 copy_id: copy,
1483 region_id: RegionId::new(2),
1484 kind: RegionalCopyKind::Projection,
1485 required: true,
1486 state: RegionalPurgeState::Registered,
1487 });
1488 assert!(!registry.report().complete);
1489 registry
1490 .transition(copy, RegionalPurgeState::Purged)
1491 .and_then(|()| registry.transition(copy, RegionalPurgeState::Verified))
1492 .unwrap_or_else(|error| panic!("{error}"));
1493 assert!(registry.report().complete);
1494 }
1495
1496 #[test]
1497 fn edge_artifact_requires_integrity_epoch_and_residency() {
1498 let residency = ResidencyPolicy {
1499 allowed_regions: BTreeSet::from([RegionId::new(1), RegionId::new(2)]),
1500 authority_region: RegionId::new(1),
1501 replica_regions: BTreeSet::from([RegionId::new(2)]),
1502 key_regions: BTreeSet::from([RegionId::new(1)]),
1503 };
1504 let evidence = RegionalArtifactEvidence {
1505 kind: RegionalArtifactKind::SnapshotChunk,
1506 authority_id: AuthorityId::LOCAL_DEVELOPMENT,
1507 authority_epoch: AuthorityEpoch::INITIAL,
1508 origin_region: RegionId::new(1),
1509 delivery_region: RegionId::new(2),
1510 authority_signature_verified: true,
1511 content_digest_verified: true,
1512 };
1513 evidence
1514 .verify(
1515 AuthorityId::LOCAL_DEVELOPMENT,
1516 AuthorityEpoch::INITIAL,
1517 &residency,
1518 )
1519 .unwrap_or_else(|error| panic!("{error}"));
1520 }
1521}