1pub mod admin;
5
6pub mod diagnostics;
8
9pub mod jobs;
11mod sdk;
12
13pub use sdk::{
14 AequoraServer, AequoraServerBuilder, DomainRegistrationError, DomainRegistry,
15 DomainRegistryBuilder, ServerOperationOutcome, StableServerError,
16};
17
18pub mod prelude {
20 pub use crate::admin::{AdminControlPlane, AdminPlaneConfig};
21 pub use crate::diagnostics::{DiagnosticAccessPolicy, ServerDiagnosticCollector};
22 pub use crate::jobs::{
23 JobAccessPolicy, JobAdminService, JobWorkerConfig, JobWorkerPlanner, WorkerShutdown,
24 };
25 pub use crate::{
26 AdmittedExchangeService, AequoraServer, AequoraServerBuilder, DomainRegistrationError,
27 DomainRegistry, DomainRegistryBuilder, ExchangeService, ServerBuildError,
28 ServerCommandOutcome, ServerConfig, ServerError, ServerOperationOutcome,
29 ServerRegionalReadGuard, StableServerError, SyncServer, SyncServerBuilder,
30 };
31 pub use aequora_adapter_sdk::{
32 Authenticator, AuthorityService, DomainContext, DomainHandler, DomainOutcome,
33 };
34 pub use aequora_admission::{AdmissionController, AdmissionPolicy, HierarchicalAdmission};
35 pub use aequora_authority::{
36 AuthorityController, AuthorityPromotionPolicy, AuthorityRole, AuthorityRuntimeMode,
37 AuthorityState,
38 };
39 pub use aequora_clock::{Clock, SystemClock};
40 pub use aequora_conflict::{ConflictResolver, RejectConflicts};
41 pub use aequora_executor::{
42 AuthContext, DerivedEventProvenance, DomainOperation, JobProvenance, OperationExecutor,
43 OperationHandler, OperationRegistry, ScopeAuthorizer, TrustedProvenance,
44 };
45 pub use aequora_jobs::{JobRegistry, JobStore};
46 pub use aequora_region::{
47 ReadResponseMetadata, RegionError, RegionalReadRequest, ReplicaObservation,
48 };
49 pub use aequora_side_effects::{
50 AuthoritativeOperationSink, SideEffectProvider, SideEffectStore,
51 };
52 pub use aequora_store::{
53 AdapterCapabilities, AdapterManifest, AdapterManifestProvider, AdapterRequirements,
54 AdapterRole, AdapterTier, AuthoritativeStore, ProductionAdapterPair,
55 };
56 pub use aequora_workflow::{WorkflowDefinition, WorkflowStore};
57}
58
59use aequora_admission::{
60 AdmissionController, AdmissionRejection, CostUnits, RequestShape, ResourceDomain,
61 ServerPriorityPolicy, WorkDescriptor,
62};
63use aequora_authority::{
64 AuthorityController, AuthorityError, AuthorityPromotionPolicy, AuthorityRole,
65 AuthorityRuntimeMode, AuthorityState, CursorDisposition, validate_cursor,
66};
67use aequora_clock::Clock;
68use aequora_compute::{ComputeError, ComputePool};
69use aequora_conflict::{
70 ConflictResolver, MergeDecision, MergeError, MergeInput, VersionCheck, check_version,
71};
72use aequora_executor::{
73 AuthContext, DependencyError, DependencyPlan, ExecutionError, IncomingOperation,
74 OperationExecutor, plan_dependencies,
75};
76use aequora_observability::{
77 MetricEvent, NoopObserver, Observer, OutcomeKind, ServerPhaseKind, TraceContext,
78 TransactionOutcomeKind,
79};
80use aequora_performance::PerformancePolicy;
81use aequora_protocol::{
82 BootstrapRequest, BootstrapResponse, ClientLimits, Conflict, ConflictPolicy, OperationAck,
83 OperationEnvelope, OperationRejection, RejectionCode, ResyncReason, SessionMetadata,
84 SyncDirective, SyncRequest, SyncResponse,
85};
86use aequora_region::{
87 ReadResponseMetadata, RegionError, RegionalReadRequest, ReplicaObservation, ReplicaReadGuard,
88};
89use aequora_store::{
90 AdapterCompatibilityError, AdapterManifestProvider, AdapterRequirements, AuthoritativeStore,
91 CommitOperation, CommitOutcome, StoreError, StoreErrorKind,
92};
93use aequora_types::{
94 AuthorityId, AuthorityInstanceId, EntityVersion, EventId, OperationId, RequestId, Sequence,
95 SessionId, SyncScopeId,
96};
97use aequora_validator::{
98 ProtocolLimits, ValidationError, validate_bootstrap_request, validate_request,
99};
100use async_trait::async_trait;
101use rayon::prelude::*;
102use std::{
103 collections::HashSet,
104 sync::{
105 Arc,
106 atomic::{AtomicU8, Ordering},
107 },
108 time::Instant,
109};
110use thiserror::Error;
111
112#[derive(Clone, Debug)]
117pub struct ServerRegionalReadGuard {
118 observation: ReplicaObservation,
119}
120
121impl ServerRegionalReadGuard {
122 #[must_use]
124 pub const fn new(observation: ReplicaObservation) -> Self {
125 Self { observation }
126 }
127
128 pub fn update(&mut self, observation: ReplicaObservation) {
130 self.observation = observation;
131 }
132
133 pub fn authorize(
139 &self,
140 request: RegionalReadRequest,
141 ) -> Result<ReadResponseMetadata, RegionError> {
142 ReplicaReadGuard::validate(&self.observation, request)
143 }
144}
145
146#[derive(Clone, Copy, Debug)]
148pub struct ServerConfig {
149 pub limits: ProtocolLimits,
151 pub max_pull_changes: usize,
153 pub performance: PerformancePolicy,
155}
156
157impl Default for ServerConfig {
158 fn default() -> Self {
159 Self {
160 limits: ProtocolLimits::default(),
161 max_pull_changes: 1_024,
162 performance: PerformancePolicy::default(),
163 }
164 }
165}
166
167#[derive(Debug, Error)]
169pub enum ServerError {
170 #[error("sync admission rejected work: {0}")]
172 Admission(#[from] AdmissionRejection),
173 #[error("invalid sync request: {0}")]
175 Validation(#[from] ValidationError),
176 #[error("sync session identity does not match authenticated identity")]
178 IdentityMismatch,
179 #[error("sync persistence failed: {0}")]
181 Store(#[from] StoreError),
182 #[error("entity version overflow")]
184 VersionOverflow,
185 #[error("invalid operation dependency graph: {0}")]
187 Dependency(#[from] DependencyError),
188 #[error("sync scope is not authorized: {0}")]
190 ScopeAuthorization(ExecutionError),
191 #[error("snapshot page limits are too small to make progress")]
193 SnapshotNoProgress,
194 #[error("snapshot bootstrap is not available")]
196 BootstrapUnavailable,
197 #[error("sync compute work failed: {0}")]
199 Compute(#[from] ComputeError),
200 #[error("client response limit is too small to make synchronization progress")]
202 ResponseLimit,
203 #[error("sync response encoding failed: {0}")]
205 Codec(#[from] aequora_codec::CodecError),
206 #[error("{domain} memory use {actual} exceeds configured limit {maximum}")]
208 MemoryBudget {
209 domain: &'static str,
210 actual: usize,
211 maximum: usize,
212 },
213 #[error("sync conflict merge failed: {0}")]
215 Merge(#[from] MergeError),
216 #[error("sync is unavailable while the server is in {mode:?} maintenance mode")]
218 Maintenance {
219 mode: MaintenanceMode,
221 retry_after_seconds: u64,
223 },
224 #[error("authority rejected synchronization: {0}")]
226 Authority(#[from] AuthorityError),
227}
228
229#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
231#[repr(u8)]
232pub enum MaintenanceMode {
233 #[default]
235 Normal = 0,
236 ReadOnly = 1,
238 SyncPaused = 2,
240}
241
242impl MaintenanceMode {
243 #[must_use]
245 pub const fn allows_exchange(self, has_operations: bool) -> bool {
246 match self {
247 Self::Normal => true,
248 Self::ReadOnly => !has_operations,
249 Self::SyncPaused => false,
250 }
251 }
252
253 #[must_use]
255 pub const fn allows_bootstrap(self) -> bool {
256 matches!(self, Self::Normal | Self::ReadOnly)
257 }
258}
259
260#[derive(Clone, Default)]
262pub struct MaintenanceController {
263 mode: Arc<AtomicU8>,
264}
265
266impl MaintenanceController {
267 #[must_use]
269 pub fn mode(&self) -> MaintenanceMode {
270 match self.mode.load(Ordering::Acquire) {
271 1 => MaintenanceMode::ReadOnly,
272 2 => MaintenanceMode::SyncPaused,
273 _ => MaintenanceMode::Normal,
274 }
275 }
276
277 #[must_use]
279 pub fn set_mode(&self, mode: MaintenanceMode) -> MaintenanceMode {
280 let previous = self.mode.swap(mode as u8, Ordering::AcqRel);
281 match previous {
282 1 => MaintenanceMode::ReadOnly,
283 2 => MaintenanceMode::SyncPaused,
284 _ => MaintenanceMode::Normal,
285 }
286 }
287}
288
289pub struct MaintenanceService<S: ?Sized> {
291 inner: Arc<S>,
292 controller: MaintenanceController,
293 retry_after_seconds: u64,
294}
295
296impl<S: ?Sized> MaintenanceService<S> {
297 #[must_use]
299 pub fn new(inner: Arc<S>, controller: MaintenanceController, retry_after_seconds: u64) -> Self {
300 Self {
301 inner,
302 controller,
303 retry_after_seconds: retry_after_seconds.max(1),
304 }
305 }
306
307 #[must_use]
309 pub const fn controller(&self) -> &MaintenanceController {
310 &self.controller
311 }
312
313 fn rejection(&self, mode: MaintenanceMode) -> ServerError {
314 ServerError::Maintenance {
315 mode,
316 retry_after_seconds: self.retry_after_seconds,
317 }
318 }
319}
320
321#[async_trait]
322impl<S> ExchangeService for MaintenanceService<S>
323where
324 S: ExchangeService + ?Sized,
325{
326 async fn exchange(
327 &self,
328 auth: AuthContext,
329 request: SyncRequest,
330 ) -> Result<SyncResponse, ServerError> {
331 let mode = self.controller.mode();
332 if mode.allows_exchange(!request.operations.is_empty()) {
333 self.inner.exchange(auth, request).await
334 } else {
335 Err(self.rejection(mode))
336 }
337 }
338
339 async fn bootstrap(
340 &self,
341 auth: AuthContext,
342 request: BootstrapRequest,
343 ) -> Result<BootstrapResponse, ServerError> {
344 let mode = self.controller.mode();
345 if mode.allows_bootstrap() {
346 self.inner.bootstrap(auth, request).await
347 } else {
348 Err(self.rejection(mode))
349 }
350 }
351}
352
353#[derive(Clone, Debug, Eq, PartialEq)]
355pub enum ServerCommandOutcome {
356 Acknowledged(OperationAck),
358 Rejected(OperationRejection),
360 Conflict(Conflict),
362}
363
364#[async_trait]
366pub trait ExchangeService: Send + Sync {
367 async fn exchange(
369 &self,
370 auth: AuthContext,
371 request: SyncRequest,
372 ) -> Result<SyncResponse, ServerError>;
373
374 async fn bootstrap(
376 &self,
377 _auth: AuthContext,
378 _request: BootstrapRequest,
379 ) -> Result<BootstrapResponse, ServerError> {
380 Err(ServerError::BootstrapUnavailable)
381 }
382}
383
384pub struct AdmittedExchangeService<S: ?Sized> {
386 inner: Arc<S>,
387 admission: Arc<dyn AdmissionController>,
388 priority: ServerPriorityPolicy,
389 tenant_weight: u16,
390}
391
392impl<S: ?Sized> AdmittedExchangeService<S> {
393 #[must_use]
395 pub fn new(inner: Arc<S>, admission: Arc<dyn AdmissionController>) -> Self {
396 Self {
397 inner,
398 admission,
399 priority: ServerPriorityPolicy::default(),
400 tenant_weight: 1,
401 }
402 }
403
404 #[must_use]
406 pub fn with_priority_policy(mut self, priority: ServerPriorityPolicy) -> Self {
407 self.priority = priority;
408 self
409 }
410
411 #[must_use]
413 pub fn with_tenant_weight(mut self, tenant_weight: u16) -> Self {
414 self.tenant_weight = tenant_weight.max(1);
415 self
416 }
417
418 fn exchange_work(&self, auth: &AuthContext, request: &SyncRequest) -> WorkDescriptor {
419 let operation_count = request.operations.len();
420 let dependency_edges = request
421 .operations
422 .iter()
423 .map(|operation| operation.metadata.dependencies.len())
424 .sum::<usize>();
425 let payload_bytes = request.operations.iter().fold(0u64, |total, operation| {
426 total.saturating_add(u64::try_from(operation.payload.len()).unwrap_or(u64::MAX))
427 });
428 let kind = if request.operations.is_empty() {
429 aequora_scheduler::WorkKind::PullChanges
430 } else {
431 aequora_scheduler::WorkKind::PushOperations
432 };
433 let class = self.priority.classify(kind, None);
434 let mut resources = vec![
435 ResourceDomain::SyncExchange,
436 ResourceDomain::DatabaseTransaction,
437 ];
438 if !request.operations.is_empty() {
439 resources.push(ResourceDomain::InteractiveCpu);
440 }
441 WorkDescriptor {
442 tenant_id: auth.tenant_id,
443 tenant_weight: self.tenant_weight,
444 kind,
445 class,
446 cost: CostUnits::new(
447 u32::try_from(
448 operation_count
449 .saturating_add(dependency_edges)
450 .saturating_add(1),
451 )
452 .unwrap_or(u32::MAX),
453 ),
454 shape: RequestShape {
455 operations: operation_count,
456 encoded_bytes: payload_bytes,
457 decompressed_bytes: payload_bytes,
458 scopes: request.session.partitions.len(),
459 dependency_edges,
460 dependency_depth: request
461 .operations
462 .iter()
463 .map(|operation| operation.metadata.dependencies.len())
464 .max()
465 .unwrap_or(0),
466 },
467 scope: Some(request.session.scope_id),
468 estimated_response_bytes: u64::from(request.limits.max_response_bytes),
469 resources,
470 }
471 }
472
473 fn bootstrap_work(&self, auth: &AuthContext, request: &BootstrapRequest) -> WorkDescriptor {
474 let kind = aequora_scheduler::WorkKind::Bootstrap;
475 WorkDescriptor {
476 tenant_id: auth.tenant_id,
477 tenant_weight: self.tenant_weight,
478 kind,
479 class: self.priority.classify(kind, None),
480 cost: CostUnits::new(request.limits.max_entities.max(1)),
481 shape: RequestShape {
482 scopes: request.session.partitions.len(),
483 ..RequestShape::default()
484 },
485 scope: Some(request.session.scope_id),
486 estimated_response_bytes: u64::from(request.limits.max_payload_bytes),
487 resources: vec![
488 ResourceDomain::SyncExchange,
489 ResourceDomain::DatabaseTransaction,
490 ResourceDomain::SnapshotDownload,
491 ],
492 }
493 }
494}
495
496#[async_trait]
497impl<S> ExchangeService for AdmittedExchangeService<S>
498where
499 S: ExchangeService + ?Sized,
500{
501 async fn exchange(
502 &self,
503 auth: AuthContext,
504 request: SyncRequest,
505 ) -> Result<SyncResponse, ServerError> {
506 let work = self.exchange_work(&auth, &request);
507 let _permit = self.admission.admit(&work).await?;
508 self.inner.exchange(auth, request).await
509 }
510
511 async fn bootstrap(
512 &self,
513 auth: AuthContext,
514 request: BootstrapRequest,
515 ) -> Result<BootstrapResponse, ServerError> {
516 let work = self.bootstrap_work(&auth, &request);
517 let _permit = self.admission.admit(&work).await?;
518 self.inner.bootstrap(auth, request).await
519 }
520}
521
522pub struct SyncServer<S, E, R, C> {
524 store: Arc<S>,
525 executor: Arc<E>,
526 conflicts: Arc<R>,
527 clock: Arc<C>,
528 config: ServerConfig,
529 compute: Option<Arc<ComputePool>>,
530 observer: Arc<dyn Observer>,
531 authority: AuthorityController,
532}
533
534#[doc(hidden)]
536#[derive(Clone, Copy, Debug, Default)]
537pub struct MissingServerStore;
538
539#[doc(hidden)]
541#[derive(Clone, Copy, Debug, Default)]
542pub struct MissingServerExecutor;
543
544#[doc(hidden)]
546#[derive(Clone, Copy, Debug, Default)]
547pub struct MissingServerConflicts;
548
549#[doc(hidden)]
551#[derive(Clone, Copy, Debug, Default)]
552pub struct MissingServerClock;
553
554pub struct SyncServerBuilder<
556 S = MissingServerStore,
557 E = MissingServerExecutor,
558 R = MissingServerConflicts,
559 C = MissingServerClock,
560> {
561 store: S,
562 executor: E,
563 conflicts: R,
564 clock: C,
565 config: ServerConfig,
566 compute: Option<Arc<ComputePool>>,
567 observer: Arc<dyn Observer>,
568 authority: AuthorityController,
569 authority_explicit: bool,
570}
571
572#[derive(Clone, Copy, Debug, Error, Eq, PartialEq)]
574pub enum ServerBuildError {
575 #[error(transparent)]
577 Adapter(#[from] AdapterCompatibilityError),
578 #[error("production server requires an explicit authority controller")]
580 AuthorityRequired,
581}
582
583impl Default for SyncServerBuilder {
584 fn default() -> Self {
585 Self {
586 store: MissingServerStore,
587 executor: MissingServerExecutor,
588 conflicts: MissingServerConflicts,
589 clock: MissingServerClock,
590 config: ServerConfig::default(),
591 compute: None,
592 observer: Arc::new(NoopObserver),
593 authority: development_authority(),
594 authority_explicit: false,
595 }
596 }
597}
598
599impl SyncServerBuilder {
600 #[must_use]
602 pub fn new() -> Self {
603 Self::default()
604 }
605}
606
607impl<S, E, R, C> SyncServerBuilder<S, E, R, C> {
608 #[must_use]
610 pub fn store<S2>(self, store: Arc<S2>) -> SyncServerBuilder<Arc<S2>, E, R, C> {
611 SyncServerBuilder {
612 store,
613 executor: self.executor,
614 conflicts: self.conflicts,
615 clock: self.clock,
616 config: self.config,
617 compute: self.compute,
618 observer: self.observer,
619 authority: self.authority,
620 authority_explicit: self.authority_explicit,
621 }
622 }
623
624 #[must_use]
626 pub fn executor<E2>(self, executor: Arc<E2>) -> SyncServerBuilder<S, Arc<E2>, R, C> {
627 SyncServerBuilder {
628 store: self.store,
629 executor,
630 conflicts: self.conflicts,
631 clock: self.clock,
632 config: self.config,
633 compute: self.compute,
634 observer: self.observer,
635 authority: self.authority,
636 authority_explicit: self.authority_explicit,
637 }
638 }
639
640 #[must_use]
642 pub fn conflicts<R2>(self, conflicts: Arc<R2>) -> SyncServerBuilder<S, E, Arc<R2>, C> {
643 SyncServerBuilder {
644 store: self.store,
645 executor: self.executor,
646 conflicts,
647 clock: self.clock,
648 config: self.config,
649 compute: self.compute,
650 observer: self.observer,
651 authority: self.authority,
652 authority_explicit: self.authority_explicit,
653 }
654 }
655
656 #[must_use]
658 pub fn clock<C2>(self, clock: Arc<C2>) -> SyncServerBuilder<S, E, R, Arc<C2>> {
659 SyncServerBuilder {
660 store: self.store,
661 executor: self.executor,
662 conflicts: self.conflicts,
663 clock,
664 config: self.config,
665 compute: self.compute,
666 observer: self.observer,
667 authority: self.authority,
668 authority_explicit: self.authority_explicit,
669 }
670 }
671
672 #[must_use]
674 pub const fn config(mut self, config: ServerConfig) -> Self {
675 self.config = config;
676 self
677 }
678
679 #[must_use]
681 pub fn compute_pool(mut self, compute: Arc<ComputePool>) -> Self {
682 self.compute = Some(compute);
683 self
684 }
685
686 #[must_use]
688 pub fn observer(mut self, observer: Arc<dyn Observer>) -> Self {
689 self.observer = observer;
690 self
691 }
692
693 #[must_use]
695 pub fn authority(mut self, authority: AuthorityController) -> Self {
696 self.authority = authority;
697 self.authority_explicit = true;
698 self
699 }
700}
701
702impl<S, E, R, C> SyncServerBuilder<Arc<S>, Arc<E>, Arc<R>, Arc<C>>
703where
704 S: AuthoritativeStore + 'static,
705 E: OperationExecutor + 'static,
706 R: ConflictResolver + 'static,
707 C: Clock + 'static,
708{
709 #[must_use]
711 pub fn build(self) -> SyncServer<S, E, R, C> {
712 SyncServer {
713 store: self.store,
714 executor: self.executor,
715 conflicts: self.conflicts,
716 clock: self.clock,
717 config: self.config,
718 compute: self.compute,
719 observer: self.observer,
720 authority: self.authority,
721 }
722 }
723}
724
725impl<S, E, R, C> SyncServerBuilder<Arc<S>, Arc<E>, Arc<R>, Arc<C>>
726where
727 S: AuthoritativeStore + AdapterManifestProvider + 'static,
728 E: OperationExecutor + 'static,
729 R: ConflictResolver + 'static,
730 C: Clock + 'static,
731{
732 pub fn build_production(self) -> Result<SyncServer<S, E, R, C>, ServerBuildError> {
740 if !self.authority_explicit {
741 return Err(ServerBuildError::AuthorityRequired);
742 }
743 AdapterRequirements::PRODUCTION_AUTHORITATIVE.verify(self.store.adapter_manifest())?;
744 Ok(self.build())
745 }
746}
747
748impl<S, E, R, C> SyncServer<S, E, R, C> {
749 #[must_use]
751 pub fn new(store: Arc<S>, executor: Arc<E>, conflicts: Arc<R>, clock: Arc<C>) -> Self {
752 Self {
753 store,
754 executor,
755 conflicts,
756 clock,
757 config: ServerConfig::default(),
758 compute: None,
759 observer: Arc::new(NoopObserver),
760 authority: development_authority(),
761 }
762 }
763
764 #[must_use]
766 pub const fn with_config(mut self, config: ServerConfig) -> Self {
767 self.config = config;
768 self
769 }
770
771 #[must_use]
773 pub fn with_compute_pool(mut self, compute: Arc<ComputePool>) -> Self {
774 self.compute = Some(compute);
775 self
776 }
777
778 #[must_use]
780 pub fn with_observer(mut self, observer: Arc<dyn Observer>) -> Self {
781 self.observer = observer;
782 self
783 }
784
785 #[must_use]
787 pub fn with_authority(mut self, authority: AuthorityController) -> Self {
788 self.authority = authority;
789 self
790 }
791
792 #[must_use]
794 pub const fn authority(&self) -> &AuthorityController {
795 &self.authority
796 }
797}
798
799impl<S, E, R, C> SyncServer<S, E, R, C>
800where
801 S: AuthoritativeStore + 'static,
802 E: OperationExecutor + 'static,
803 R: ConflictResolver + 'static,
804 C: Clock + 'static,
805{
806 pub async fn execute_server_command(
817 &self,
818 auth: AuthContext,
819 scope_id: SyncScopeId,
820 operation: OperationEnvelope,
821 ) -> Result<ServerCommandOutcome, ServerError> {
822 let session = SessionMetadata {
823 session_id: SessionId::new(),
824 device_id: auth.device_id,
825 actor_id: auth.actor_id,
826 tenant_id: auth.tenant_id,
827 scope_id,
828 partitions: Vec::new(),
829 };
830 let validation_started = Instant::now();
831 let validation = validate_request(
832 SyncRequest {
833 protocol: operation.protocol_version,
834 request_id: RequestId::new(),
835 session: session.clone(),
836 cursor: None,
837 operations: vec![operation.clone()],
838 limits: ClientLimits {
839 max_changes: 0,
840 max_response_bytes: 1,
841 },
842 capabilities: Vec::new(),
843 },
844 self.config.limits,
845 );
846 self.record_phase(ServerPhaseKind::Validation, validation_started);
847 validation?;
848 self.executor
849 .authorize_scope(&auth, &session)
850 .await
851 .map_err(ServerError::ScopeAuthorization)?;
852 let outcome = self
853 .process_operation(
854 &auth,
855 &operation,
856 &HashSet::new(),
857 scope_id,
858 *blake3::hash(&operation.payload).as_bytes(),
859 )
860 .await?;
861 Ok(match outcome {
862 ProcessResult::Acknowledged(acknowledgement) => {
863 ServerCommandOutcome::Acknowledged(acknowledgement)
864 }
865 ProcessResult::Rejected(rejection) => ServerCommandOutcome::Rejected(rejection),
866 ProcessResult::Conflict(conflict) => ServerCommandOutcome::Conflict(conflict),
867 })
868 }
869
870 #[allow(clippy::too_many_lines)]
871 async fn process_operation(
872 &self,
873 auth: &AuthContext,
874 operation: &OperationEnvelope,
875 completed: &HashSet<OperationId>,
876 scope_id: SyncScopeId,
877 command_digest: [u8; 32],
878 ) -> Result<ProcessResult, ServerError> {
879 let authority_commit = self.authority.authorize_write()?;
880 let authenticated = match IncomingOperation::new(operation).authenticate(auth) {
881 Ok(authenticated) => authenticated,
882 Err(error) => {
883 return Ok(ProcessResult::Rejected(OperationRejection {
884 operation_id: operation.operation_id,
885 code: error.code,
886 message: error.message,
887 }));
888 }
889 };
890 let provenance = authenticated.provenance(auth);
891
892 let database_started = Instant::now();
893 let previous = self
894 .store
895 .operation_result(auth.tenant_id, operation.operation_id)
896 .await?;
897 self.record_phase(ServerPhaseKind::Database, database_started);
898 if let Some(mut previous) = previous {
899 let database_started = Instant::now();
900 let original_lineage = self
901 .store
902 .operation_lineage(auth.tenant_id, operation.operation_id)
903 .await?;
904 self.record_phase(ServerPhaseKind::Database, database_started);
905 if original_lineage
906 != Some(
907 operation
908 .metadata
909 .lineage
910 .resolved_for_operation(operation.operation_id),
911 )
912 {
913 return Ok(ProcessResult::Rejected(rejection(
914 operation,
915 RejectionCode::InvalidOperation,
916 "retry lineage differs from the original operation",
917 )));
918 }
919 self.observer.record(MetricEvent::ServerTransaction {
920 outcome: TransactionOutcomeKind::Duplicate,
921 });
922 previous.duplicate = true;
923 return Ok(ProcessResult::Acknowledged(previous));
924 }
925
926 for dependency in &operation.metadata.dependencies {
927 if completed.contains(dependency) {
928 continue;
929 }
930 let database_started = Instant::now();
931 let dependency_result = self
932 .store
933 .operation_result(auth.tenant_id, *dependency)
934 .await?;
935 self.record_phase(ServerPhaseKind::Database, database_started);
936 if dependency_result.is_none() {
937 return Ok(ProcessResult::Rejected(rejection(
938 operation,
939 RejectionCode::Dependency,
940 "a required operation has not completed",
941 )));
942 }
943 }
944
945 let execution_started = Instant::now();
946 let authorization = self.executor.authorize(auth, authenticated).await;
947 self.record_phase(ServerPhaseKind::Execution, execution_started);
948 let authorized = match authorization {
949 Ok(authorized) => authorized,
950 Err(error) => {
951 return Ok(ProcessResult::Rejected(OperationRejection {
952 operation_id: operation.operation_id,
953 code: error.code,
954 message: error.message,
955 }));
956 }
957 };
958
959 let database_started = Instant::now();
960 let snapshot = self
961 .store
962 .read_entity(auth.tenant_id, operation.entity)
963 .await?;
964 self.record_phase(ServerPhaseKind::Database, database_started);
965 let current = snapshot.as_ref().map(|value| &value.current);
966 let current_version = current.map(|value| value.version);
967 let merge_policy = match check_version(operation.base_version, current_version) {
968 VersionCheck::Current => None,
969 VersionCheck::Missing => {
970 return Ok(ProcessResult::Conflict(conflict(
971 operation,
972 current_version,
973 self.conflicts.policy(operation.operation_kind.0),
974 "the entity no longer exists",
975 )));
976 }
977 VersionCheck::Diverged { .. } => {
978 let policy = self.conflicts.policy(operation.operation_kind.0);
979 match policy {
980 ConflictPolicy::ClientWins | ConflictPolicy::CommutativeOperation => None,
981 ConflictPolicy::FieldMerge
982 | ConflictPolicy::LastWriterWins
983 | ConflictPolicy::CustomMerge
984 | ConflictPolicy::Crdt => Some(policy),
985 _ => {
986 return Ok(ProcessResult::Conflict(conflict(
987 operation,
988 current_version,
989 policy,
990 "the operation is based on a stale entity version",
991 )));
992 }
993 }
994 }
995 };
996
997 let executable = authorized.validate().executable();
998 let execution_started = Instant::now();
999 let execution = self.executor.execute(auth, executable, current).await;
1000 self.record_phase(ServerPhaseKind::Execution, execution_started);
1001 let mut mutation = match execution {
1002 Ok(mutation) => mutation,
1003 Err(error) => {
1004 return Ok(ProcessResult::Rejected(OperationRejection {
1005 operation_id: operation.operation_id,
1006 code: error.code,
1007 message: error.message,
1008 }));
1009 }
1010 };
1011 if let Some(policy) = merge_policy {
1012 let Some(current) = current else {
1013 return Ok(ProcessResult::Conflict(conflict(
1014 operation,
1015 current_version,
1016 policy,
1017 "the authoritative entity is unavailable for merge",
1018 )));
1019 };
1020 match self.conflicts.merge(MergeInput {
1021 operation,
1022 current_payload: ¤t.payload,
1023 current_tombstone: current.tombstone,
1024 candidate_payload: &mutation.payload,
1025 candidate_kind: mutation.change_kind,
1026 })? {
1027 MergeDecision::Merged {
1028 payload,
1029 change_kind,
1030 } => {
1031 mutation.payload = payload;
1032 mutation.change_kind = change_kind;
1033 }
1034 MergeDecision::Unresolved { message } => {
1035 return Ok(ProcessResult::Conflict(conflict(
1036 operation,
1037 current_version,
1038 policy,
1039 &message,
1040 )));
1041 }
1042 }
1043 }
1044 let next_version = match current_version {
1045 Some(version) => version.checked_next().ok_or(ServerError::VersionOverflow)?,
1046 None => EntityVersion::INITIAL,
1047 };
1048 let event = provenance.primary_event(EventId::new());
1049 let commit = CommitOperation {
1050 authority: Some(authority_commit),
1051 operation_id: operation.operation_id,
1052 event_id: event.event_id,
1053 operation_lineage: provenance.operation_lineage(),
1054 actor_id: provenance.actor_id(),
1055 device_id: provenance.device_id(),
1056 operation_kind: operation.operation_kind.0,
1057 tenant_id: provenance.tenant_id(),
1058 scope_id,
1059 entity: operation.entity,
1060 expected_version: current_version,
1061 next_version,
1062 payload: mutation.payload,
1063 change_kind: mutation.change_kind,
1064 timestamp: self.clock.now(),
1065 command_digest,
1066 };
1067 self.authority.verify_commit_context(authority_commit)?;
1068 let database_started = Instant::now();
1069 let outcome = self.store.commit_operation(commit).await;
1070 self.record_phase(ServerPhaseKind::Database, database_started);
1071 let outcome = match outcome {
1072 Ok(outcome) => outcome,
1073 Err(error) => {
1074 self.observer.record(MetricEvent::ServerTransaction {
1075 outcome: TransactionOutcomeKind::Failed,
1076 });
1077 return Err(ServerError::Store(error));
1078 }
1079 };
1080 match outcome {
1081 CommitOutcome::Applied(ack) => {
1082 self.observer.record(MetricEvent::ServerTransaction {
1083 outcome: TransactionOutcomeKind::Applied,
1084 });
1085 Ok(ProcessResult::Acknowledged(ack))
1086 }
1087 CommitOutcome::Duplicate(mut ack) => {
1088 self.observer.record(MetricEvent::ServerTransaction {
1089 outcome: TransactionOutcomeKind::Duplicate,
1090 });
1091 ack.duplicate = true;
1092 Ok(ProcessResult::Acknowledged(ack))
1093 }
1094 CommitOutcome::VersionChanged { current } => {
1095 self.observer.record(MetricEvent::ServerTransaction {
1096 outcome: TransactionOutcomeKind::VersionChanged,
1097 });
1098 Ok(ProcessResult::Conflict(conflict(
1099 operation,
1100 current,
1101 self.conflicts.policy(operation.operation_kind.0),
1102 "the entity changed while this operation was being committed",
1103 )))
1104 }
1105 CommitOutcome::Rejected(rejection) => Ok(ProcessResult::Rejected(rejection)),
1106 }
1107 }
1108
1109 #[allow(clippy::too_many_lines)]
1110 async fn process(
1111 &self,
1112 auth: AuthContext,
1113 request: SyncRequest,
1114 ) -> Result<SyncResponse, ServerError> {
1115 let authority = self.authority.state()?;
1116 if request.protocol < self.config.limits.minimum_protocol
1117 || request.protocol > self.config.limits.current_protocol
1118 {
1119 return Ok(SyncResponse {
1120 protocol: self.config.limits.current_protocol,
1121 directive: SyncDirective::UpgradeRequired {
1122 minimum: self.config.limits.minimum_protocol,
1123 current: self.config.limits.current_protocol,
1124 },
1125 acknowledged: Vec::new(),
1126 rejected: Vec::new(),
1127 conflicts: Vec::new(),
1128 changes: Vec::new(),
1129 next_cursor: request
1130 .cursor
1131 .unwrap_or_else(|| authority.cursor(request.session.scope_id, Sequence(0))),
1132 has_more: false,
1133 server_time: self.clock.now(),
1134 });
1135 }
1136 let validation_started = Instant::now();
1137 let validation = validate_request(request, self.config.limits);
1138 self.record_phase(ServerPhaseKind::Validation, validation_started);
1139 let mut request = validation?.into_inner();
1140 self.validate_typed_request_budget(&request)?;
1141 if request.session.tenant_id != auth.tenant_id
1142 || request.session.actor_id != auth.actor_id
1143 || request.session.device_id != auth.device_id
1144 {
1145 return Err(ServerError::IdentityMismatch);
1146 }
1147 self.executor
1148 .authorize_scope(&auth, &request.session)
1149 .await
1150 .map_err(ServerError::ScopeAuthorization)?;
1151
1152 if let Some(cursor) = request.cursor {
1153 match validate_cursor(authority, cursor)? {
1154 CursorDisposition::Continue => {}
1155 CursorDisposition::AuthorityChanged { previous, current } => {
1156 return Ok(SyncResponse {
1157 protocol: request.protocol,
1158 directive: SyncDirective::AuthorityChanged {
1159 authority_id: authority.authority_id,
1160 previous_epoch: previous,
1161 current_epoch: current,
1162 },
1163 acknowledged: Vec::new(),
1164 rejected: Vec::new(),
1165 conflicts: Vec::new(),
1166 changes: Vec::new(),
1167 next_cursor: authority.cursor(request.session.scope_id, Sequence(0)),
1168 has_more: false,
1169 server_time: self.clock.now(),
1170 });
1171 }
1172 }
1173 }
1174
1175 let start = request.cursor.map_or(Sequence(0), |cursor| cursor.sequence);
1176 let database_started = Instant::now();
1177 let minimum_cursor = self
1178 .store
1179 .minimum_retained_cursor(auth.tenant_id, request.session.scope_id)
1180 .await?;
1181 self.record_phase(ServerPhaseKind::Database, database_started);
1182 if start < minimum_cursor {
1183 return Ok(SyncResponse {
1184 protocol: request.protocol,
1185 directive: SyncDirective::ResyncRequired {
1186 reason: ResyncReason::CursorExpired,
1187 },
1188 acknowledged: Vec::new(),
1189 rejected: Vec::new(),
1190 conflicts: Vec::new(),
1191 changes: Vec::new(),
1192 next_cursor: authority.cursor(request.session.scope_id, minimum_cursor),
1193 has_more: false,
1194 server_time: self.clock.now(),
1195 });
1196 }
1197
1198 let operations = std::mem::take(&mut request.operations);
1199 let (operations, dependency_plan, command_digests) =
1200 self.prepare_operations(operations).await?;
1201 let mut acknowledged = Vec::with_capacity(operations.len());
1202 let mut rejected = Vec::new();
1203 let mut conflicts = Vec::new();
1204 let mut completed = HashSet::new();
1205 for &operation_index in dependency_plan.ordered_indices() {
1206 let operation = &operations[operation_index];
1207 match self
1208 .process_operation(
1209 &auth,
1210 operation,
1211 &completed,
1212 request.session.scope_id,
1213 command_digests[operation_index],
1214 )
1215 .await?
1216 {
1217 ProcessResult::Acknowledged(ack) => {
1218 completed.insert(ack.operation_id);
1219 acknowledged.push(ack);
1220 }
1221 ProcessResult::Rejected(rejection) => rejected.push(rejection),
1222 ProcessResult::Conflict(conflict) => conflicts.push(conflict),
1223 }
1224 }
1225
1226 let requested_limit = usize::try_from(request.limits.max_changes).unwrap_or(usize::MAX);
1227 let database_started = Instant::now();
1228 let page = self
1229 .store
1230 .read_changes_after(
1231 auth.tenant_id,
1232 request.session.scope_id,
1233 start,
1234 requested_limit.min(self.config.max_pull_changes),
1235 usize::try_from(request.limits.max_response_bytes).unwrap_or(usize::MAX),
1236 )
1237 .await?;
1238 self.record_phase(ServerPhaseKind::Database, database_started);
1239 let journal_head = page.journal_head;
1240 let mut response = SyncResponse {
1241 protocol: request.protocol,
1242 directive: SyncDirective::Continue,
1243 acknowledged,
1244 rejected,
1245 conflicts,
1246 changes: page.changes,
1247 next_cursor: authority.cursor(request.session.scope_id, page.next_sequence),
1248 has_more: page.has_more,
1249 server_time: self.clock.now(),
1250 };
1251 self.fit_response(&mut response, start, request.limits.max_response_bytes)?;
1252 self.observer.record(MetricEvent::ServerJournalLag {
1253 sequences: journal_head
1254 .0
1255 .saturating_sub(response.next_cursor.sequence.0),
1256 });
1257 Ok(response)
1258 }
1259
1260 fn fit_response(
1261 &self,
1262 response: &mut SyncResponse,
1263 start: Sequence,
1264 maximum_bytes: u32,
1265 ) -> Result<(), ServerError> {
1266 let maximum = usize::try_from(maximum_bytes)
1267 .unwrap_or(usize::MAX)
1268 .min(self.config.performance.memory.response_bytes);
1269 loop {
1270 let encoded = aequora_codec::encode(
1271 response.protocol,
1272 aequora_codec::MessageKind::SyncResponse,
1273 response,
1274 )?;
1275 if encoded.len() <= maximum {
1276 return Ok(());
1277 }
1278 if response.changes.pop().is_none() {
1279 return Err(ServerError::ResponseLimit);
1280 }
1281 response.has_more = true;
1282 response.next_cursor.sequence = response
1283 .changes
1284 .last()
1285 .map_or(start, |change| change.sequence);
1286 if response.changes.is_empty()
1287 && response.acknowledged.is_empty()
1288 && response.rejected.is_empty()
1289 && response.conflicts.is_empty()
1290 {
1291 return Err(ServerError::ResponseLimit);
1292 }
1293 }
1294 }
1295
1296 fn validate_typed_request_budget(&self, request: &SyncRequest) -> Result<(), ServerError> {
1297 if request.operations.len() > self.config.performance.memory.pending_decode_records {
1298 return Err(ServerError::MemoryBudget {
1299 domain: "pending decode records",
1300 actual: request.operations.len(),
1301 maximum: self.config.performance.memory.pending_decode_records,
1302 });
1303 }
1304 let bytes = request
1305 .operations
1306 .iter()
1307 .try_fold(0_usize, |total, operation| {
1308 total.checked_add(operation.payload.len())
1309 });
1310 let actual = bytes.unwrap_or(usize::MAX);
1311 if actual > self.config.performance.memory.decode_bytes {
1312 return Err(ServerError::MemoryBudget {
1313 domain: "typed request payload",
1314 actual,
1315 maximum: self.config.performance.memory.decode_bytes,
1316 });
1317 }
1318 Ok(())
1319 }
1320
1321 async fn prepare_operations(
1322 &self,
1323 operations: Vec<OperationEnvelope>,
1324 ) -> Result<(Vec<OperationEnvelope>, DependencyPlan, Vec<[u8; 32]>), ServerError> {
1325 let Some(compute) = &self.compute else {
1326 let plan = plan_dependencies(&operations)?;
1327 let digests = operations
1328 .iter()
1329 .map(|operation| *blake3::hash(&operation.payload).as_bytes())
1330 .collect();
1331 return Ok((operations, plan, digests));
1332 };
1333 if !compute.should_parallelize(operations.len()) {
1334 let plan = plan_dependencies(&operations)?;
1335 let digests = operations
1336 .iter()
1337 .map(|operation| *blake3::hash(&operation.payload).as_bytes())
1338 .collect();
1339 return Ok((operations, plan, digests));
1340 }
1341 self.observer.record(MetricEvent::ComputeOffload {
1342 items: usize_to_u64(operations.len()),
1343 });
1344 Ok(compute
1345 .run(move || {
1346 let plan = plan_dependencies(&operations)?;
1347 let digests = operations
1348 .par_iter()
1349 .map(|operation| *blake3::hash(&operation.payload).as_bytes())
1350 .collect();
1351 Ok::<_, DependencyError>((operations, plan, digests))
1352 })
1353 .await??)
1354 }
1355
1356 fn record_phase(&self, phase: ServerPhaseKind, started: Instant) {
1357 self.observer.record(MetricEvent::ServerPhase {
1358 phase,
1359 duration_micros: duration_micros(started.elapsed()),
1360 });
1361 }
1362
1363 async fn process_bootstrap(
1364 &self,
1365 auth: AuthContext,
1366 request: BootstrapRequest,
1367 ) -> Result<BootstrapResponse, ServerError> {
1368 let authority = self.authority.state()?;
1369 self.authority.validate_startup()?;
1370 let validation_started = Instant::now();
1371 let validation = validate_bootstrap_request(&request, self.config.limits);
1372 self.record_phase(ServerPhaseKind::Validation, validation_started);
1373 validation?;
1374 if request.session.tenant_id != auth.tenant_id
1375 || request.session.actor_id != auth.actor_id
1376 || request.session.device_id != auth.device_id
1377 {
1378 return Err(ServerError::IdentityMismatch);
1379 }
1380 self.executor
1381 .authorize_scope(&auth, &request.session)
1382 .await
1383 .map_err(ServerError::ScopeAuthorization)?;
1384 let snapshot_id = if let Some(snapshot_id) = request.snapshot_id {
1385 snapshot_id
1386 } else {
1387 let database_started = Instant::now();
1388 let descriptor = self
1389 .store
1390 .create_snapshot_in_timeline(
1391 auth.tenant_id,
1392 request.session.scope_id,
1393 &request.session.partitions,
1394 authority.authority_id,
1395 authority.epoch,
1396 )
1397 .await?;
1398 self.record_phase(ServerPhaseKind::Database, database_started);
1399 descriptor.snapshot_id
1400 };
1401 let database_started = Instant::now();
1402 let page = self
1403 .store
1404 .read_snapshot(
1405 auth.tenant_id,
1406 snapshot_id,
1407 request.offset,
1408 usize::try_from(request.limits.max_entities)
1409 .unwrap_or(usize::MAX)
1410 .min(self.config.performance.snapshot.max_records_per_chunk),
1411 usize::try_from(request.limits.max_payload_bytes)
1412 .unwrap_or(usize::MAX)
1413 .min(self.config.performance.snapshot.max_chunk_bytes),
1414 )
1415 .await?;
1416 self.record_phase(ServerPhaseKind::Database, database_started);
1417 if page.descriptor.cursor.authority_id != AuthorityId::LEGACY_UNBOUND {
1418 if page.descriptor.cursor.authority_id != authority.authority_id {
1419 return Err(AuthorityError::AuthorityIdChanged {
1420 trusted: authority.authority_id,
1421 presented: page.descriptor.cursor.authority_id,
1422 }
1423 .into());
1424 }
1425 if page.descriptor.cursor.authority_epoch != authority.epoch {
1426 return Err(AuthorityError::ArtifactEpochMismatch {
1427 artifact: page.descriptor.cursor.authority_epoch,
1428 current: authority.epoch,
1429 }
1430 .into());
1431 }
1432 }
1433 if page.descriptor.cursor.scope != request.session.scope_id {
1434 return Err(ServerError::IdentityMismatch);
1435 }
1436 if page.has_more && page.entities.is_empty() && page.next_offset == request.offset {
1437 return Err(ServerError::SnapshotNoProgress);
1438 }
1439 Ok(BootstrapResponse {
1440 protocol: request.protocol,
1441 snapshot_id: page.descriptor.snapshot_id,
1442 cursor: authority.cursor(request.session.scope_id, page.descriptor.cursor.sequence),
1443 offset: request.offset,
1444 entities: page.entities,
1445 next_offset: page.next_offset,
1446 has_more: page.has_more,
1447 server_time: self.clock.now(),
1448 })
1449 }
1450}
1451
1452#[async_trait]
1453impl<S, E, R, C> ExchangeService for SyncServer<S, E, R, C>
1454where
1455 S: AuthoritativeStore + 'static,
1456 E: OperationExecutor + 'static,
1457 R: ConflictResolver + 'static,
1458 C: Clock + 'static,
1459{
1460 async fn exchange(
1461 &self,
1462 auth: AuthContext,
1463 request: SyncRequest,
1464 ) -> Result<SyncResponse, ServerError> {
1465 let operations = request.operations.len();
1466 let trace = TraceContext {
1467 sync_session_id: request.session.session_id,
1468 request_id: request.request_id,
1469 device_id: request.session.device_id,
1470 tenant_id: request.session.tenant_id,
1471 };
1472 let started = Instant::now();
1473 let result = self.process(auth, request).await;
1474 let (changes, conflicts, rejections) = result.as_ref().map_or((0, 0, 0), |response| {
1475 (
1476 response.changes.len(),
1477 response.conflicts.len(),
1478 response.rejected.len(),
1479 )
1480 });
1481 self.observer.record_with_context(
1482 trace,
1483 MetricEvent::ServerExchange {
1484 duration_micros: duration_micros(started.elapsed()),
1485 operations: usize_to_u64(operations),
1486 changes: usize_to_u64(changes),
1487 conflicts: usize_to_u64(conflicts),
1488 rejections: usize_to_u64(rejections),
1489 outcome: server_result_outcome(&result),
1490 },
1491 );
1492 result
1493 }
1494
1495 async fn bootstrap(
1496 &self,
1497 auth: AuthContext,
1498 request: BootstrapRequest,
1499 ) -> Result<BootstrapResponse, ServerError> {
1500 let trace = TraceContext {
1501 sync_session_id: request.session.session_id,
1502 request_id: request.request_id,
1503 device_id: request.session.device_id,
1504 tenant_id: request.session.tenant_id,
1505 };
1506 let started = Instant::now();
1507 let result = self.process_bootstrap(auth, request).await;
1508 self.observer.record_with_context(
1509 trace,
1510 MetricEvent::BootstrapPage {
1511 duration_micros: duration_micros(started.elapsed()),
1512 entities: result
1513 .as_ref()
1514 .map_or(0, |response| usize_to_u64(response.entities.len())),
1515 outcome: server_result_outcome(&result),
1516 },
1517 );
1518 result
1519 }
1520}
1521
1522fn duration_micros(duration: std::time::Duration) -> u64 {
1523 u64::try_from(duration.as_micros()).unwrap_or(u64::MAX)
1524}
1525
1526fn usize_to_u64(value: usize) -> u64 {
1527 u64::try_from(value).unwrap_or(u64::MAX)
1528}
1529
1530fn development_authority() -> AuthorityController {
1531 let mut state = AuthorityState::new(
1532 AuthorityId::LOCAL_DEVELOPMENT,
1533 AuthorityInstanceId::new(),
1534 AuthorityRole::Primary,
1535 0,
1536 );
1537 state.runtime_mode = AuthorityRuntimeMode::Serving;
1538 AuthorityController::new(state, AuthorityPromotionPolicy::default())
1539}
1540
1541fn server_result_outcome<T>(result: &Result<T, ServerError>) -> OutcomeKind {
1542 match result {
1543 Ok(_) => OutcomeKind::Success,
1544 Err(ServerError::Store(error)) if error.kind == StoreErrorKind::Transient => {
1545 OutcomeKind::TransientFailure
1546 }
1547 Err(ServerError::Maintenance { .. }) => OutcomeKind::TransientFailure,
1548 Err(ServerError::Admission(rejection)) if rejection.retryable() => {
1549 OutcomeKind::TransientFailure
1550 }
1551 Err(_) => OutcomeKind::PermanentFailure,
1552 }
1553}
1554
1555enum ProcessResult {
1556 Acknowledged(OperationAck),
1557 Rejected(OperationRejection),
1558 Conflict(Conflict),
1559}
1560
1561fn rejection(
1562 operation: &OperationEnvelope,
1563 code: RejectionCode,
1564 message: &str,
1565) -> OperationRejection {
1566 OperationRejection {
1567 operation_id: operation.operation_id,
1568 code,
1569 message: message.to_owned(),
1570 }
1571}
1572
1573fn conflict(
1574 operation: &OperationEnvelope,
1575 current: Option<EntityVersion>,
1576 policy: ConflictPolicy,
1577 message: &str,
1578) -> Conflict {
1579 Conflict {
1580 operation_id: operation.operation_id,
1581 entity: operation.entity,
1582 client_base: operation.base_version,
1583 server_version: current,
1584 policy,
1585 message: message.to_owned(),
1586 }
1587}
1588
1589#[cfg(test)]
1590mod tests {
1591 use std::sync::atomic::{AtomicUsize, Ordering};
1592
1593 use super::*;
1594
1595 struct CountingService(AtomicUsize);
1596
1597 #[async_trait]
1598 impl ExchangeService for CountingService {
1599 async fn exchange(
1600 &self,
1601 _auth: AuthContext,
1602 _request: SyncRequest,
1603 ) -> Result<SyncResponse, ServerError> {
1604 self.0.fetch_add(1, Ordering::Relaxed);
1605 Err(ServerError::BootstrapUnavailable)
1606 }
1607 }
1608
1609 fn empty_request(auth: AuthContext) -> SyncRequest {
1610 SyncRequest {
1611 protocol: aequora_types::ProtocolVersion::V1,
1612 request_id: aequora_types::RequestId::new(),
1613 session: SessionMetadata {
1614 session_id: aequora_types::SessionId::new(),
1615 device_id: auth.device_id,
1616 actor_id: auth.actor_id,
1617 tenant_id: auth.tenant_id,
1618 scope_id: aequora_types::SyncScopeId::new(),
1619 partitions: Vec::new(),
1620 },
1621 cursor: None,
1622 operations: Vec::new(),
1623 limits: ClientLimits::default(),
1624 capabilities: Vec::new(),
1625 }
1626 }
1627
1628 #[test]
1629 fn maintenance_controller_transitions_atomically() {
1630 let controller = MaintenanceController::default();
1631 assert_eq!(controller.mode(), MaintenanceMode::Normal);
1632 assert_eq!(
1633 controller.set_mode(MaintenanceMode::ReadOnly),
1634 MaintenanceMode::Normal
1635 );
1636 assert_eq!(controller.mode(), MaintenanceMode::ReadOnly);
1637 assert_eq!(
1638 controller.set_mode(MaintenanceMode::SyncPaused),
1639 MaintenanceMode::ReadOnly
1640 );
1641 assert_eq!(controller.mode(), MaintenanceMode::SyncPaused);
1642 }
1643
1644 #[test]
1645 fn maintenance_policy_preserves_read_only_pull_and_blocks_writes() {
1646 assert!(MaintenanceMode::Normal.allows_exchange(false));
1647 assert!(MaintenanceMode::Normal.allows_exchange(true));
1648 assert!(MaintenanceMode::ReadOnly.allows_exchange(false));
1649 assert!(!MaintenanceMode::ReadOnly.allows_exchange(true));
1650 assert!(!MaintenanceMode::SyncPaused.allows_exchange(false));
1651 assert!(!MaintenanceMode::SyncPaused.allows_exchange(true));
1652 assert!(MaintenanceMode::Normal.allows_bootstrap());
1653 assert!(MaintenanceMode::ReadOnly.allows_bootstrap());
1654 assert!(!MaintenanceMode::SyncPaused.allows_bootstrap());
1655 }
1656
1657 #[tokio::test]
1658 async fn admission_decorator_rejects_before_inner_service_and_releases_with_raii() {
1659 let mut policy = aequora_admission::AdmissionPolicy::default();
1660 policy.global.max_in_flight = 1;
1661 policy.tenant.budget.max_in_flight = 1;
1662 policy.tenant.max_tracked_tenants = 1;
1663 policy.critical.reserved_in_flight = 0;
1664 policy.interactive.reserved_in_flight = 0;
1665 policy.normal.reserved_in_flight = 0;
1666 policy.bulk.reserved_in_flight = 0;
1667 policy.background.reserved_in_flight = 0;
1668 policy.maintenance.reserved_in_flight = 0;
1669 let admission = Arc::new(
1670 aequora_admission::HierarchicalAdmission::new(policy)
1671 .unwrap_or_else(|error| panic!("{error}")),
1672 );
1673 let inner = Arc::new(CountingService(AtomicUsize::new(0)));
1674 let service = AdmittedExchangeService::new(Arc::clone(&inner), admission.clone());
1675 let auth = AuthContext {
1676 actor_id: aequora_types::ActorId::new(),
1677 tenant_id: aequora_types::TenantId::new(),
1678 device_id: aequora_types::DeviceId::new(),
1679 };
1680 let request = empty_request(auth);
1681 let blocker = admission
1682 .try_admit(&service.exchange_work(&auth, &request))
1683 .unwrap_or_else(|error| panic!("{error}"));
1684 assert!(matches!(
1685 service.exchange(auth, request.clone()).await,
1686 Err(ServerError::Admission(
1687 AdmissionRejection::ServerBusy { .. }
1688 ))
1689 ));
1690 assert_eq!(inner.0.load(Ordering::Relaxed), 0);
1691 drop(blocker);
1692 assert!(matches!(
1693 service.exchange(auth, request).await,
1694 Err(ServerError::BootstrapUnavailable)
1695 ));
1696 assert_eq!(inner.0.load(Ordering::Relaxed), 1);
1697 }
1698}