Skip to main content

aequora_server/
lib.rs

1//! Authoritative sync-session orchestration.
2
3/// Thin server-side boundary for a separately hosted operational control plane.
4pub mod admin;
5
6/// Read-only, tenant-authorized forensic collection boundary.
7pub mod diagnostics;
8
9/// Database-neutral server orchestration for durable background work.
10pub mod jobs;
11mod sdk;
12
13pub use sdk::{
14    AequoraServer, AequoraServerBuilder, DomainRegistrationError, DomainRegistry,
15    DomainRegistryBuilder, ServerOperationOutcome, StableServerError,
16};
17
18/// Focused imports for application server integrations.
19pub mod prelude {
20    pub use crate::admin::{AdminControlPlane, AdminPlaneConfig};
21    pub use crate::diagnostics::{DiagnosticAccessPolicy, ServerDiagnosticCollector};
22    pub use crate::jobs::{
23        JobAccessPolicy, JobAdminService, JobWorkerConfig, JobWorkerPlanner, WorkerShutdown,
24    };
25    pub use crate::{
26        AdmittedExchangeService, AequoraServer, AequoraServerBuilder, DomainRegistrationError,
27        DomainRegistry, DomainRegistryBuilder, ExchangeService, ServerBuildError,
28        ServerCommandOutcome, ServerConfig, ServerError, ServerOperationOutcome,
29        ServerRegionalReadGuard, StableServerError, SyncServer, SyncServerBuilder,
30    };
31    pub use aequora_adapter_sdk::{
32        Authenticator, AuthorityService, DomainContext, DomainHandler, DomainOutcome,
33    };
34    pub use aequora_admission::{AdmissionController, AdmissionPolicy, HierarchicalAdmission};
35    pub use aequora_authority::{
36        AuthorityController, AuthorityPromotionPolicy, AuthorityRole, AuthorityRuntimeMode,
37        AuthorityState,
38    };
39    pub use aequora_clock::{Clock, SystemClock};
40    pub use aequora_conflict::{ConflictResolver, RejectConflicts};
41    pub use aequora_executor::{
42        AuthContext, DerivedEventProvenance, DomainOperation, JobProvenance, OperationExecutor,
43        OperationHandler, OperationRegistry, ScopeAuthorizer, TrustedProvenance,
44    };
45    pub use aequora_jobs::{JobRegistry, JobStore};
46    pub use aequora_region::{
47        ReadResponseMetadata, RegionError, RegionalReadRequest, ReplicaObservation,
48    };
49    pub use aequora_side_effects::{
50        AuthoritativeOperationSink, SideEffectProvider, SideEffectStore,
51    };
52    pub use aequora_store::{
53        AdapterCapabilities, AdapterManifest, AdapterManifestProvider, AdapterRequirements,
54        AdapterRole, AdapterTier, AuthoritativeStore, ProductionAdapterPair,
55    };
56    pub use aequora_workflow::{WorkflowDefinition, WorkflowStore};
57}
58
59use aequora_admission::{
60    AdmissionController, AdmissionRejection, CostUnits, RequestShape, ResourceDomain,
61    ServerPriorityPolicy, WorkDescriptor,
62};
63use aequora_authority::{
64    AuthorityController, AuthorityError, AuthorityPromotionPolicy, AuthorityRole,
65    AuthorityRuntimeMode, AuthorityState, CursorDisposition, validate_cursor,
66};
67use aequora_clock::Clock;
68use aequora_compute::{ComputeError, ComputePool};
69use aequora_conflict::{
70    ConflictResolver, MergeDecision, MergeError, MergeInput, VersionCheck, check_version,
71};
72use aequora_executor::{
73    AuthContext, DependencyError, DependencyPlan, ExecutionError, IncomingOperation,
74    OperationExecutor, plan_dependencies,
75};
76use aequora_observability::{
77    MetricEvent, NoopObserver, Observer, OutcomeKind, ServerPhaseKind, TraceContext,
78    TransactionOutcomeKind,
79};
80use aequora_performance::PerformancePolicy;
81use aequora_protocol::{
82    BootstrapRequest, BootstrapResponse, ClientLimits, Conflict, ConflictPolicy, OperationAck,
83    OperationEnvelope, OperationRejection, RejectionCode, ResyncReason, SessionMetadata,
84    SyncDirective, SyncRequest, SyncResponse,
85};
86use aequora_region::{
87    ReadResponseMetadata, RegionError, RegionalReadRequest, ReplicaObservation, ReplicaReadGuard,
88};
89use aequora_store::{
90    AdapterCompatibilityError, AdapterManifestProvider, AdapterRequirements, AuthoritativeStore,
91    CommitOperation, CommitOutcome, StoreError, StoreErrorKind,
92};
93use aequora_types::{
94    AuthorityId, AuthorityInstanceId, EntityVersion, EventId, OperationId, RequestId, Sequence,
95    SessionId, SyncScopeId,
96};
97use aequora_validator::{
98    ProtocolLimits, ValidationError, validate_bootstrap_request, validate_request,
99};
100use async_trait::async_trait;
101use rayon::prelude::*;
102use std::{
103    collections::HashSet,
104    sync::{
105        Arc,
106        atomic::{AtomicU8, Ordering},
107    },
108    time::Instant,
109};
110use thiserror::Error;
111
112/// Replica-side defense-in-depth for regional read API handlers.
113///
114/// Routing metadata can be stale, so every server instance validates its own durable watermark,
115/// epoch, projection version, and authorization freshness immediately before reading.
116#[derive(Clone, Debug)]
117pub struct ServerRegionalReadGuard {
118    observation: ReplicaObservation,
119}
120
121impl ServerRegionalReadGuard {
122    /// Creates a guard from the adapter's latest durable apply observation.
123    #[must_use]
124    pub const fn new(observation: ReplicaObservation) -> Self {
125        Self { observation }
126    }
127
128    /// Replaces health and watermark state after an atomic adapter observation refresh.
129    pub fn update(&mut self, observation: ReplicaObservation) {
130        self.observation = observation;
131    }
132
133    /// Authorizes one regional read and returns freshness metadata for its response.
134    ///
135    /// # Errors
136    ///
137    /// Fails closed if this node cannot prove the requested consistency and security bounds.
138    pub fn authorize(
139        &self,
140        request: RegionalReadRequest,
141    ) -> Result<ReadResponseMetadata, RegionError> {
142        ReplicaReadGuard::validate(&self.observation, request)
143    }
144}
145
146/// Server processing configuration.
147#[derive(Clone, Copy, Debug)]
148pub struct ServerConfig {
149    /// Structural protocol bounds.
150    pub limits: ProtocolLimits,
151    /// Absolute maximum pull page size.
152    pub max_pull_changes: usize,
153    /// Cross-layer request, response, snapshot, and CPU memory policy.
154    pub performance: PerformancePolicy,
155}
156
157impl Default for ServerConfig {
158    fn default() -> Self {
159        Self {
160            limits: ProtocolLimits::default(),
161            max_pull_changes: 1_024,
162            performance: PerformancePolicy::default(),
163        }
164    }
165}
166
167/// A request-level failure. Per-operation business failures are represented in `SyncResponse`.
168#[derive(Debug, Error)]
169pub enum ServerError {
170    /// Cross-transport admission rejected work before authoritative mutation.
171    #[error("sync admission rejected work: {0}")]
172    Admission(#[from] AdmissionRejection),
173    /// Request failed structural validation.
174    #[error("invalid sync request: {0}")]
175    Validation(#[from] ValidationError),
176    /// Session identity did not match authenticated identity.
177    #[error("sync session identity does not match authenticated identity")]
178    IdentityMismatch,
179    /// Persistence failed before a safe response could be produced.
180    #[error("sync persistence failed: {0}")]
181    Store(#[from] StoreError),
182    /// Entity version exhausted its integer representation.
183    #[error("entity version overflow")]
184    VersionOverflow,
185    /// The operation dependency graph was invalid.
186    #[error("invalid operation dependency graph: {0}")]
187    Dependency(#[from] DependencyError),
188    /// The application rejected access to the requested partial sync scope.
189    #[error("sync scope is not authorized: {0}")]
190    ScopeAuthorization(ExecutionError),
191    /// Snapshot page limits were too small to return even one remaining entity.
192    #[error("snapshot page limits are too small to make progress")]
193    SnapshotNoProgress,
194    /// This service implementation does not provide bootstrap snapshots.
195    #[error("snapshot bootstrap is not available")]
196    BootstrapUnavailable,
197    /// Dedicated CPU work could not complete.
198    #[error("sync compute work failed: {0}")]
199    Compute(#[from] ComputeError),
200    /// The client advertised a response ceiling too small for mandatory progress.
201    #[error("client response limit is too small to make synchronization progress")]
202    ResponseLimit,
203    /// A response could not be measured with the production wire codec.
204    #[error("sync response encoding failed: {0}")]
205    Codec(#[from] aequora_codec::CodecError),
206    /// A typed request or response exceeded the configured in-memory domain budget.
207    #[error("{domain} memory use {actual} exceeds configured limit {maximum}")]
208    MemoryBudget {
209        domain: &'static str,
210        actual: usize,
211        maximum: usize,
212    },
213    /// An application-registered deterministic merger failed.
214    #[error("sync conflict merge failed: {0}")]
215    Merge(#[from] MergeError),
216    /// Operator-selected maintenance mode currently rejects this synchronization action.
217    #[error("sync is unavailable while the server is in {mode:?} maintenance mode")]
218    Maintenance {
219        /// Active maintenance policy.
220        mode: MaintenanceMode,
221        /// Retry delay safe to expose to clients and transports.
222        retry_after_seconds: u64,
223    },
224    /// Authority identity, epoch, runtime mode, or write fence rejected the request.
225    #[error("authority rejected synchronization: {0}")]
226    Authority(#[from] AuthorityError),
227}
228
229/// Runtime maintenance policy for an [`ExchangeService`].
230#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
231#[repr(u8)]
232pub enum MaintenanceMode {
233    /// Bidirectional synchronization and bootstrap are enabled.
234    #[default]
235    Normal = 0,
236    /// Pull and bootstrap remain available, but exchanges containing writes are rejected.
237    ReadOnly = 1,
238    /// All exchange and bootstrap work is rejected with a retryable typed error.
239    SyncPaused = 2,
240}
241
242impl MaintenanceMode {
243    /// Whether one exchange may proceed in this mode.
244    #[must_use]
245    pub const fn allows_exchange(self, has_operations: bool) -> bool {
246        match self {
247            Self::Normal => true,
248            Self::ReadOnly => !has_operations,
249            Self::SyncPaused => false,
250        }
251    }
252
253    /// Whether snapshot bootstrap may proceed in this mode.
254    #[must_use]
255    pub const fn allows_bootstrap(self) -> bool {
256        matches!(self, Self::Normal | Self::ReadOnly)
257    }
258}
259
260/// Cloneable, lock-free maintenance policy handle suitable for an authenticated control plane.
261#[derive(Clone, Default)]
262pub struct MaintenanceController {
263    mode: Arc<AtomicU8>,
264}
265
266impl MaintenanceController {
267    /// Returns the active mode.
268    #[must_use]
269    pub fn mode(&self) -> MaintenanceMode {
270        match self.mode.load(Ordering::Acquire) {
271            1 => MaintenanceMode::ReadOnly,
272            2 => MaintenanceMode::SyncPaused,
273            _ => MaintenanceMode::Normal,
274        }
275    }
276
277    /// Atomically changes the active mode and returns the previous value.
278    #[must_use]
279    pub fn set_mode(&self, mode: MaintenanceMode) -> MaintenanceMode {
280        let previous = self.mode.swap(mode as u8, Ordering::AcqRel);
281        match previous {
282            1 => MaintenanceMode::ReadOnly,
283            2 => MaintenanceMode::SyncPaused,
284            _ => MaintenanceMode::Normal,
285        }
286    }
287}
288
289/// Maintenance-policy decorator that preserves pending client work by rejecting before execution.
290pub struct MaintenanceService<S: ?Sized> {
291    inner: Arc<S>,
292    controller: MaintenanceController,
293    retry_after_seconds: u64,
294}
295
296impl<S: ?Sized> MaintenanceService<S> {
297    /// Wraps a service with a runtime maintenance gate.
298    #[must_use]
299    pub fn new(inner: Arc<S>, controller: MaintenanceController, retry_after_seconds: u64) -> Self {
300        Self {
301            inner,
302            controller,
303            retry_after_seconds: retry_after_seconds.max(1),
304        }
305    }
306
307    /// Returns the handle used by the host's authenticated control plane.
308    #[must_use]
309    pub const fn controller(&self) -> &MaintenanceController {
310        &self.controller
311    }
312
313    fn rejection(&self, mode: MaintenanceMode) -> ServerError {
314        ServerError::Maintenance {
315            mode,
316            retry_after_seconds: self.retry_after_seconds,
317        }
318    }
319}
320
321#[async_trait]
322impl<S> ExchangeService for MaintenanceService<S>
323where
324    S: ExchangeService + ?Sized,
325{
326    async fn exchange(
327        &self,
328        auth: AuthContext,
329        request: SyncRequest,
330    ) -> Result<SyncResponse, ServerError> {
331        let mode = self.controller.mode();
332        if mode.allows_exchange(!request.operations.is_empty()) {
333            self.inner.exchange(auth, request).await
334        } else {
335            Err(self.rejection(mode))
336        }
337    }
338
339    async fn bootstrap(
340        &self,
341        auth: AuthContext,
342        request: BootstrapRequest,
343    ) -> Result<BootstrapResponse, ServerError> {
344        let mode = self.controller.mode();
345        if mode.allows_bootstrap() {
346            self.inner.bootstrap(auth, request).await
347        } else {
348            Err(self.rejection(mode))
349        }
350    }
351}
352
353/// Result of a server-originated command executed through the same authoritative pipeline as sync.
354#[derive(Clone, Debug, Eq, PartialEq)]
355pub enum ServerCommandOutcome {
356    /// The command committed or replayed its durable ledger result.
357    Acknowledged(OperationAck),
358    /// Application authorization or business validation rejected the command.
359    Rejected(OperationRejection),
360    /// The command requires conflict handling.
361    Conflict(Conflict),
362}
363
364/// Object-safe server boundary used by Axum and in-process transports.
365#[async_trait]
366pub trait ExchangeService: Send + Sync {
367    /// Processes one authenticated bidirectional exchange.
368    async fn exchange(
369        &self,
370        auth: AuthContext,
371        request: SyncRequest,
372    ) -> Result<SyncResponse, ServerError>;
373
374    /// Begins or resumes one authenticated consistent snapshot.
375    async fn bootstrap(
376        &self,
377        _auth: AuthContext,
378        _request: BootstrapRequest,
379    ) -> Result<BootstrapResponse, ServerError> {
380        Err(ServerError::BootstrapUnavailable)
381    }
382}
383
384/// Optional cross-transport admission decorator held across authoritative execution.
385pub struct AdmittedExchangeService<S: ?Sized> {
386    inner: Arc<S>,
387    admission: Arc<dyn AdmissionController>,
388    priority: ServerPriorityPolicy,
389    tenant_weight: u16,
390}
391
392impl<S: ?Sized> AdmittedExchangeService<S> {
393    /// Wraps a service with server-derived work classification and hierarchical RAII admission.
394    #[must_use]
395    pub fn new(inner: Arc<S>, admission: Arc<dyn AdmissionController>) -> Self {
396        Self {
397            inner,
398            admission,
399            priority: ServerPriorityPolicy::default(),
400            tenant_weight: 1,
401        }
402    }
403
404    /// Replaces the server-owned work-kind classification policy.
405    #[must_use]
406    pub fn with_priority_policy(mut self, priority: ServerPriorityPolicy) -> Self {
407        self.priority = priority;
408        self
409    }
410
411    /// Applies a server-derived contractual tenant weight, never a request-body value.
412    #[must_use]
413    pub fn with_tenant_weight(mut self, tenant_weight: u16) -> Self {
414        self.tenant_weight = tenant_weight.max(1);
415        self
416    }
417
418    fn exchange_work(&self, auth: &AuthContext, request: &SyncRequest) -> WorkDescriptor {
419        let operation_count = request.operations.len();
420        let dependency_edges = request
421            .operations
422            .iter()
423            .map(|operation| operation.metadata.dependencies.len())
424            .sum::<usize>();
425        let payload_bytes = request.operations.iter().fold(0u64, |total, operation| {
426            total.saturating_add(u64::try_from(operation.payload.len()).unwrap_or(u64::MAX))
427        });
428        let kind = if request.operations.is_empty() {
429            aequora_scheduler::WorkKind::PullChanges
430        } else {
431            aequora_scheduler::WorkKind::PushOperations
432        };
433        let class = self.priority.classify(kind, None);
434        let mut resources = vec![
435            ResourceDomain::SyncExchange,
436            ResourceDomain::DatabaseTransaction,
437        ];
438        if !request.operations.is_empty() {
439            resources.push(ResourceDomain::InteractiveCpu);
440        }
441        WorkDescriptor {
442            tenant_id: auth.tenant_id,
443            tenant_weight: self.tenant_weight,
444            kind,
445            class,
446            cost: CostUnits::new(
447                u32::try_from(
448                    operation_count
449                        .saturating_add(dependency_edges)
450                        .saturating_add(1),
451                )
452                .unwrap_or(u32::MAX),
453            ),
454            shape: RequestShape {
455                operations: operation_count,
456                encoded_bytes: payload_bytes,
457                decompressed_bytes: payload_bytes,
458                scopes: request.session.partitions.len(),
459                dependency_edges,
460                dependency_depth: request
461                    .operations
462                    .iter()
463                    .map(|operation| operation.metadata.dependencies.len())
464                    .max()
465                    .unwrap_or(0),
466            },
467            scope: Some(request.session.scope_id),
468            estimated_response_bytes: u64::from(request.limits.max_response_bytes),
469            resources,
470        }
471    }
472
473    fn bootstrap_work(&self, auth: &AuthContext, request: &BootstrapRequest) -> WorkDescriptor {
474        let kind = aequora_scheduler::WorkKind::Bootstrap;
475        WorkDescriptor {
476            tenant_id: auth.tenant_id,
477            tenant_weight: self.tenant_weight,
478            kind,
479            class: self.priority.classify(kind, None),
480            cost: CostUnits::new(request.limits.max_entities.max(1)),
481            shape: RequestShape {
482                scopes: request.session.partitions.len(),
483                ..RequestShape::default()
484            },
485            scope: Some(request.session.scope_id),
486            estimated_response_bytes: u64::from(request.limits.max_payload_bytes),
487            resources: vec![
488                ResourceDomain::SyncExchange,
489                ResourceDomain::DatabaseTransaction,
490                ResourceDomain::SnapshotDownload,
491            ],
492        }
493    }
494}
495
496#[async_trait]
497impl<S> ExchangeService for AdmittedExchangeService<S>
498where
499    S: ExchangeService + ?Sized,
500{
501    async fn exchange(
502        &self,
503        auth: AuthContext,
504        request: SyncRequest,
505    ) -> Result<SyncResponse, ServerError> {
506        let work = self.exchange_work(&auth, &request);
507        let _permit = self.admission.admit(&work).await?;
508        self.inner.exchange(auth, request).await
509    }
510
511    async fn bootstrap(
512        &self,
513        auth: AuthContext,
514        request: BootstrapRequest,
515    ) -> Result<BootstrapResponse, ServerError> {
516        let work = self.bootstrap_work(&auth, &request);
517        let _permit = self.admission.admit(&work).await?;
518        self.inner.bootstrap(auth, request).await
519    }
520}
521
522/// Server-authoritative synchronization service.
523pub struct SyncServer<S, E, R, C> {
524    store: Arc<S>,
525    executor: Arc<E>,
526    conflicts: Arc<R>,
527    clock: Arc<C>,
528    config: ServerConfig,
529    compute: Option<Arc<ComputePool>>,
530    observer: Arc<dyn Observer>,
531    authority: AuthorityController,
532}
533
534/// Marker used only before a server builder receives its authoritative store.
535#[doc(hidden)]
536#[derive(Clone, Copy, Debug, Default)]
537pub struct MissingServerStore;
538
539/// Marker used only before a server builder receives its operation executor.
540#[doc(hidden)]
541#[derive(Clone, Copy, Debug, Default)]
542pub struct MissingServerExecutor;
543
544/// Marker used only before a server builder receives its conflict resolver.
545#[doc(hidden)]
546#[derive(Clone, Copy, Debug, Default)]
547pub struct MissingServerConflicts;
548
549/// Marker used only before a server builder receives its clock.
550#[doc(hidden)]
551#[derive(Clone, Copy, Debug, Default)]
552pub struct MissingServerClock;
553
554/// Fluent type-state builder for an authoritative synchronization service.
555pub struct SyncServerBuilder<
556    S = MissingServerStore,
557    E = MissingServerExecutor,
558    R = MissingServerConflicts,
559    C = MissingServerClock,
560> {
561    store: S,
562    executor: E,
563    conflicts: R,
564    clock: C,
565    config: ServerConfig,
566    compute: Option<Arc<ComputePool>>,
567    observer: Arc<dyn Observer>,
568    authority: AuthorityController,
569    authority_explicit: bool,
570}
571
572/// Production server assembly failure.
573#[derive(Clone, Copy, Debug, Error, Eq, PartialEq)]
574pub enum ServerBuildError {
575    /// Selected authority adapter does not meet production requirements.
576    #[error(transparent)]
577    Adapter(#[from] AdapterCompatibilityError),
578    /// Production assembly requires a deployment-specific authority identity/controller.
579    #[error("production server requires an explicit authority controller")]
580    AuthorityRequired,
581}
582
583impl Default for SyncServerBuilder {
584    fn default() -> Self {
585        Self {
586            store: MissingServerStore,
587            executor: MissingServerExecutor,
588            conflicts: MissingServerConflicts,
589            clock: MissingServerClock,
590            config: ServerConfig::default(),
591            compute: None,
592            observer: Arc::new(NoopObserver),
593            authority: development_authority(),
594            authority_explicit: false,
595        }
596    }
597}
598
599impl SyncServerBuilder {
600    /// Starts an empty type-state builder.
601    #[must_use]
602    pub fn new() -> Self {
603        Self::default()
604    }
605}
606
607impl<S, E, R, C> SyncServerBuilder<S, E, R, C> {
608    /// Installs or replaces the authoritative store.
609    #[must_use]
610    pub fn store<S2>(self, store: Arc<S2>) -> SyncServerBuilder<Arc<S2>, E, R, C> {
611        SyncServerBuilder {
612            store,
613            executor: self.executor,
614            conflicts: self.conflicts,
615            clock: self.clock,
616            config: self.config,
617            compute: self.compute,
618            observer: self.observer,
619            authority: self.authority,
620            authority_explicit: self.authority_explicit,
621        }
622    }
623
624    /// Installs or replaces the typed operation executor or registry.
625    #[must_use]
626    pub fn executor<E2>(self, executor: Arc<E2>) -> SyncServerBuilder<S, Arc<E2>, R, C> {
627        SyncServerBuilder {
628            store: self.store,
629            executor,
630            conflicts: self.conflicts,
631            clock: self.clock,
632            config: self.config,
633            compute: self.compute,
634            observer: self.observer,
635            authority: self.authority,
636            authority_explicit: self.authority_explicit,
637        }
638    }
639
640    /// Installs or replaces the conflict-policy resolver.
641    #[must_use]
642    pub fn conflicts<R2>(self, conflicts: Arc<R2>) -> SyncServerBuilder<S, E, Arc<R2>, C> {
643        SyncServerBuilder {
644            store: self.store,
645            executor: self.executor,
646            conflicts,
647            clock: self.clock,
648            config: self.config,
649            compute: self.compute,
650            observer: self.observer,
651            authority: self.authority,
652            authority_explicit: self.authority_explicit,
653        }
654    }
655
656    /// Installs or replaces the hybrid timestamp clock.
657    #[must_use]
658    pub fn clock<C2>(self, clock: Arc<C2>) -> SyncServerBuilder<S, E, R, Arc<C2>> {
659        SyncServerBuilder {
660            store: self.store,
661            executor: self.executor,
662            conflicts: self.conflicts,
663            clock,
664            config: self.config,
665            compute: self.compute,
666            observer: self.observer,
667            authority: self.authority,
668            authority_explicit: self.authority_explicit,
669        }
670    }
671
672    /// Applies request-validation and pull-page settings.
673    #[must_use]
674    pub const fn config(mut self, config: ServerConfig) -> Self {
675        self.config = config;
676        self
677    }
678
679    /// Installs a dedicated pool for CPU-heavy planning.
680    #[must_use]
681    pub fn compute_pool(mut self, compute: Arc<ComputePool>) -> Self {
682        self.compute = Some(compute);
683        self
684    }
685
686    /// Installs a non-blocking, payload-free observer.
687    #[must_use]
688    pub fn observer(mut self, observer: Arc<dyn Observer>) -> Self {
689        self.observer = observer;
690        self
691    }
692
693    /// Installs authority metadata, fencing, failover, and recovery enforcement.
694    #[must_use]
695    pub fn authority(mut self, authority: AuthorityController) -> Self {
696        self.authority = authority;
697        self.authority_explicit = true;
698        self
699    }
700}
701
702impl<S, E, R, C> SyncServerBuilder<Arc<S>, Arc<E>, Arc<R>, Arc<C>>
703where
704    S: AuthoritativeStore + 'static,
705    E: OperationExecutor + 'static,
706    R: ConflictResolver + 'static,
707    C: Clock + 'static,
708{
709    /// Builds the service. This method only exists once every required component is present.
710    #[must_use]
711    pub fn build(self) -> SyncServer<S, E, R, C> {
712        SyncServer {
713            store: self.store,
714            executor: self.executor,
715            conflicts: self.conflicts,
716            clock: self.clock,
717            config: self.config,
718            compute: self.compute,
719            observer: self.observer,
720            authority: self.authority,
721        }
722    }
723}
724
725impl<S, E, R, C> SyncServerBuilder<Arc<S>, Arc<E>, Arc<R>, Arc<C>>
726where
727    S: AuthoritativeStore + AdapterManifestProvider + 'static,
728    E: OperationExecutor + 'static,
729    R: ConflictResolver + 'static,
730    C: Clock + 'static,
731{
732    /// Verifies the authority adapter's production manifest before building the server.
733    ///
734    /// Reference/test stores should continue to use [`Self::build`].
735    ///
736    /// # Errors
737    ///
738    /// Returns a typed adapter compatibility failure before any request is accepted.
739    pub fn build_production(self) -> Result<SyncServer<S, E, R, C>, ServerBuildError> {
740        if !self.authority_explicit {
741            return Err(ServerBuildError::AuthorityRequired);
742        }
743        AdapterRequirements::PRODUCTION_AUTHORITATIVE.verify(self.store.adapter_manifest())?;
744        Ok(self.build())
745    }
746}
747
748impl<S, E, R, C> SyncServer<S, E, R, C> {
749    /// Creates a server with explicit outer-layer components.
750    #[must_use]
751    pub fn new(store: Arc<S>, executor: Arc<E>, conflicts: Arc<R>, clock: Arc<C>) -> Self {
752        Self {
753            store,
754            executor,
755            conflicts,
756            clock,
757            config: ServerConfig::default(),
758            compute: None,
759            observer: Arc::new(NoopObserver),
760            authority: development_authority(),
761        }
762    }
763
764    /// Applies request limits and pull configuration.
765    #[must_use]
766    pub const fn with_config(mut self, config: ServerConfig) -> Self {
767        self.config = config;
768        self
769    }
770
771    /// Installs a dedicated Rayon pool for large CPU-bound planning workloads.
772    #[must_use]
773    pub fn with_compute_pool(mut self, compute: Arc<ComputePool>) -> Self {
774        self.compute = Some(compute);
775        self
776    }
777
778    /// Installs a non-blocking payload-free metrics and tracing observer.
779    #[must_use]
780    pub fn with_observer(mut self, observer: Arc<dyn Observer>) -> Self {
781        self.observer = observer;
782        self
783    }
784
785    /// Installs authority metadata, fencing, failover, and recovery enforcement.
786    #[must_use]
787    pub fn with_authority(mut self, authority: AuthorityController) -> Self {
788        self.authority = authority;
789        self
790    }
791
792    /// Returns the failover controller used by administrative/control-plane code.
793    #[must_use]
794    pub const fn authority(&self) -> &AuthorityController {
795        &self.authority
796    }
797}
798
799impl<S, E, R, C> SyncServer<S, E, R, C>
800where
801    S: AuthoritativeStore + 'static,
802    E: OperationExecutor + 'static,
803    R: ConflictResolver + 'static,
804    C: Clock + 'static,
805{
806    /// Executes an admin/API/background-job command through validation, authorization, domain
807    /// execution, and the atomic entity/journal/ledger/audit transaction.
808    ///
809    /// This prevents server-side callers from bypassing the synchronization journal while still
810    /// reusing the application's registered command handlers.
811    ///
812    /// # Errors
813    ///
814    /// Returns [`ServerError`] for structural, scope-authorization, conflict-merge, execution, or
815    /// persistence failures.
816    pub async fn execute_server_command(
817        &self,
818        auth: AuthContext,
819        scope_id: SyncScopeId,
820        operation: OperationEnvelope,
821    ) -> Result<ServerCommandOutcome, ServerError> {
822        let session = SessionMetadata {
823            session_id: SessionId::new(),
824            device_id: auth.device_id,
825            actor_id: auth.actor_id,
826            tenant_id: auth.tenant_id,
827            scope_id,
828            partitions: Vec::new(),
829        };
830        let validation_started = Instant::now();
831        let validation = validate_request(
832            SyncRequest {
833                protocol: operation.protocol_version,
834                request_id: RequestId::new(),
835                session: session.clone(),
836                cursor: None,
837                operations: vec![operation.clone()],
838                limits: ClientLimits {
839                    max_changes: 0,
840                    max_response_bytes: 1,
841                },
842                capabilities: Vec::new(),
843            },
844            self.config.limits,
845        );
846        self.record_phase(ServerPhaseKind::Validation, validation_started);
847        validation?;
848        self.executor
849            .authorize_scope(&auth, &session)
850            .await
851            .map_err(ServerError::ScopeAuthorization)?;
852        let outcome = self
853            .process_operation(
854                &auth,
855                &operation,
856                &HashSet::new(),
857                scope_id,
858                *blake3::hash(&operation.payload).as_bytes(),
859            )
860            .await?;
861        Ok(match outcome {
862            ProcessResult::Acknowledged(acknowledgement) => {
863                ServerCommandOutcome::Acknowledged(acknowledgement)
864            }
865            ProcessResult::Rejected(rejection) => ServerCommandOutcome::Rejected(rejection),
866            ProcessResult::Conflict(conflict) => ServerCommandOutcome::Conflict(conflict),
867        })
868    }
869
870    #[allow(clippy::too_many_lines)]
871    async fn process_operation(
872        &self,
873        auth: &AuthContext,
874        operation: &OperationEnvelope,
875        completed: &HashSet<OperationId>,
876        scope_id: SyncScopeId,
877        command_digest: [u8; 32],
878    ) -> Result<ProcessResult, ServerError> {
879        let authority_commit = self.authority.authorize_write()?;
880        let authenticated = match IncomingOperation::new(operation).authenticate(auth) {
881            Ok(authenticated) => authenticated,
882            Err(error) => {
883                return Ok(ProcessResult::Rejected(OperationRejection {
884                    operation_id: operation.operation_id,
885                    code: error.code,
886                    message: error.message,
887                }));
888            }
889        };
890        let provenance = authenticated.provenance(auth);
891
892        let database_started = Instant::now();
893        let previous = self
894            .store
895            .operation_result(auth.tenant_id, operation.operation_id)
896            .await?;
897        self.record_phase(ServerPhaseKind::Database, database_started);
898        if let Some(mut previous) = previous {
899            let database_started = Instant::now();
900            let original_lineage = self
901                .store
902                .operation_lineage(auth.tenant_id, operation.operation_id)
903                .await?;
904            self.record_phase(ServerPhaseKind::Database, database_started);
905            if original_lineage
906                != Some(
907                    operation
908                        .metadata
909                        .lineage
910                        .resolved_for_operation(operation.operation_id),
911                )
912            {
913                return Ok(ProcessResult::Rejected(rejection(
914                    operation,
915                    RejectionCode::InvalidOperation,
916                    "retry lineage differs from the original operation",
917                )));
918            }
919            self.observer.record(MetricEvent::ServerTransaction {
920                outcome: TransactionOutcomeKind::Duplicate,
921            });
922            previous.duplicate = true;
923            return Ok(ProcessResult::Acknowledged(previous));
924        }
925
926        for dependency in &operation.metadata.dependencies {
927            if completed.contains(dependency) {
928                continue;
929            }
930            let database_started = Instant::now();
931            let dependency_result = self
932                .store
933                .operation_result(auth.tenant_id, *dependency)
934                .await?;
935            self.record_phase(ServerPhaseKind::Database, database_started);
936            if dependency_result.is_none() {
937                return Ok(ProcessResult::Rejected(rejection(
938                    operation,
939                    RejectionCode::Dependency,
940                    "a required operation has not completed",
941                )));
942            }
943        }
944
945        let execution_started = Instant::now();
946        let authorization = self.executor.authorize(auth, authenticated).await;
947        self.record_phase(ServerPhaseKind::Execution, execution_started);
948        let authorized = match authorization {
949            Ok(authorized) => authorized,
950            Err(error) => {
951                return Ok(ProcessResult::Rejected(OperationRejection {
952                    operation_id: operation.operation_id,
953                    code: error.code,
954                    message: error.message,
955                }));
956            }
957        };
958
959        let database_started = Instant::now();
960        let snapshot = self
961            .store
962            .read_entity(auth.tenant_id, operation.entity)
963            .await?;
964        self.record_phase(ServerPhaseKind::Database, database_started);
965        let current = snapshot.as_ref().map(|value| &value.current);
966        let current_version = current.map(|value| value.version);
967        let merge_policy = match check_version(operation.base_version, current_version) {
968            VersionCheck::Current => None,
969            VersionCheck::Missing => {
970                return Ok(ProcessResult::Conflict(conflict(
971                    operation,
972                    current_version,
973                    self.conflicts.policy(operation.operation_kind.0),
974                    "the entity no longer exists",
975                )));
976            }
977            VersionCheck::Diverged { .. } => {
978                let policy = self.conflicts.policy(operation.operation_kind.0);
979                match policy {
980                    ConflictPolicy::ClientWins | ConflictPolicy::CommutativeOperation => None,
981                    ConflictPolicy::FieldMerge
982                    | ConflictPolicy::LastWriterWins
983                    | ConflictPolicy::CustomMerge
984                    | ConflictPolicy::Crdt => Some(policy),
985                    _ => {
986                        return Ok(ProcessResult::Conflict(conflict(
987                            operation,
988                            current_version,
989                            policy,
990                            "the operation is based on a stale entity version",
991                        )));
992                    }
993                }
994            }
995        };
996
997        let executable = authorized.validate().executable();
998        let execution_started = Instant::now();
999        let execution = self.executor.execute(auth, executable, current).await;
1000        self.record_phase(ServerPhaseKind::Execution, execution_started);
1001        let mut mutation = match execution {
1002            Ok(mutation) => mutation,
1003            Err(error) => {
1004                return Ok(ProcessResult::Rejected(OperationRejection {
1005                    operation_id: operation.operation_id,
1006                    code: error.code,
1007                    message: error.message,
1008                }));
1009            }
1010        };
1011        if let Some(policy) = merge_policy {
1012            let Some(current) = current else {
1013                return Ok(ProcessResult::Conflict(conflict(
1014                    operation,
1015                    current_version,
1016                    policy,
1017                    "the authoritative entity is unavailable for merge",
1018                )));
1019            };
1020            match self.conflicts.merge(MergeInput {
1021                operation,
1022                current_payload: &current.payload,
1023                current_tombstone: current.tombstone,
1024                candidate_payload: &mutation.payload,
1025                candidate_kind: mutation.change_kind,
1026            })? {
1027                MergeDecision::Merged {
1028                    payload,
1029                    change_kind,
1030                } => {
1031                    mutation.payload = payload;
1032                    mutation.change_kind = change_kind;
1033                }
1034                MergeDecision::Unresolved { message } => {
1035                    return Ok(ProcessResult::Conflict(conflict(
1036                        operation,
1037                        current_version,
1038                        policy,
1039                        &message,
1040                    )));
1041                }
1042            }
1043        }
1044        let next_version = match current_version {
1045            Some(version) => version.checked_next().ok_or(ServerError::VersionOverflow)?,
1046            None => EntityVersion::INITIAL,
1047        };
1048        let event = provenance.primary_event(EventId::new());
1049        let commit = CommitOperation {
1050            authority: Some(authority_commit),
1051            operation_id: operation.operation_id,
1052            event_id: event.event_id,
1053            operation_lineage: provenance.operation_lineage(),
1054            actor_id: provenance.actor_id(),
1055            device_id: provenance.device_id(),
1056            operation_kind: operation.operation_kind.0,
1057            tenant_id: provenance.tenant_id(),
1058            scope_id,
1059            entity: operation.entity,
1060            expected_version: current_version,
1061            next_version,
1062            payload: mutation.payload,
1063            change_kind: mutation.change_kind,
1064            timestamp: self.clock.now(),
1065            command_digest,
1066        };
1067        self.authority.verify_commit_context(authority_commit)?;
1068        let database_started = Instant::now();
1069        let outcome = self.store.commit_operation(commit).await;
1070        self.record_phase(ServerPhaseKind::Database, database_started);
1071        let outcome = match outcome {
1072            Ok(outcome) => outcome,
1073            Err(error) => {
1074                self.observer.record(MetricEvent::ServerTransaction {
1075                    outcome: TransactionOutcomeKind::Failed,
1076                });
1077                return Err(ServerError::Store(error));
1078            }
1079        };
1080        match outcome {
1081            CommitOutcome::Applied(ack) => {
1082                self.observer.record(MetricEvent::ServerTransaction {
1083                    outcome: TransactionOutcomeKind::Applied,
1084                });
1085                Ok(ProcessResult::Acknowledged(ack))
1086            }
1087            CommitOutcome::Duplicate(mut ack) => {
1088                self.observer.record(MetricEvent::ServerTransaction {
1089                    outcome: TransactionOutcomeKind::Duplicate,
1090                });
1091                ack.duplicate = true;
1092                Ok(ProcessResult::Acknowledged(ack))
1093            }
1094            CommitOutcome::VersionChanged { current } => {
1095                self.observer.record(MetricEvent::ServerTransaction {
1096                    outcome: TransactionOutcomeKind::VersionChanged,
1097                });
1098                Ok(ProcessResult::Conflict(conflict(
1099                    operation,
1100                    current,
1101                    self.conflicts.policy(operation.operation_kind.0),
1102                    "the entity changed while this operation was being committed",
1103                )))
1104            }
1105            CommitOutcome::Rejected(rejection) => Ok(ProcessResult::Rejected(rejection)),
1106        }
1107    }
1108
1109    #[allow(clippy::too_many_lines)]
1110    async fn process(
1111        &self,
1112        auth: AuthContext,
1113        request: SyncRequest,
1114    ) -> Result<SyncResponse, ServerError> {
1115        let authority = self.authority.state()?;
1116        if request.protocol < self.config.limits.minimum_protocol
1117            || request.protocol > self.config.limits.current_protocol
1118        {
1119            return Ok(SyncResponse {
1120                protocol: self.config.limits.current_protocol,
1121                directive: SyncDirective::UpgradeRequired {
1122                    minimum: self.config.limits.minimum_protocol,
1123                    current: self.config.limits.current_protocol,
1124                },
1125                acknowledged: Vec::new(),
1126                rejected: Vec::new(),
1127                conflicts: Vec::new(),
1128                changes: Vec::new(),
1129                next_cursor: request
1130                    .cursor
1131                    .unwrap_or_else(|| authority.cursor(request.session.scope_id, Sequence(0))),
1132                has_more: false,
1133                server_time: self.clock.now(),
1134            });
1135        }
1136        let validation_started = Instant::now();
1137        let validation = validate_request(request, self.config.limits);
1138        self.record_phase(ServerPhaseKind::Validation, validation_started);
1139        let mut request = validation?.into_inner();
1140        self.validate_typed_request_budget(&request)?;
1141        if request.session.tenant_id != auth.tenant_id
1142            || request.session.actor_id != auth.actor_id
1143            || request.session.device_id != auth.device_id
1144        {
1145            return Err(ServerError::IdentityMismatch);
1146        }
1147        self.executor
1148            .authorize_scope(&auth, &request.session)
1149            .await
1150            .map_err(ServerError::ScopeAuthorization)?;
1151
1152        if let Some(cursor) = request.cursor {
1153            match validate_cursor(authority, cursor)? {
1154                CursorDisposition::Continue => {}
1155                CursorDisposition::AuthorityChanged { previous, current } => {
1156                    return Ok(SyncResponse {
1157                        protocol: request.protocol,
1158                        directive: SyncDirective::AuthorityChanged {
1159                            authority_id: authority.authority_id,
1160                            previous_epoch: previous,
1161                            current_epoch: current,
1162                        },
1163                        acknowledged: Vec::new(),
1164                        rejected: Vec::new(),
1165                        conflicts: Vec::new(),
1166                        changes: Vec::new(),
1167                        next_cursor: authority.cursor(request.session.scope_id, Sequence(0)),
1168                        has_more: false,
1169                        server_time: self.clock.now(),
1170                    });
1171                }
1172            }
1173        }
1174
1175        let start = request.cursor.map_or(Sequence(0), |cursor| cursor.sequence);
1176        let database_started = Instant::now();
1177        let minimum_cursor = self
1178            .store
1179            .minimum_retained_cursor(auth.tenant_id, request.session.scope_id)
1180            .await?;
1181        self.record_phase(ServerPhaseKind::Database, database_started);
1182        if start < minimum_cursor {
1183            return Ok(SyncResponse {
1184                protocol: request.protocol,
1185                directive: SyncDirective::ResyncRequired {
1186                    reason: ResyncReason::CursorExpired,
1187                },
1188                acknowledged: Vec::new(),
1189                rejected: Vec::new(),
1190                conflicts: Vec::new(),
1191                changes: Vec::new(),
1192                next_cursor: authority.cursor(request.session.scope_id, minimum_cursor),
1193                has_more: false,
1194                server_time: self.clock.now(),
1195            });
1196        }
1197
1198        let operations = std::mem::take(&mut request.operations);
1199        let (operations, dependency_plan, command_digests) =
1200            self.prepare_operations(operations).await?;
1201        let mut acknowledged = Vec::with_capacity(operations.len());
1202        let mut rejected = Vec::new();
1203        let mut conflicts = Vec::new();
1204        let mut completed = HashSet::new();
1205        for &operation_index in dependency_plan.ordered_indices() {
1206            let operation = &operations[operation_index];
1207            match self
1208                .process_operation(
1209                    &auth,
1210                    operation,
1211                    &completed,
1212                    request.session.scope_id,
1213                    command_digests[operation_index],
1214                )
1215                .await?
1216            {
1217                ProcessResult::Acknowledged(ack) => {
1218                    completed.insert(ack.operation_id);
1219                    acknowledged.push(ack);
1220                }
1221                ProcessResult::Rejected(rejection) => rejected.push(rejection),
1222                ProcessResult::Conflict(conflict) => conflicts.push(conflict),
1223            }
1224        }
1225
1226        let requested_limit = usize::try_from(request.limits.max_changes).unwrap_or(usize::MAX);
1227        let database_started = Instant::now();
1228        let page = self
1229            .store
1230            .read_changes_after(
1231                auth.tenant_id,
1232                request.session.scope_id,
1233                start,
1234                requested_limit.min(self.config.max_pull_changes),
1235                usize::try_from(request.limits.max_response_bytes).unwrap_or(usize::MAX),
1236            )
1237            .await?;
1238        self.record_phase(ServerPhaseKind::Database, database_started);
1239        let journal_head = page.journal_head;
1240        let mut response = SyncResponse {
1241            protocol: request.protocol,
1242            directive: SyncDirective::Continue,
1243            acknowledged,
1244            rejected,
1245            conflicts,
1246            changes: page.changes,
1247            next_cursor: authority.cursor(request.session.scope_id, page.next_sequence),
1248            has_more: page.has_more,
1249            server_time: self.clock.now(),
1250        };
1251        self.fit_response(&mut response, start, request.limits.max_response_bytes)?;
1252        self.observer.record(MetricEvent::ServerJournalLag {
1253            sequences: journal_head
1254                .0
1255                .saturating_sub(response.next_cursor.sequence.0),
1256        });
1257        Ok(response)
1258    }
1259
1260    fn fit_response(
1261        &self,
1262        response: &mut SyncResponse,
1263        start: Sequence,
1264        maximum_bytes: u32,
1265    ) -> Result<(), ServerError> {
1266        let maximum = usize::try_from(maximum_bytes)
1267            .unwrap_or(usize::MAX)
1268            .min(self.config.performance.memory.response_bytes);
1269        loop {
1270            let encoded = aequora_codec::encode(
1271                response.protocol,
1272                aequora_codec::MessageKind::SyncResponse,
1273                response,
1274            )?;
1275            if encoded.len() <= maximum {
1276                return Ok(());
1277            }
1278            if response.changes.pop().is_none() {
1279                return Err(ServerError::ResponseLimit);
1280            }
1281            response.has_more = true;
1282            response.next_cursor.sequence = response
1283                .changes
1284                .last()
1285                .map_or(start, |change| change.sequence);
1286            if response.changes.is_empty()
1287                && response.acknowledged.is_empty()
1288                && response.rejected.is_empty()
1289                && response.conflicts.is_empty()
1290            {
1291                return Err(ServerError::ResponseLimit);
1292            }
1293        }
1294    }
1295
1296    fn validate_typed_request_budget(&self, request: &SyncRequest) -> Result<(), ServerError> {
1297        if request.operations.len() > self.config.performance.memory.pending_decode_records {
1298            return Err(ServerError::MemoryBudget {
1299                domain: "pending decode records",
1300                actual: request.operations.len(),
1301                maximum: self.config.performance.memory.pending_decode_records,
1302            });
1303        }
1304        let bytes = request
1305            .operations
1306            .iter()
1307            .try_fold(0_usize, |total, operation| {
1308                total.checked_add(operation.payload.len())
1309            });
1310        let actual = bytes.unwrap_or(usize::MAX);
1311        if actual > self.config.performance.memory.decode_bytes {
1312            return Err(ServerError::MemoryBudget {
1313                domain: "typed request payload",
1314                actual,
1315                maximum: self.config.performance.memory.decode_bytes,
1316            });
1317        }
1318        Ok(())
1319    }
1320
1321    async fn prepare_operations(
1322        &self,
1323        operations: Vec<OperationEnvelope>,
1324    ) -> Result<(Vec<OperationEnvelope>, DependencyPlan, Vec<[u8; 32]>), ServerError> {
1325        let Some(compute) = &self.compute else {
1326            let plan = plan_dependencies(&operations)?;
1327            let digests = operations
1328                .iter()
1329                .map(|operation| *blake3::hash(&operation.payload).as_bytes())
1330                .collect();
1331            return Ok((operations, plan, digests));
1332        };
1333        if !compute.should_parallelize(operations.len()) {
1334            let plan = plan_dependencies(&operations)?;
1335            let digests = operations
1336                .iter()
1337                .map(|operation| *blake3::hash(&operation.payload).as_bytes())
1338                .collect();
1339            return Ok((operations, plan, digests));
1340        }
1341        self.observer.record(MetricEvent::ComputeOffload {
1342            items: usize_to_u64(operations.len()),
1343        });
1344        Ok(compute
1345            .run(move || {
1346                let plan = plan_dependencies(&operations)?;
1347                let digests = operations
1348                    .par_iter()
1349                    .map(|operation| *blake3::hash(&operation.payload).as_bytes())
1350                    .collect();
1351                Ok::<_, DependencyError>((operations, plan, digests))
1352            })
1353            .await??)
1354    }
1355
1356    fn record_phase(&self, phase: ServerPhaseKind, started: Instant) {
1357        self.observer.record(MetricEvent::ServerPhase {
1358            phase,
1359            duration_micros: duration_micros(started.elapsed()),
1360        });
1361    }
1362
1363    async fn process_bootstrap(
1364        &self,
1365        auth: AuthContext,
1366        request: BootstrapRequest,
1367    ) -> Result<BootstrapResponse, ServerError> {
1368        let authority = self.authority.state()?;
1369        self.authority.validate_startup()?;
1370        let validation_started = Instant::now();
1371        let validation = validate_bootstrap_request(&request, self.config.limits);
1372        self.record_phase(ServerPhaseKind::Validation, validation_started);
1373        validation?;
1374        if request.session.tenant_id != auth.tenant_id
1375            || request.session.actor_id != auth.actor_id
1376            || request.session.device_id != auth.device_id
1377        {
1378            return Err(ServerError::IdentityMismatch);
1379        }
1380        self.executor
1381            .authorize_scope(&auth, &request.session)
1382            .await
1383            .map_err(ServerError::ScopeAuthorization)?;
1384        let snapshot_id = if let Some(snapshot_id) = request.snapshot_id {
1385            snapshot_id
1386        } else {
1387            let database_started = Instant::now();
1388            let descriptor = self
1389                .store
1390                .create_snapshot_in_timeline(
1391                    auth.tenant_id,
1392                    request.session.scope_id,
1393                    &request.session.partitions,
1394                    authority.authority_id,
1395                    authority.epoch,
1396                )
1397                .await?;
1398            self.record_phase(ServerPhaseKind::Database, database_started);
1399            descriptor.snapshot_id
1400        };
1401        let database_started = Instant::now();
1402        let page = self
1403            .store
1404            .read_snapshot(
1405                auth.tenant_id,
1406                snapshot_id,
1407                request.offset,
1408                usize::try_from(request.limits.max_entities)
1409                    .unwrap_or(usize::MAX)
1410                    .min(self.config.performance.snapshot.max_records_per_chunk),
1411                usize::try_from(request.limits.max_payload_bytes)
1412                    .unwrap_or(usize::MAX)
1413                    .min(self.config.performance.snapshot.max_chunk_bytes),
1414            )
1415            .await?;
1416        self.record_phase(ServerPhaseKind::Database, database_started);
1417        if page.descriptor.cursor.authority_id != AuthorityId::LEGACY_UNBOUND {
1418            if page.descriptor.cursor.authority_id != authority.authority_id {
1419                return Err(AuthorityError::AuthorityIdChanged {
1420                    trusted: authority.authority_id,
1421                    presented: page.descriptor.cursor.authority_id,
1422                }
1423                .into());
1424            }
1425            if page.descriptor.cursor.authority_epoch != authority.epoch {
1426                return Err(AuthorityError::ArtifactEpochMismatch {
1427                    artifact: page.descriptor.cursor.authority_epoch,
1428                    current: authority.epoch,
1429                }
1430                .into());
1431            }
1432        }
1433        if page.descriptor.cursor.scope != request.session.scope_id {
1434            return Err(ServerError::IdentityMismatch);
1435        }
1436        if page.has_more && page.entities.is_empty() && page.next_offset == request.offset {
1437            return Err(ServerError::SnapshotNoProgress);
1438        }
1439        Ok(BootstrapResponse {
1440            protocol: request.protocol,
1441            snapshot_id: page.descriptor.snapshot_id,
1442            cursor: authority.cursor(request.session.scope_id, page.descriptor.cursor.sequence),
1443            offset: request.offset,
1444            entities: page.entities,
1445            next_offset: page.next_offset,
1446            has_more: page.has_more,
1447            server_time: self.clock.now(),
1448        })
1449    }
1450}
1451
1452#[async_trait]
1453impl<S, E, R, C> ExchangeService for SyncServer<S, E, R, C>
1454where
1455    S: AuthoritativeStore + 'static,
1456    E: OperationExecutor + 'static,
1457    R: ConflictResolver + 'static,
1458    C: Clock + 'static,
1459{
1460    async fn exchange(
1461        &self,
1462        auth: AuthContext,
1463        request: SyncRequest,
1464    ) -> Result<SyncResponse, ServerError> {
1465        let operations = request.operations.len();
1466        let trace = TraceContext {
1467            sync_session_id: request.session.session_id,
1468            request_id: request.request_id,
1469            device_id: request.session.device_id,
1470            tenant_id: request.session.tenant_id,
1471        };
1472        let started = Instant::now();
1473        let result = self.process(auth, request).await;
1474        let (changes, conflicts, rejections) = result.as_ref().map_or((0, 0, 0), |response| {
1475            (
1476                response.changes.len(),
1477                response.conflicts.len(),
1478                response.rejected.len(),
1479            )
1480        });
1481        self.observer.record_with_context(
1482            trace,
1483            MetricEvent::ServerExchange {
1484                duration_micros: duration_micros(started.elapsed()),
1485                operations: usize_to_u64(operations),
1486                changes: usize_to_u64(changes),
1487                conflicts: usize_to_u64(conflicts),
1488                rejections: usize_to_u64(rejections),
1489                outcome: server_result_outcome(&result),
1490            },
1491        );
1492        result
1493    }
1494
1495    async fn bootstrap(
1496        &self,
1497        auth: AuthContext,
1498        request: BootstrapRequest,
1499    ) -> Result<BootstrapResponse, ServerError> {
1500        let trace = TraceContext {
1501            sync_session_id: request.session.session_id,
1502            request_id: request.request_id,
1503            device_id: request.session.device_id,
1504            tenant_id: request.session.tenant_id,
1505        };
1506        let started = Instant::now();
1507        let result = self.process_bootstrap(auth, request).await;
1508        self.observer.record_with_context(
1509            trace,
1510            MetricEvent::BootstrapPage {
1511                duration_micros: duration_micros(started.elapsed()),
1512                entities: result
1513                    .as_ref()
1514                    .map_or(0, |response| usize_to_u64(response.entities.len())),
1515                outcome: server_result_outcome(&result),
1516            },
1517        );
1518        result
1519    }
1520}
1521
1522fn duration_micros(duration: std::time::Duration) -> u64 {
1523    u64::try_from(duration.as_micros()).unwrap_or(u64::MAX)
1524}
1525
1526fn usize_to_u64(value: usize) -> u64 {
1527    u64::try_from(value).unwrap_or(u64::MAX)
1528}
1529
1530fn development_authority() -> AuthorityController {
1531    let mut state = AuthorityState::new(
1532        AuthorityId::LOCAL_DEVELOPMENT,
1533        AuthorityInstanceId::new(),
1534        AuthorityRole::Primary,
1535        0,
1536    );
1537    state.runtime_mode = AuthorityRuntimeMode::Serving;
1538    AuthorityController::new(state, AuthorityPromotionPolicy::default())
1539}
1540
1541fn server_result_outcome<T>(result: &Result<T, ServerError>) -> OutcomeKind {
1542    match result {
1543        Ok(_) => OutcomeKind::Success,
1544        Err(ServerError::Store(error)) if error.kind == StoreErrorKind::Transient => {
1545            OutcomeKind::TransientFailure
1546        }
1547        Err(ServerError::Maintenance { .. }) => OutcomeKind::TransientFailure,
1548        Err(ServerError::Admission(rejection)) if rejection.retryable() => {
1549            OutcomeKind::TransientFailure
1550        }
1551        Err(_) => OutcomeKind::PermanentFailure,
1552    }
1553}
1554
1555enum ProcessResult {
1556    Acknowledged(OperationAck),
1557    Rejected(OperationRejection),
1558    Conflict(Conflict),
1559}
1560
1561fn rejection(
1562    operation: &OperationEnvelope,
1563    code: RejectionCode,
1564    message: &str,
1565) -> OperationRejection {
1566    OperationRejection {
1567        operation_id: operation.operation_id,
1568        code,
1569        message: message.to_owned(),
1570    }
1571}
1572
1573fn conflict(
1574    operation: &OperationEnvelope,
1575    current: Option<EntityVersion>,
1576    policy: ConflictPolicy,
1577    message: &str,
1578) -> Conflict {
1579    Conflict {
1580        operation_id: operation.operation_id,
1581        entity: operation.entity,
1582        client_base: operation.base_version,
1583        server_version: current,
1584        policy,
1585        message: message.to_owned(),
1586    }
1587}
1588
1589#[cfg(test)]
1590mod tests {
1591    use std::sync::atomic::{AtomicUsize, Ordering};
1592
1593    use super::*;
1594
1595    struct CountingService(AtomicUsize);
1596
1597    #[async_trait]
1598    impl ExchangeService for CountingService {
1599        async fn exchange(
1600            &self,
1601            _auth: AuthContext,
1602            _request: SyncRequest,
1603        ) -> Result<SyncResponse, ServerError> {
1604            self.0.fetch_add(1, Ordering::Relaxed);
1605            Err(ServerError::BootstrapUnavailable)
1606        }
1607    }
1608
1609    fn empty_request(auth: AuthContext) -> SyncRequest {
1610        SyncRequest {
1611            protocol: aequora_types::ProtocolVersion::V1,
1612            request_id: aequora_types::RequestId::new(),
1613            session: SessionMetadata {
1614                session_id: aequora_types::SessionId::new(),
1615                device_id: auth.device_id,
1616                actor_id: auth.actor_id,
1617                tenant_id: auth.tenant_id,
1618                scope_id: aequora_types::SyncScopeId::new(),
1619                partitions: Vec::new(),
1620            },
1621            cursor: None,
1622            operations: Vec::new(),
1623            limits: ClientLimits::default(),
1624            capabilities: Vec::new(),
1625        }
1626    }
1627
1628    #[test]
1629    fn maintenance_controller_transitions_atomically() {
1630        let controller = MaintenanceController::default();
1631        assert_eq!(controller.mode(), MaintenanceMode::Normal);
1632        assert_eq!(
1633            controller.set_mode(MaintenanceMode::ReadOnly),
1634            MaintenanceMode::Normal
1635        );
1636        assert_eq!(controller.mode(), MaintenanceMode::ReadOnly);
1637        assert_eq!(
1638            controller.set_mode(MaintenanceMode::SyncPaused),
1639            MaintenanceMode::ReadOnly
1640        );
1641        assert_eq!(controller.mode(), MaintenanceMode::SyncPaused);
1642    }
1643
1644    #[test]
1645    fn maintenance_policy_preserves_read_only_pull_and_blocks_writes() {
1646        assert!(MaintenanceMode::Normal.allows_exchange(false));
1647        assert!(MaintenanceMode::Normal.allows_exchange(true));
1648        assert!(MaintenanceMode::ReadOnly.allows_exchange(false));
1649        assert!(!MaintenanceMode::ReadOnly.allows_exchange(true));
1650        assert!(!MaintenanceMode::SyncPaused.allows_exchange(false));
1651        assert!(!MaintenanceMode::SyncPaused.allows_exchange(true));
1652        assert!(MaintenanceMode::Normal.allows_bootstrap());
1653        assert!(MaintenanceMode::ReadOnly.allows_bootstrap());
1654        assert!(!MaintenanceMode::SyncPaused.allows_bootstrap());
1655    }
1656
1657    #[tokio::test]
1658    async fn admission_decorator_rejects_before_inner_service_and_releases_with_raii() {
1659        let mut policy = aequora_admission::AdmissionPolicy::default();
1660        policy.global.max_in_flight = 1;
1661        policy.tenant.budget.max_in_flight = 1;
1662        policy.tenant.max_tracked_tenants = 1;
1663        policy.critical.reserved_in_flight = 0;
1664        policy.interactive.reserved_in_flight = 0;
1665        policy.normal.reserved_in_flight = 0;
1666        policy.bulk.reserved_in_flight = 0;
1667        policy.background.reserved_in_flight = 0;
1668        policy.maintenance.reserved_in_flight = 0;
1669        let admission = Arc::new(
1670            aequora_admission::HierarchicalAdmission::new(policy)
1671                .unwrap_or_else(|error| panic!("{error}")),
1672        );
1673        let inner = Arc::new(CountingService(AtomicUsize::new(0)));
1674        let service = AdmittedExchangeService::new(Arc::clone(&inner), admission.clone());
1675        let auth = AuthContext {
1676            actor_id: aequora_types::ActorId::new(),
1677            tenant_id: aequora_types::TenantId::new(),
1678            device_id: aequora_types::DeviceId::new(),
1679        };
1680        let request = empty_request(auth);
1681        let blocker = admission
1682            .try_admit(&service.exchange_work(&auth, &request))
1683            .unwrap_or_else(|error| panic!("{error}"));
1684        assert!(matches!(
1685            service.exchange(auth, request.clone()).await,
1686            Err(ServerError::Admission(
1687                AdmissionRejection::ServerBusy { .. }
1688            ))
1689        ));
1690        assert_eq!(inner.0.load(Ordering::Relaxed), 0);
1691        drop(blocker);
1692        assert!(matches!(
1693            service.exchange(auth, request).await,
1694            Err(ServerError::BootstrapUnavailable)
1695        ));
1696        assert_eq!(inner.0.load(Ordering::Relaxed), 1);
1697    }
1698}