1use std::collections::VecDeque;
2
3use ozzy_journal::progress::{JournalProgress, JournalSnapshot};
4use ozzy_proto::NodeId;
5
6use crate::{
7 Commit, Configuration, JournalGeneration, OpNumber, PipelineLimits, Prefix, PrepareOk,
8 PreparedOperation, QuorumPolicy, ReplicationError, RetainedPrepareOk, Scope, SyncTicket,
9 WriteTicket,
10};
11
12#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14pub enum Status {
15 Normal,
17 Fenced,
19 Faulted,
21}
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
25pub enum Admission {
26 Duplicate,
28 Write {
30 ticket: WriteTicket,
32 first_new: usize,
34 },
35}
36
37#[derive(Debug, Clone, Copy, PartialEq, Eq)]
39pub struct ReplicaSnapshot {
40 pub scope: Scope,
42 pub status: Status,
44 pub journal: JournalSnapshot,
46 pub accepted: Prefix,
48 pub committed: Prefix,
50 pub applied: Prefix,
52 pub pending_operations: usize,
55 pub pending_body_bytes: usize,
57 pub journal_tail: Option<Prefix>,
60 pub ready_for_appends: bool,
64}
65
66#[derive(Debug)]
74pub struct NormalReplica {
75 configuration: Configuration,
76 local: usize,
77 status: Status,
78 journal: JournalProgress,
79 limits: PipelineLimits,
80 operations: VecDeque<PreparedOperation>,
81 body_bytes: usize,
82 accepted: Prefix,
83 committed: Prefix,
84 applied: Prefix,
85 retained_base: Prefix,
87 known_commit: OpNumber,
88 acknowledged: [OpNumber; 3],
89 activation_tail: OpNumber,
90 installed_view: Option<crate::StartView>,
91 journal_tail: Option<Prefix>,
92}
93
94impl NormalReplica {
95 pub const fn reclaimed_prefix(&self) -> Prefix {
97 self.retained_base
98 }
99 pub(crate) fn driver_identity(&self) -> (Configuration, NodeId) {
100 (self.configuration, self.configuration.voters[self.local])
101 }
102
103 pub fn into_view_change(self, view: u64) -> Result<crate::ViewChange, crate::ViewChangeError> {
109 let configuration = self.configuration;
110 let local = self.local;
111 crate::ViewChange::begin(self, configuration, local, view)
112 }
113
114 pub fn bootstrap(
121 configuration: Configuration,
122 local: NodeId,
123 generation: JournalGeneration,
124 limits: PipelineLimits,
125 ) -> Result<Self, ReplicationError> {
126 let local = configuration.voter_index(local)?;
127 if limits.max_operations == 0
128 || limits.max_body_bytes == 0
129 || u64::try_from(limits.max_operations).is_err()
130 {
131 return Err(ReplicationError::InvalidLimits);
132 }
133 let mut operations = VecDeque::new();
134 operations
135 .try_reserve_exact(limits.max_operations)
136 .map_err(|_| ReplicationError::Capacity)?;
137 Ok(Self {
138 configuration,
139 local,
140 status: Status::Normal,
141 journal: JournalProgress::recover(generation, OpNumber(0)),
142 limits,
143 operations,
144 body_bytes: 0,
145 accepted: Prefix::GENESIS,
146 committed: Prefix::GENESIS,
147 applied: Prefix::GENESIS,
148 retained_base: Prefix::GENESIS,
149 known_commit: OpNumber(0),
150 acknowledged: [OpNumber(0); 3],
151 activation_tail: OpNumber(0),
152 installed_view: None,
153 journal_tail: None,
154 })
155 }
156
157 pub(crate) fn recover_fenced(
160 mut configuration: Configuration,
161 local: NodeId,
162 generation: JournalGeneration,
163 log: crate::FrozenLog,
164 limits: PipelineLimits,
165 ) -> Result<Self, ReplicationError> {
166 configuration.scope.view = log.last_normal_view;
167 let mut normal = Self::bootstrap(configuration, local, generation, limits)?;
168 normal.restore_fenced(generation, log);
169 Ok(normal)
170 }
171
172 pub(crate) fn restore_fenced(&mut self, generation: JournalGeneration, log: crate::FrozenLog) {
175 self.configuration.scope.view = log.last_normal_view;
176 self.status = Status::Fenced;
177 self.journal = JournalProgress::recover(generation, log.accepted.op);
178 self.operations.clear();
179 self.body_bytes = 0;
180 self.accepted = log.accepted;
181 self.committed = log.committed;
182 self.applied = Prefix::GENESIS;
183 self.retained_base = Prefix::GENESIS;
184 self.known_commit = log.committed.op;
185 self.acknowledged = [OpNumber(0); 3];
186 self.activation_tail = log.accepted.op;
187 self.installed_view = None;
188 self.journal_tail = (log.accepted != Prefix::GENESIS).then_some(log.accepted);
189 }
190
191 pub fn snapshot(&self) -> ReplicaSnapshot {
193 ReplicaSnapshot {
194 scope: self.configuration.scope(),
195 status: self.status,
196 journal: self.journal.snapshot(),
197 accepted: self.accepted,
198 committed: self.committed,
199 applied: self.applied,
200 pending_operations: self.operations.len(),
201 pending_body_bytes: self.body_bytes,
202 journal_tail: self.journal_tail,
203 ready_for_appends: self.ready_for_appends(),
204 }
205 }
206
207 pub fn prepare(
214 &mut self,
215 from: NodeId,
216 scope: Scope,
217 operations: &[PreparedOperation],
218 ) -> Result<Admission, ReplicationError> {
219 self.require_scope(scope)?;
220 self.require_primary(from)?;
221 if operations.is_empty() {
222 return Err(ReplicationError::EmptyPrepare);
223 }
224 if operations.len() > self.limits.max_operations {
225 return Err(ReplicationError::Capacity);
226 }
227
228 let mut predecessor = operations[0]
229 .prefix
230 .op
231 .0
232 .checked_sub(1)
233 .ok_or(ReplicationError::HistoryGap)
234 .and_then(|op| self.prefix_at(OpNumber(op)))?;
235 let mut first_new = operations.len();
236 let mut additional_bytes = 0usize;
237 for (index, operation) in operations.iter().enumerate() {
238 if operation.group_id != scope.group_id
239 || operation.configuration_epoch != scope.configuration_epoch
240 || operation.original_view > scope.view
241 || (operation.prefix.op > self.accepted.op && operation.original_view != scope.view)
242 {
243 return Err(ReplicationError::ScopeMismatch);
244 }
245 if predecessor.op.0.checked_add(1) != Some(operation.prefix.op.0) {
246 return Err(ReplicationError::HistoryGap);
247 }
248 if operation.previous_digest != predecessor.digest {
249 return Err(self.conflict());
250 }
251 if operation.prefix.op <= self.accepted.op {
252 if self.prefix_at(operation.prefix.op)? != operation.prefix {
253 return Err(self.conflict());
254 }
255 } else {
256 first_new = first_new.min(index);
257 additional_bytes = additional_bytes
258 .checked_add(operation.body_bytes)
259 .ok_or(ReplicationError::Capacity)?;
260 }
261 predecessor = operation.prefix;
262 }
263 if first_new == operations.len() {
264 return Ok(Admission::Duplicate);
265 }
266 if !self.ready_for_appends() {
267 return Err(ReplicationError::ActivationPending);
268 }
269 let new_count = operations.len() - first_new;
270 if new_count > self.limits.max_operations - self.operations.len()
271 || additional_bytes > self.limits.max_body_bytes - self.body_bytes
272 {
273 return Err(ReplicationError::Capacity);
274 }
275 let ticket = self.journal.admit(new_count as u64)?;
276 self.operations.extend(&operations[first_new..]);
277 self.body_bytes += additional_bytes;
278 self.accepted = predecessor;
279 Ok(Admission::Write { ticket, first_new })
280 }
281
282 pub fn complete_write(&mut self, ticket: WriteTicket) -> Result<(), ReplicationError> {
284 self.journal.complete_write(ticket)?;
285 self.reclaim();
286 Ok(())
287 }
288
289 pub fn stored_through(&self) -> OpNumber {
293 match self.configuration.policy() {
294 QuorumPolicy::Durable => self.journal.snapshot().durable,
295 QuorumPolicy::Replicated => self.journal.snapshot().written,
296 }
297 }
298
299 pub fn begin_sync(&self) -> Result<SyncTicket, ReplicationError> {
301 Ok(self.journal.begin_sync()?)
302 }
303
304 pub fn complete_sync(&mut self, ticket: SyncTicket) -> Result<(), ReplicationError> {
307 self.journal.complete_sync(ticket)?;
308 self.advance_commit();
309 self.reclaim();
310 Ok(())
311 }
312
313 pub fn complete_durable_write(&mut self, ticket: WriteTicket) -> Result<(), ReplicationError> {
315 self.journal.complete_durable_write(ticket)?;
316 self.advance_commit();
317 self.reclaim();
318 Ok(())
319 }
320
321 pub fn acknowledgment(&self) -> Result<PrepareOk, ReplicationError> {
323 self.require_normal()?;
324 if self.configuration.policy() != QuorumPolicy::Durable {
325 return Err(ReplicationError::PolicyMismatch);
326 }
327 if self.is_primary() {
328 return Err(ReplicationError::WrongRole);
329 }
330 Ok(PrepareOk {
331 scope: self.configuration.scope(),
332 durable: self.prefix_at(self.journal.snapshot().durable)?,
333 })
334 }
335
336 pub fn receive_ack(&mut self, from: NodeId, ack: PrepareOk) -> Result<(), ReplicationError> {
339 if self.configuration.policy() != QuorumPolicy::Durable {
340 return Err(ReplicationError::PolicyMismatch);
341 }
342 self.receive_prefix(from, ack.scope, ack.durable)
343 }
344
345 pub fn retained_acknowledgment(&self) -> Result<RetainedPrepareOk, ReplicationError> {
348 self.require_normal()?;
349 if self.configuration.policy() != QuorumPolicy::Replicated {
350 return Err(ReplicationError::PolicyMismatch);
351 }
352 if self.is_primary() {
353 return Err(ReplicationError::WrongRole);
354 }
355 Ok(RetainedPrepareOk {
356 scope: self.configuration.scope(),
357 retained: self.accepted,
358 })
359 }
360
361 pub fn receive_retained_ack(
363 &mut self,
364 from: NodeId,
365 ack: RetainedPrepareOk,
366 ) -> Result<(), ReplicationError> {
367 if self.configuration.policy() != QuorumPolicy::Replicated {
368 return Err(ReplicationError::PolicyMismatch);
369 }
370 self.receive_prefix(from, ack.scope, ack.retained)
371 }
372
373 fn receive_prefix(
374 &mut self,
375 from: NodeId,
376 scope: Scope,
377 prefix: Prefix,
378 ) -> Result<(), ReplicationError> {
379 self.require_scope(scope)?;
380 let voter = self.configuration.voter_index(from)?;
381 if !self.is_primary() || voter == self.local {
382 return Err(ReplicationError::WrongRole);
383 }
384 if prefix.op < self.applied.op {
385 return Ok(());
386 }
387 if self.prefix_at(prefix.op)? != prefix {
388 return Err(self.conflict());
389 }
390 self.acknowledged[voter] = self.acknowledged[voter].max(prefix.op);
391 self.advance_commit();
392 Ok(())
393 }
394
395 pub fn announcement(&self) -> Result<Commit, ReplicationError> {
397 self.require_normal()?;
398 if !self.is_primary() {
399 return Err(ReplicationError::WrongRole);
400 }
401 Ok(Commit {
402 scope: self.configuration.scope(),
403 committed: self.committed,
404 })
405 }
406
407 pub fn start_view(&self) -> Result<Option<crate::StartView>, ReplicationError> {
411 self.require_normal()?;
412 if !self.is_primary() {
413 return Err(ReplicationError::WrongRole);
414 }
415 Ok(self.installed_view)
416 }
417
418 pub(crate) fn activation_prefix(&self) -> Result<Prefix, ReplicationError> {
419 self.require_normal()?;
420 let snapshot = self.snapshot();
421 if self
422 .installed_view
423 .is_none_or(|installed| installed.accepted != snapshot.accepted)
424 || snapshot.committed != snapshot.accepted
425 || snapshot.journal.durable < snapshot.accepted.op
426 || (self.is_primary()
427 && self.acknowledged[(self.local + 1) % 3]
428 .max(self.acknowledged[(self.local + 2) % 3])
429 < snapshot.accepted.op)
430 {
431 return Err(ReplicationError::ActivationPending);
432 }
433 Ok(snapshot.accepted)
434 }
435
436 pub(crate) fn observe_installed_start(
437 &mut self,
438 from: NodeId,
439 start: crate::StartView,
440 ) -> Result<(), ReplicationError> {
441 self.require_scope(start.scope)?;
442 self.require_primary(from)?;
443 if self.is_primary() {
444 return Err(ReplicationError::WrongRole);
445 }
446 if self.installed_view != Some(start) {
447 return Err(self.conflict());
448 }
449 Ok(())
450 }
451
452 pub fn receive_commit(&mut self, from: NodeId, commit: Commit) -> Result<(), ReplicationError> {
456 self.require_scope(commit.scope)?;
457 self.require_primary(from)?;
458 if self.is_primary() {
459 return Err(ReplicationError::WrongRole);
460 }
461 if commit.committed.op < self.applied.op {
462 return Ok(());
463 }
464 if self.prefix_at(commit.committed.op)? != commit.committed {
465 return Err(self.conflict());
466 }
467 self.known_commit = self.known_commit.max(commit.committed.op);
468 self.advance_commit();
469 Ok(())
470 }
471
472 pub fn apply_through(&mut self, through: Prefix) -> Result<(), ReplicationError> {
478 self.require_normal()?;
479 if through.op < self.applied.op {
480 return Err(ReplicationError::HistoryUnavailable);
481 }
482 if through.op > self.committed.op {
483 return Err(ReplicationError::ApplyBeyondCommit);
484 }
485 if self.prefix_at(through.op)? != through {
486 return Err(self.conflict());
487 }
488 self.applied = through;
489 self.reclaim();
490 if self.journal_tail.is_some_and(|tail| through.op >= tail.op) {
491 self.journal_tail = None;
492 }
493 Ok(())
494 }
495
496 pub fn fence(&mut self) {
500 if self.status == Status::Normal {
501 self.status = Status::Fenced;
502 }
503 }
504
505 pub fn fail_io(&mut self, generation: JournalGeneration) -> Result<(), ReplicationError> {
507 self.journal.fail(generation)?;
508 self.status = Status::Faulted;
509 Ok(())
510 }
511
512 fn is_primary(&self) -> bool {
513 self.configuration.voters[self.local]
514 == self.configuration.primary(self.configuration.scope.view)
515 }
516
517 fn ready_for_appends(&self) -> bool {
518 self.status == Status::Normal
519 && self.journal_tail.is_none()
520 && (!self.is_primary()
521 || (self.applied.op >= self.activation_tail
522 && self.journal.snapshot().durable >= self.activation_tail
523 && self.acknowledged[(self.local + 1) % 3]
524 .max(self.acknowledged[(self.local + 2) % 3])
525 >= self.activation_tail))
526 }
527
528 pub(crate) fn validate_install(
532 &self,
533 ticket: crate::InstallTicket,
534 operations: &[PreparedOperation],
535 ) -> Result<usize, ReplicationError> {
536 let (previous, body_bytes) =
537 self.validate_install_chunk(ticket, ticket.committed(), operations)?;
538 if previous != ticket.accepted() {
539 return Err(ReplicationError::ConflictingHistory);
540 }
541 Ok(body_bytes)
542 }
543
544 pub(crate) fn validate_install_chunk(
545 &self,
546 ticket: crate::InstallTicket,
547 mut previous: Prefix,
548 operations: &[PreparedOperation],
549 ) -> Result<(Prefix, usize), ReplicationError> {
550 if self.status != Status::Fenced {
551 return Err(ReplicationError::NotNormal);
552 }
553 if operations.len() > self.limits.max_operations {
554 return Err(ReplicationError::Capacity);
555 }
556 let mut body_bytes = 0usize;
557 for operation in operations {
558 if operation.group_id != ticket.scope().group_id
559 || operation.configuration_epoch != ticket.scope().configuration_epoch
560 || operation.original_view >= ticket.scope().view
561 {
562 return Err(ReplicationError::ScopeMismatch);
563 }
564 if previous.op.0.checked_add(1) != Some(operation.prefix.op.0) {
565 return Err(ReplicationError::HistoryGap);
566 }
567 if operation.previous_digest != previous.digest {
568 return Err(ReplicationError::ConflictingHistory);
569 }
570 body_bytes = body_bytes
571 .checked_add(operation.body_bytes)
572 .ok_or(ReplicationError::Capacity)?;
573 if body_bytes > self.limits.max_body_bytes {
574 return Err(ReplicationError::Capacity);
575 }
576 previous = operation.prefix;
577 }
578 if previous.op > ticket.accepted().op
579 || (previous.op == ticket.accepted().op && previous != ticket.accepted())
580 {
581 return Err(ReplicationError::ConflictingHistory);
582 }
583 Ok((previous, body_bytes))
584 }
585
586 pub(crate) fn stage_install(
587 mut self,
588 ticket: crate::InstallTicket,
589 operations: &[PreparedOperation],
590 start: crate::StartView,
591 body_bytes: usize,
592 ) -> Self {
593 self.operations.clear();
594 self.operations.extend(operations);
595 self.body_bytes = body_bytes;
596 self.configuration.scope = ticket.scope();
597 self.journal = JournalProgress::recover(ticket.generation(), ticket.accepted().op);
598 self.accepted = ticket.accepted();
599 self.committed = ticket.committed();
600 self.applied = ticket.committed();
601 self.retained_base = ticket.committed();
602 self.known_commit = ticket.committed().op;
603 self.acknowledged = [OpNumber(0); 3];
604 self.activation_tail = ticket.accepted().op;
605 self.installed_view = Some(start);
606 self.journal_tail = None;
607 self
608 }
609
610 pub(crate) fn retain_installed_tail_on_disk(&mut self) {
611 debug_assert!(self.operations.is_empty());
612 self.journal_tail = (self.accepted.op > self.applied.op).then_some(self.accepted);
613 }
614
615 pub(crate) fn activate_installed(&mut self) {
616 debug_assert_eq!(self.status, Status::Fenced);
617 self.status = Status::Normal;
618 }
619
620 fn require_normal(&self) -> Result<(), ReplicationError> {
621 if self.status != Status::Normal {
622 return Err(ReplicationError::NotNormal);
623 }
624 Ok(())
625 }
626
627 fn require_scope(&self, scope: Scope) -> Result<(), ReplicationError> {
628 self.require_normal()?;
629 if scope != self.configuration.scope() {
630 return Err(ReplicationError::ScopeMismatch);
631 }
632 Ok(())
633 }
634
635 fn require_primary(&self, from: NodeId) -> Result<(), ReplicationError> {
636 self.configuration.voter_index(from)?;
637 if from != self.configuration.primary(self.configuration.scope.view) {
638 return Err(ReplicationError::WrongRole);
639 }
640 Ok(())
641 }
642
643 fn prefix_at(&self, op: OpNumber) -> Result<Prefix, ReplicationError> {
644 if op == self.applied.op {
645 return Ok(self.applied);
646 }
647 if op == self.retained_base.op {
648 return Ok(self.retained_base);
649 }
650 if op < self.retained_base.op {
651 return Err(ReplicationError::HistoryUnavailable);
652 }
653 if op > self.accepted.op {
654 return Err(ReplicationError::HistoryGap);
655 }
656 if let Some(tail) = self.journal_tail {
657 if op == tail.op {
658 return Ok(tail);
659 }
660 if op == self.committed.op {
661 return Ok(self.committed);
662 }
663 return Err(ReplicationError::HistoryUnavailable);
666 }
667 let index = (op.0 - self.retained_base.op.0 - 1) as usize;
669 Ok(self.operations[index].prefix)
670 }
671
672 fn reclaim(&mut self) {
673 let through = self.applied.op.min(self.stored_through());
674 if through <= self.retained_base.op {
675 return;
676 }
677 let prefix = self.prefix_at(through).expect("applied persisted prefix");
678 while self
679 .operations
680 .front()
681 .is_some_and(|operation| operation.prefix.op <= through)
682 {
683 let operation = self.operations.pop_front().expect("checked front");
684 self.body_bytes -= operation.body_bytes;
685 }
686 self.retained_base = prefix;
687 }
688
689 fn advance_commit(&mut self) {
690 if self.status != Status::Normal {
691 return;
692 }
693 let available = match self.configuration.policy() {
694 QuorumPolicy::Durable => self.journal.snapshot().durable,
695 QuorumPolicy::Replicated => self.accepted.op,
696 };
697 let eligible = if self.is_primary() {
698 let backup = self.acknowledged[(self.local + 1) % 3]
700 .max(self.acknowledged[(self.local + 2) % 3]);
701 available.min(backup)
702 } else {
703 available.min(self.known_commit)
704 };
705 if eligible > self.committed.op {
706 self.committed = self.prefix_at(eligible).expect("verified retained prefix");
707 }
708 }
709
710 fn conflict(&mut self) -> ReplicationError {
711 self.journal
712 .fail(self.journal.snapshot().generation)
713 .expect("current generation");
714 self.status = Status::Faulted;
715 ReplicationError::ConflictingHistory
716 }
717}