1use std::collections::HashMap;
10use std::io::Write;
11use std::ops::{Bound, Range};
12use std::path::{Path, PathBuf};
13use std::time::{Instant, SystemTime, UNIX_EPOCH};
14
15use crate::branch::{BranchCatalog, BranchInfo, BranchName, BranchStatus};
16use crate::commit::CommitPolicy;
17use crate::event::{Body, Event};
18use crate::format::{
19 derive_stream_id, generate_id_bytes, BatchId, BranchId, CodecId, EventId, EventType, Metadata,
20 OwnedStoredRecord, RecordEnvelopeV2, StreamId, StreamRevision,
21};
22use crate::log::reader::{FrameFilter, ResolvedFilter};
23use crate::log::{Log, LogReader, RecordReader, ReplayEnd, ReplayPlan, StreamSelector};
24use crate::projection::{decode_stored_event, replay_into, NamespaceScoped, Projection};
25use crate::stream::{event_fingerprint, StreamCatalog};
26use crate::view::{catch_up, View};
27use crate::{
28 AppendReceipt, AppendRequest, Durability, ExpectedRevision, ReceiptDurability, Result,
29 SalamanderError, StreamName,
30};
31
32#[derive(Debug, Clone, PartialEq, Eq)]
34pub struct RetentionSegment {
35 pub base_position: u64,
37 pub bytes: u64,
39}
40
41#[derive(Debug, Clone, PartialEq, Eq)]
43pub enum RetentionBlocker {
44 EngineAnchorUnavailable,
46 BranchRequiresBootstrap {
48 branch: BranchName,
50 fork_position: u64,
52 },
53 ProjectionRequiresBootstrap {
55 name: String,
57 },
58 ConsumerRequiresBootstrap {
60 consumer_id: String,
62 position: u64,
64 },
65 MaintenanceHandlesOpen {
67 readers: usize,
69 feeds: usize,
71 },
72}
73
74#[derive(Debug, Clone, PartialEq, Eq)]
76pub struct RetentionPlan {
77 pub plan_id: [u8; 16],
79 pub generation: u64,
81 pub requested_floor: u64,
83 pub effective_floor: u64,
85 pub current_floor: u64,
87 pub durable_head: u64,
89 pub reclaimable_segments: Vec<RetentionSegment>,
91 pub reclaimable_bytes: u64,
93 pub blockers: Vec<RetentionBlocker>,
95}
96
97#[derive(Debug, Clone, PartialEq, Eq)]
99pub struct RetentionApplyResult {
100 pub generation: u64,
102 pub floor: u64,
104 pub reclaimed_bytes: u64,
106}
107
108#[derive(Debug, Clone, PartialEq, Eq)]
110pub struct RetentionCleanupStatus {
111 pub pending_segments: Vec<RetentionSegment>,
113 pub pending_bytes: u64,
115}
116
117#[derive(Debug, Clone, Copy, PartialEq, Eq)]
119pub enum RetentionPolicy {
120 KeepFrom(u64),
122 KeepLatestEvents(u64),
124 KeepNewerThan(i64),
126 TargetLogBytes(u64),
128}
129
130#[derive(Debug, Clone, PartialEq, Eq)]
132pub struct RetentionPolicyPreview {
133 pub policy: RetentionPolicy,
135 pub selected_floor: u64,
137 pub target_satisfied: bool,
139 pub explanation: String,
141 pub plan: RetentionPlan,
143}
144
145pub struct Salamander<B> {
153 pub(crate) log: Log,
154 views: HashMap<String, Box<dyn View<B>>>,
158 policy: CommitPolicy,
161 pending_bytes: u64,
162 pending_count: u64,
163 last_commit: Instant,
164 catalog: StreamCatalog,
165 branches: BranchCatalog,
166 retention_anchor: Option<crate::RetentionAnchorInfo>,
167 retention_projection_coverage: Vec<crate::RetentionProjectionCoverage>,
168 retention_branch_bootstraps: Vec<crate::RetentionBranchBootstrap>,
169 retention_consumer_bootstraps: Vec<crate::RetentionConsumerBootstrap>,
170 retention_system_records: Vec<crate::retention::AnchoredSystemRecord>,
171 durable_head: u64,
172 root: PathBuf,
173}
174
175fn load_core_catalog(root: &Path, database_id: [u8; 16], head: u64) -> Option<StreamCatalog> {
176 let bytes = std::fs::read(root.join("core-catalog.bin")).ok()?;
177 if bytes.len() > 256 * 1024 * 1024 {
178 return None;
179 }
180 let checkpoint: CoreCatalogCheckpoint = bincode::deserialize(&bytes).ok()?;
181 if checkpoint.database_id != database_id || checkpoint.head != head {
182 return None;
183 }
184 let payload =
185 bincode::serialize(&(checkpoint.database_id, checkpoint.head, &checkpoint.catalog)).ok()?;
186 (crc32c::crc32c(&payload) == checkpoint.checksum).then_some(checkpoint.catalog)
187}
188
189fn persist_core_catalog(
190 root: &Path,
191 database_id: [u8; 16],
192 head: u64,
193 catalog: &StreamCatalog,
194) -> Result<()> {
195 let payload = bincode::serialize(&(database_id, head, catalog))
196 .map_err(|error| SalamanderError::Serialization(error.to_string()))?;
197 let checkpoint = CoreCatalogCheckpoint {
198 database_id,
199 head,
200 catalog: catalog.clone(),
201 checksum: crc32c::crc32c(&payload),
202 };
203 let bytes = bincode::serialize(&checkpoint)
204 .map_err(|error| SalamanderError::Serialization(error.to_string()))?;
205 let temporary = root.join("core-catalog.tmp");
206 let final_path = root.join("core-catalog.bin");
207 {
208 let mut file = std::fs::File::create(&temporary)?;
209 file.write_all(&bytes)?;
210 file.sync_all()?;
211 }
212 std::fs::rename(temporary, final_path)?;
213 Ok(())
214}
215
216#[derive(serde::Serialize, serde::Deserialize)]
217struct CoreCatalogCheckpoint {
218 database_id: [u8; 16],
219 head: u64,
220 catalog: StreamCatalog,
221 checksum: u32,
222}
223
224impl<B: Body> Salamander<B> {
225 pub fn open(dir: impl AsRef<Path>) -> Result<Self> {
234 Self::open_with_policy(dir, CommitPolicy::default())
235 }
236
237 pub fn open_with_policy(dir: impl AsRef<Path>, policy: CommitPolicy) -> Result<Self> {
241 Self::open_internal(dir.as_ref(), policy, None)
242 }
243
244 pub(crate) fn open_with_policy_and_segment_max(
245 dir: impl AsRef<Path>,
246 policy: CommitPolicy,
247 segment_max_bytes: u64,
248 ) -> Result<Self> {
249 Self::open_internal(dir.as_ref(), policy, Some(segment_max_bytes))
250 }
251
252 fn open_internal(
253 dir: &Path,
254 policy: CommitPolicy,
255 segment_max_bytes: Option<u64>,
256 ) -> Result<Self> {
257 let root = dir.to_path_buf();
258 let log = match segment_max_bytes {
259 Some(maximum) => Log::open_with_segment_max_bytes(&root, maximum)?,
260 None => Log::open(&root)?,
261 };
262 let database_id = log.database_id().into_bytes();
263 let authoritative_anchor = log.retention_anchor_checksum().is_some();
264 let loaded_anchor = match crate::retention::load(&root) {
265 Ok(anchor) => anchor,
266 Err(error) if log.has_complete_prefix() && !authoritative_anchor => {
267 eprintln!("salamander: ignoring non-authoritative retention anchor: {error}");
268 None
269 }
270 Err(error) => return Err(error),
271 };
272 let exact_anchor = loaded_anchor.and_then(|(anchor, info)| {
273 (crate::retention::validate_identity(
274 &anchor,
275 database_id,
276 log.retention_floor(),
277 log.head(),
278 )
279 .is_ok()
280 && log
281 .retention_anchor_checksum()
282 .is_none_or(|checksum| checksum == info.checksum))
283 .then_some((anchor, info))
284 });
285 let (
286 catalog,
287 branches,
288 retention_anchor,
289 retention_projection_coverage,
290 retention_branch_bootstraps,
291 retention_consumer_bootstraps,
292 retention_system_records,
293 ) = if let Some((anchor, info)) = exact_anchor {
294 (
295 anchor.stream_catalog,
296 anchor.branch_catalog,
297 Some(info),
298 anchor.projection_coverage,
299 anchor.branch_bootstraps,
300 anchor.consumer_bootstraps,
301 anchor.system_records,
302 )
303 } else if authoritative_anchor {
304 return Err(SalamanderError::InvalidFormat(
305 "manifest has no matching authoritative retention anchor".into(),
306 ));
307 } else if log.has_complete_prefix() {
308 let catalog = match load_core_catalog(&root, database_id, log.head()) {
309 Some(catalog) => catalog,
310 None => {
311 let catalog = StreamCatalog::rebuild(log.records_from(0))?;
312 let _ = persist_core_catalog(&root, database_id, log.head(), &catalog);
315 catalog
316 }
317 };
318 let branches = BranchCatalog::rebuild(log.system_records())?;
319 (
320 catalog,
321 branches,
322 None,
323 Vec::new(),
324 Vec::new(),
325 Vec::new(),
326 Vec::new(),
327 )
328 } else {
329 return Err(SalamanderError::InvalidFormat(
330 "retained database has no compatible authoritative core anchor".into(),
331 ));
332 };
333 let durable_head = log.head();
334 Ok(Salamander {
335 log,
336 views: HashMap::new(),
337 policy,
338 pending_bytes: 0,
339 pending_count: 0,
340 last_commit: Instant::now(),
341 catalog,
342 branches,
343 retention_anchor,
344 retention_projection_coverage,
345 retention_branch_bootstraps,
346 retention_consumer_bootstraps,
347 retention_system_records,
348 durable_head,
349 root,
350 })
351 }
352
353 pub fn set_commit_policy(&mut self, policy: CommitPolicy) {
357 self.policy = policy;
358 }
359
360 pub fn commit_policy(&self) -> CommitPolicy {
362 self.policy
363 }
364
365 pub fn retention_floor(&self) -> u64 {
367 self.log.retention_floor()
368 }
369
370 pub fn plan_retention(&self, requested_floor: u64) -> Result<RetentionPlan> {
376 let head = self.durable_head();
377 if requested_floor > head {
378 return Err(SalamanderError::OffsetBeyondHead(requested_floor));
379 }
380 if requested_floor < self.retention_floor() {
381 return Err(SalamanderError::InvalidArgument(format!(
382 "retention floor cannot move backward from {} to {requested_floor}",
383 self.retention_floor()
384 )));
385 }
386 let (effective_floor, segments) = self.log.retention_boundary(requested_floor);
387 let reclaimable_segments: Vec<_> = segments
388 .into_iter()
389 .map(|(base_position, bytes)| RetentionSegment {
390 base_position,
391 bytes,
392 })
393 .collect();
394 let anchor_ready = self.retention_anchor.as_ref().is_some_and(|anchor| {
395 anchor.database_id == self.log.database_id().into_bytes()
396 && anchor.floor == effective_floor
397 && anchor.head == head
398 });
399 let mut blockers = Vec::new();
400 if !anchor_ready {
401 blockers.push(RetentionBlocker::EngineAnchorUnavailable);
402 }
403 blockers.extend(self.branches.all().filter_map(|branch| {
404 let fork_position = branch.fork_position?;
405 (fork_position < effective_floor
406 && !self.has_retention_branch_bootstrap(branch.id.into_bytes(), effective_floor))
407 .then(|| RetentionBlocker::BranchRequiresBootstrap {
408 branch: branch.name.clone(),
409 fork_position,
410 })
411 }));
412 blockers.extend(
413 self.views
414 .keys()
415 .cloned()
416 .map(|name| RetentionBlocker::ProjectionRequiresBootstrap { name }),
417 );
418 Ok(RetentionPlan {
419 plan_id: generate_id_bytes(),
420 generation: self.log.retention_generation(),
421 requested_floor,
422 effective_floor,
423 current_floor: self.retention_floor(),
424 durable_head: head,
425 reclaimable_bytes: reclaimable_segments
426 .iter()
427 .map(|segment| segment.bytes)
428 .sum(),
429 reclaimable_segments,
430 blockers,
431 })
432 }
433
434 pub fn preview_retention_policy(
440 &self,
441 policy: RetentionPolicy,
442 ) -> Result<RetentionPolicyPreview> {
443 let current = self.retention_floor();
444 let head = self.durable_head();
445 let (selected_floor, target_satisfied, explanation) = match policy {
446 RetentionPolicy::KeepFrom(position) => (
447 position,
448 true,
449 format!("selected explicit position {position}"),
450 ),
451 RetentionPolicy::KeepLatestEvents(count) => {
452 let available = head.saturating_sub(current);
453 let selected = head.saturating_sub(count.min(available)).max(current);
454 (
455 selected,
456 true,
457 format!("selected position {selected} to keep the latest {count} event(s)"),
458 )
459 }
460 RetentionPolicy::KeepNewerThan(cutoff) => {
461 let mut selected = head;
462 for item in self.log.records_from(current) {
463 let record = item?;
464 if record.envelope.timestamp_unix_nanos >= cutoff {
465 selected = selected.min(record.position);
466 }
467 }
468 (
469 selected,
470 true,
471 format!("selected earliest event at or after unix-nanosecond cutoff {cutoff}"),
472 )
473 }
474 RetentionPolicy::TargetLogBytes(bytes) => {
475 let (selected, retained, satisfied) = self.log.retention_floor_for_bytes(bytes)?;
476 (
477 selected,
478 satisfied,
479 format!(
480 "selected position {selected} with {retained} retained segment byte(s) for target {bytes}"
481 ),
482 )
483 }
484 };
485 let plan = self.plan_retention(selected_floor)?;
486 Ok(RetentionPolicyPreview {
487 policy,
488 selected_floor,
489 target_satisfied,
490 explanation,
491 plan,
492 })
493 }
494
495 pub(crate) fn apply_retention_prevalidated(
496 &mut self,
497 effective_floor: u64,
498 durable_head: u64,
499 ) -> Result<RetentionApplyResult> {
500 if self.durable_head() != durable_head {
501 return Err(SalamanderError::InvalidArgument(
502 "retention plan is stale because durable head changed".into(),
503 ));
504 }
505 let anchor = self.retention_anchor.as_ref().ok_or_else(|| {
506 SalamanderError::InvalidArgument("retention plan has no verified anchor".into())
507 })?;
508 if anchor.floor != effective_floor || anchor.head != durable_head {
509 return Err(SalamanderError::InvalidArgument(
510 "retention plan anchor identity is stale".into(),
511 ));
512 }
513 crate::retention::crash_point("before_manifest_switch");
514 self.log
515 .activate_retention(effective_floor, anchor.checksum)?;
516 crate::retention::crash_point("after_manifest_switch");
517 let reclaimed_bytes = self.log.reclaim_below_retention_floor();
518 crate::retention::crash_point("after_cleanup");
519 Ok(RetentionApplyResult {
520 generation: self.log.retention_generation(),
521 floor: self.retention_floor(),
522 reclaimed_bytes,
523 })
524 }
525
526 pub fn create_retention_anchor(
531 &mut self,
532 requested_floor: u64,
533 ) -> Result<crate::RetentionAnchorInfo> {
534 self.create_retention_anchor_with_coverage(requested_floor, Vec::new())
535 }
536
537 pub(crate) fn create_retention_anchor_with_coverage(
538 &mut self,
539 requested_floor: u64,
540 projection_coverage: Vec<crate::RetentionProjectionCoverage>,
541 ) -> Result<crate::RetentionAnchorInfo> {
542 self.create_retention_anchor_with_all_coverage(
543 requested_floor,
544 projection_coverage,
545 Vec::new(),
546 Vec::new(),
547 )
548 }
549
550 pub(crate) fn create_retention_anchor_with_all_coverage(
551 &mut self,
552 requested_floor: u64,
553 projection_coverage: Vec<crate::RetentionProjectionCoverage>,
554 branch_bootstraps: Vec<crate::RetentionBranchBootstrap>,
555 consumer_bootstraps: Vec<crate::RetentionConsumerBootstrap>,
556 ) -> Result<crate::RetentionAnchorInfo> {
557 self.commit()?;
558 let plan = self.plan_retention(requested_floor)?;
559 let catalog = StreamCatalog::rebuild(self.log.records_from(0))?;
560 let branches = BranchCatalog::rebuild(self.log.system_records())?;
561 let system_records = self
562 .log
563 .system_records()
564 .map(|item| {
565 item.map(|record| crate::retention::AnchoredSystemRecord {
566 event_type: record.envelope.event_type.as_str().to_string(),
567 payload: record.payload,
568 })
569 })
570 .collect::<Result<Vec<_>>>()?;
571 let info = crate::retention::publish(
572 &self.root,
573 crate::retention::CoreRetentionAnchor {
574 format_version: crate::retention::FORMAT_VERSION,
575 database_id: self.log.database_id().into_bytes(),
576 floor: plan.effective_floor,
577 head: self.durable_head(),
578 stream_catalog: catalog.clone(),
579 branch_catalog: branches.clone(),
580 projection_coverage: projection_coverage.clone(),
581 branch_bootstraps: branch_bootstraps.clone(),
582 consumer_bootstraps: consumer_bootstraps.clone(),
583 system_records: system_records.clone(),
584 },
585 )?;
586 self.catalog = catalog;
587 self.branches = branches;
588 self.retention_anchor = Some(info.clone());
589 self.retention_projection_coverage = projection_coverage;
590 self.retention_branch_bootstraps = branch_bootstraps;
591 self.retention_consumer_bootstraps = consumer_bootstraps;
592 self.retention_system_records = system_records;
593 Ok(info)
594 }
595
596 pub(crate) fn has_retention_branch_bootstrap(&self, branch_id: [u8; 16], floor: u64) -> bool {
597 self.retention_branch_bootstraps.iter().any(|item| {
598 item.branch_id == branch_id
599 && item.floor == floor
600 && crc32c::crc32c(&item.checkpoint) == item.checksum
601 })
602 }
603
604 pub(crate) fn has_retention_consumer_bootstrap(&self, consumer_id: &str, floor: u64) -> bool {
605 self.retention_consumer_bootstraps.iter().any(|item| {
606 item.consumer_id == consumer_id
607 && item.floor == floor
608 && crc32c::crc32c(&item.checkpoint) == item.checksum
609 })
610 }
611
612 pub(crate) fn retention_branch_bootstrap(&self, branch_id: [u8; 16]) -> Option<Vec<u8>> {
613 self.retention_branch_bootstraps
614 .iter()
615 .find(|item| {
616 item.branch_id == branch_id && crc32c::crc32c(&item.checkpoint) == item.checksum
617 })
618 .map(|item| item.checkpoint.clone())
619 }
620
621 pub(crate) fn retention_consumer_bootstrap(&self, consumer_id: &str) -> Option<Vec<u8>> {
622 self.retention_consumer_bootstraps
623 .iter()
624 .find(|item| {
625 item.consumer_id == consumer_id && crc32c::crc32c(&item.checkpoint) == item.checksum
626 })
627 .map(|item| item.checkpoint.clone())
628 }
629
630 pub(crate) fn retention_consumer_bootstrap_info(
631 &self,
632 consumer_id: &str,
633 ) -> Option<crate::RetentionConsumerBootstrap> {
634 self.retention_consumer_bootstraps
635 .iter()
636 .find(|item| {
637 item.consumer_id == consumer_id && crc32c::crc32c(&item.checkpoint) == item.checksum
638 })
639 .cloned()
640 }
641
642 pub(crate) fn retention_identity(&self) -> ([u8; 16], u64) {
643 (
644 self.log.database_id().into_bytes(),
645 self.log.retention_generation(),
646 )
647 }
648
649 pub(crate) fn retention_cleanup_status(&self) -> RetentionCleanupStatus {
650 let (_, segments) = self.log.retention_boundary(self.retention_floor());
651 let pending_segments = segments
652 .into_iter()
653 .map(|(base_position, bytes)| RetentionSegment {
654 base_position,
655 bytes,
656 })
657 .collect::<Vec<_>>();
658 let pending_bytes = pending_segments.iter().map(|segment| segment.bytes).sum();
659 RetentionCleanupStatus {
660 pending_segments,
661 pending_bytes,
662 }
663 }
664
665 pub(crate) fn system_metadata(&self) -> Result<Vec<(String, Vec<u8>)>> {
666 let mut records = self
667 .retention_system_records
668 .iter()
669 .map(|record| (record.event_type.clone(), record.payload.clone()))
670 .collect::<Vec<_>>();
671 for item in self.log.system_records() {
672 let record = item?;
673 records.push((
674 record.envelope.event_type.as_str().to_string(),
675 record.payload,
676 ));
677 }
678 Ok(records)
679 }
680
681 pub(crate) fn has_retention_projection_coverage(
682 &self,
683 name: &str,
684 descriptor_fingerprint: [u8; 16],
685 branch_id: [u8; 16],
686 partitions: u32,
687 ) -> bool {
688 self.retention_anchor.as_ref().is_some_and(|anchor| {
689 self.retention_projection_coverage.iter().any(|coverage| {
690 let mut covered_partitions = std::collections::BTreeSet::new();
691 coverage.name == name
692 && coverage.descriptor_fingerprint == descriptor_fingerprint
693 && coverage.branch_id == branch_id
694 && coverage.cursor == anchor.head
695 && coverage.snapshot_ids.len() == partitions as usize
696 && coverage.snapshot_ids.iter().all(|id| {
697 crate::snapshot::verify(&self.root, id).is_ok_and(|info| {
698 info.manifest.projection_name == name
699 && info.manifest.descriptor_fingerprint == descriptor_fingerprint
700 && info.manifest.branch_id == branch_id
701 && info.manifest.cursor.position == anchor.head
702 && info.manifest.partition_count.unwrap_or(1) == partitions
703 && covered_partitions.insert(info.manifest.partition.unwrap_or(0))
704 })
705 })
706 && covered_partitions.len() == partitions as usize
707 })
708 })
709 }
710
711 pub fn append(&mut self, namespace: &str, body: B) -> Result<u64> {
715 let request = AppendRequest {
716 branch: BranchId::ZERO,
717 stream: crate::StreamName::new(namespace)?,
718 expected: ExpectedRevision::Any,
719 idempotency_key: None,
720 events: vec![crate::NewEvent::new(
721 EventType::new(std::any::type_name::<B>())?,
722 body,
723 )],
724 durability: Durability::Buffered,
725 };
726 Ok(self.append_batch(request)?.first_position)
727 }
728
729 pub fn append_on_branch(&mut self, branch: BranchId, namespace: &str, body: B) -> Result<u64> {
731 let request = AppendRequest {
732 branch,
733 stream: crate::StreamName::new(namespace)?,
734 expected: ExpectedRevision::Any,
735 idempotency_key: None,
736 events: vec![crate::NewEvent::new(
737 EventType::new(std::any::type_name::<B>())?,
738 body,
739 )],
740 durability: Durability::Buffered,
741 };
742 Ok(self.append_batch(request)?.first_position)
743 }
744
745 #[allow(dead_code)]
746 fn append_wp01_compat(&mut self, namespace: &str, body: B) -> Result<u64> {
747 let timestamp_ms = current_timestamp_ms();
748 let mut event = Event {
749 offset: 0,
750 timestamp_ms,
751 namespace: namespace.to_string(),
752 body,
753 };
754 let bytes = bincode::serialize(&event.body)
755 .map_err(|e| SalamanderError::Serialization(e.to_string()))?;
756 let database_id = self.log.database_id();
757 let branch_id = BranchId::ZERO;
758 let id_bytes = generate_id_bytes();
759 let mut metadata = Metadata::new();
760 metadata.insert(
761 "salamander.stream_name".to_string(),
762 namespace.as_bytes().to_vec(),
763 );
764 let envelope = RecordEnvelopeV2 {
765 event_id: EventId::from_bytes(id_bytes),
766 database_id,
767 branch_id,
768 stream_id: derive_stream_id(database_id, branch_id, namespace),
769 stream_revision: StreamRevision(self.log.head()),
773 timestamp_unix_nanos: (timestamp_ms as i64).saturating_mul(1_000_000),
774 event_type: EventType::new(std::any::type_name::<B>())?,
775 schema_version: 1,
776 codec: CodecId::RUST_BINCODE_V1,
777 batch_id: BatchId::from_bytes(id_bytes),
778 batch_index: 0,
779 metadata,
780 };
781 let (offset, last) = self.log.append_batch(&[(envelope, bytes.clone())])?;
782 debug_assert_eq!(offset, last);
783
784 event.offset = offset;
791 for view in self.views.values_mut() {
792 view.apply(&event);
793 }
794
795 self.pending_bytes += bytes.len() as u64;
798 self.pending_count += 1;
799 if self.policy.should_commit(
800 self.pending_bytes,
801 self.pending_count,
802 self.last_commit.elapsed(),
803 ) {
804 self.commit()?;
805 }
806
807 self.catalog = StreamCatalog::rebuild(self.log.records_from(0))?;
808 if self.durable_head == self.log.head() {
809 persist_core_catalog(
810 &self.root,
811 self.log.database_id().into_bytes(),
812 self.durable_head,
813 &self.catalog,
814 )?;
815 }
816 Ok(offset)
817 }
818
819 pub fn append_batch(&mut self, request: AppendRequest<B>) -> Result<AppendReceipt> {
823 self.append_batch_with_id(request, None)
824 }
825
826 pub(crate) fn append_batch_with_id(
827 &mut self,
828 request: AppendRequest<B>,
829 supplied_batch_id: Option<BatchId>,
830 ) -> Result<AppendReceipt> {
831 request.validate()?;
832 let branch = self
833 .branches
834 .get(request.branch)
835 .ok_or_else(|| SalamanderError::BranchNotFound(format!("{:?}", request.branch)))?;
836 if branch.status == BranchStatus::Archived {
837 return Err(SalamanderError::BranchArchived(
838 branch.name.as_str().to_string(),
839 ));
840 }
841 let previous = self.catalog.revision(request.branch, &request.stream);
842 let serialized: Vec<Vec<u8>> = request
843 .events
844 .iter()
845 .map(|event| {
846 bincode::serialize(&event.body)
847 .map_err(|error| SalamanderError::Serialization(error.to_string()))
848 })
849 .collect::<Result<_>>()?;
850 let request_digest = request_fingerprint(&request, &serialized);
851 if let Some(key) = &request.idempotency_key {
852 if let Some((digest, mut receipt)) = self.catalog.idempotent(request.branch, key) {
853 if digest != request_digest {
854 return Err(SalamanderError::IdempotencyConflict);
855 }
856 if request.durability == Durability::Sync
857 && receipt.durability != ReceiptDurability::Synced
858 {
859 self.commit()?;
860 receipt.durability = ReceiptDurability::Synced;
861 }
862 return Ok(receipt);
863 }
864 }
865 validate_expected(request.expected, previous)?;
866
867 let database_id = self.log.database_id();
868 let stream_id = self
869 .catalog
870 .stream_id(request.branch, &request.stream)
871 .unwrap_or_else(|| {
872 derive_stream_id(database_id, request.branch, request.stream.as_str())
873 });
874 let batch_id =
875 supplied_batch_id.unwrap_or_else(|| BatchId::from_bytes(generate_id_bytes()));
876 let first_revision = previous.map_or(0, |revision| revision.0 + 1);
877 let timestamp_ms = current_timestamp_ms();
878 let mut stored = Vec::with_capacity(request.events.len());
879 let mut digests = Vec::with_capacity(request.events.len());
880
881 for (index, (event, body)) in request.events.iter().zip(&serialized).enumerate() {
882 let event_id = event
883 .event_id
884 .unwrap_or_else(|| EventId::from_bytes(generate_id_bytes()));
885 let mut metadata = event.metadata.clone();
886 metadata.insert(
887 "salamander.stream_name".into(),
888 request.stream.as_str().as_bytes().to_vec(),
889 );
890 if let Some(key) = &request.idempotency_key {
891 metadata.insert("salamander.idempotency_key".into(), key.as_bytes().to_vec());
892 metadata.insert(
893 "salamander.request_digest".into(),
894 request_digest.to_le_bytes().to_vec(),
895 );
896 }
897 let envelope = RecordEnvelopeV2 {
898 event_id,
899 database_id,
900 branch_id: request.branch,
901 stream_id,
902 stream_revision: StreamRevision(first_revision + index as u64),
903 timestamp_unix_nanos: (timestamp_ms as i64).saturating_mul(1_000_000),
904 event_type: event.event_type.clone(),
905 schema_version: event.schema_version,
906 codec: CodecId::RUST_BINCODE_V1,
907 batch_id,
908 batch_index: index as u32,
909 metadata,
910 };
911 let record = crate::format::OwnedStoredRecord {
912 kind: crate::format::FrameKind::Event,
913 flags: 0,
914 position: self.log.head() + index as u64,
915 envelope: envelope.clone(),
916 payload: body.clone(),
917 };
918 let digest = event_fingerprint(&record);
919 if self
920 .catalog
921 .event_digest(event_id)
922 .is_some_and(|old| old != digest)
923 || digests
924 .iter()
925 .any(|(id, old)| *id == event_id && *old != digest)
926 {
927 return Err(SalamanderError::EventIdConflict);
928 }
929 digests.push((event_id, digest));
930 stored.push((envelope, body.clone()));
931 }
932
933 let supplied_ids: Vec<_> = request.events.iter().map(|event| event.event_id).collect();
934 if supplied_ids
935 .iter()
936 .flatten()
937 .any(|id| self.catalog.event_receipt(*id).is_some())
938 {
939 let mut original: Option<AppendReceipt> = None;
940 for (supplied, (_, digest)) in supplied_ids.iter().zip(&digests) {
941 let Some(id) = supplied else {
942 return Err(SalamanderError::EventIdConflict);
943 };
944 let Some((stored_digest, receipt)) = self.catalog.event_receipt(*id) else {
945 return Err(SalamanderError::EventIdConflict);
946 };
947 if stored_digest != *digest
948 || original
949 .as_ref()
950 .is_some_and(|existing| existing.batch_id != receipt.batch_id)
951 {
952 return Err(SalamanderError::EventIdConflict);
953 }
954 original = Some(receipt);
955 }
956 let mut original = original.ok_or(SalamanderError::EventIdConflict)?;
957 if supplied_batch_id.is_some_and(|id| id != original.batch_id) {
958 return Err(SalamanderError::BatchIdConflict);
959 }
960 if request.durability == Durability::Sync
961 && original.durability != ReceiptDurability::Synced
962 {
963 self.commit()?;
964 original.durability = ReceiptDurability::Synced;
965 }
966 return Ok(original);
967 }
968
969 if self.catalog.batch_receipt(batch_id).is_some() {
970 return Err(SalamanderError::BatchIdConflict);
971 }
972
973 let (first_position, last_position) = self.log.append_batch(&stored)?;
974 for (index, event) in request.events.iter().enumerate() {
975 let runtime = Event {
976 offset: first_position + index as u64,
977 timestamp_ms,
978 namespace: request.stream.as_str().to_string(),
979 body: event.body.clone(),
980 };
981 for view in self.views.values_mut() {
982 view.apply(&runtime);
983 }
984 }
985
986 self.pending_bytes += serialized.iter().map(Vec::len).sum::<usize>() as u64;
987 self.pending_count += request.events.len() as u64;
988 let sync = request.durability == Durability::Sync
989 || self.policy.should_commit(
990 self.pending_bytes,
991 self.pending_count,
992 self.last_commit.elapsed(),
993 );
994 if sync {
995 self.commit()?;
996 }
997 let receipt = AppendReceipt {
998 batch_id,
999 first_position,
1000 last_position,
1001 stream_id,
1002 previous_revision: previous,
1003 current_revision: StreamRevision(first_revision + request.events.len() as u64 - 1),
1004 durability: if sync {
1005 ReceiptDurability::Synced
1006 } else if request.durability == Durability::Flush {
1007 ReceiptDurability::Flushed
1008 } else {
1009 ReceiptDurability::Buffered
1010 },
1011 };
1012 self.catalog.record_batch(
1013 request.branch,
1014 &request.stream,
1015 stream_id,
1016 digests,
1017 request
1018 .idempotency_key
1019 .as_ref()
1020 .map(|key| (key, request_digest)),
1021 receipt.clone(),
1022 );
1023 if sync {
1024 persist_core_catalog(
1025 &self.root,
1026 self.log.database_id().into_bytes(),
1027 self.durable_head,
1028 &self.catalog,
1029 )?;
1030 }
1031 Ok(receipt)
1032 }
1033
1034 pub fn branch(&self, id: BranchId) -> Option<&BranchInfo> {
1036 self.branches.get(id)
1037 }
1038
1039 pub fn branch_named(&self, name: &str) -> Option<&BranchInfo> {
1041 self.branches.named(name)
1042 }
1043
1044 pub fn branch_ancestry(&self, id: BranchId) -> Result<Vec<BranchInfo>> {
1046 self.branches.ancestry(id)
1047 }
1048
1049 pub fn branch_children(&self, id: BranchId) -> Vec<BranchInfo> {
1051 self.branches.children(id)
1052 }
1053
1054 pub fn branch_common_ancestor(&self, left: BranchId, right: BranchId) -> Result<BranchInfo> {
1056 self.branches.common_ancestor(left, right)
1057 }
1058
1059 pub fn diff(&self, request: DiffRequest) -> Result<TimelineDiff> {
1068 request.streams.validate()?;
1069 let head = self.log.head();
1070 let resolve = |end: ReplayEnd| match end {
1071 ReplayEnd::Head => Ok(head),
1072 ReplayEnd::At(position) if position > head => {
1073 Err(SalamanderError::OffsetBeyondHead(position))
1074 }
1075 ReplayEnd::At(position) => Ok(position),
1076 };
1077 let left_until = resolve(request.left_until)?;
1078 let right_until = resolve(request.right_until)?;
1079 let branch = |id: BranchId| {
1080 self.branches
1081 .get(id)
1082 .cloned()
1083 .ok_or_else(|| SalamanderError::BranchNotFound(format!("{id:?}")))
1084 };
1085 let left = branch(request.left)?;
1086 let right = branch(request.right)?;
1087 let (common_ancestor, divergence) =
1088 self.branches
1089 .divergence(request.left, left_until, request.right, right_until)?;
1090 let floor = self.retention_floor();
1091 if divergence < floor {
1092 return Err(self.position_unavailable(divergence));
1093 }
1094 let plan = |branch: BranchId, from: u64, until: u64| ReplayPlan {
1095 branch,
1096 streams: request.streams.clone(),
1097 from: Bound::Included(from),
1098 until: ReplayEnd::At(until),
1099 ..ReplayPlan::default()
1100 };
1101 Ok(TimelineDiff {
1102 shared: plan(common_ancestor.id, floor, divergence),
1103 common_ancestor,
1104 divergence,
1105 left: DiffSide {
1106 suffix: plan(left.id, divergence, left_until),
1107 branch: left,
1108 until: left_until,
1109 },
1110 right: DiffSide {
1111 suffix: plan(right.id, divergence, right_until),
1112 branch: right,
1113 until: right_until,
1114 },
1115 })
1116 }
1117
1118 pub fn read(&self, plan: ReplayPlan) -> Result<LogReader<'_>> {
1124 plan.streams.validate()?;
1125 let head = self.log.head();
1126 let until = match plan.until {
1127 ReplayEnd::Head => head,
1128 ReplayEnd::At(position) => {
1129 if position > head {
1130 return Err(SalamanderError::OffsetBeyondHead(position));
1131 }
1132 position
1133 }
1134 };
1135 let from = match plan.from {
1136 Bound::Unbounded => 0,
1137 Bound::Included(position) => position,
1138 Bound::Excluded(position) => position.saturating_add(1),
1139 };
1140 if until < self.retention_floor() {
1141 return Err(self.position_unavailable(until));
1142 }
1143 if from < self.retention_floor() {
1144 return Err(self.position_unavailable(from));
1145 }
1146 let scopes = self.branches.replay_scopes(plan.branch, until)?;
1147 Ok(self.log.plan_reader(ResolvedFilter {
1148 from,
1149 until,
1150 selector: plan.streams,
1151 scopes: Some(scopes),
1152 time: plan.time,
1153 kinds: FrameFilter::UserEvents,
1154 max_events: plan.max_events,
1155 verification: plan.verification,
1156 }))
1157 }
1158
1159 pub fn replay_branch(
1163 &self,
1164 branch: BranchId,
1165 namespace: &str,
1166 range: Range<u64>,
1167 mut f: impl FnMut(&Event<B>),
1168 ) -> Result<()> {
1169 let mut reader = self.read(ReplayPlan {
1170 branch,
1171 from: Bound::Included(range.start),
1172 until: ReplayEnd::At(range.end),
1173 ..ReplayPlan::default()
1174 })?;
1175 while let Some(record) = reader.next()? {
1176 let record = OwnedStoredRecord::from(record);
1177 let event = decode_stored_event::<B>(&record)?;
1178 if event.namespace == namespace {
1179 f(&event);
1180 }
1181 }
1182 Ok(())
1183 }
1184
1185 pub fn fork_branch(
1190 &mut self,
1191 parent: BranchId,
1192 at: u64,
1193 name: BranchName,
1194 metadata: Metadata,
1195 ) -> Result<BranchInfo> {
1196 if self.branches.get(parent).is_none() {
1197 return Err(SalamanderError::BranchNotFound(format!("{parent:?}")));
1198 }
1199 if at > self.head() {
1200 return Err(SalamanderError::OffsetBeyondHead(at));
1201 }
1202 if at < self.retention_floor() {
1203 return Err(self.position_unavailable(at));
1204 }
1205 if !self.is_batch_boundary(at)? {
1206 return Err(SalamanderError::NotBatchBoundary(at));
1207 }
1208 let id = BranchId::from_bytes(generate_id_bytes());
1209 let info = BranchInfo {
1210 id,
1211 name,
1212 parent: Some(parent),
1213 fork_position: Some(at),
1214 created_at_unix_nanos: (current_timestamp_ms() as i64).saturating_mul(1_000_000),
1215 metadata,
1216 status: BranchStatus::Active,
1217 };
1218 let mut updated_branches = self.branches.clone();
1221 updated_branches.insert(info.clone())?;
1222 let payload = serde_json::to_vec(&info)
1223 .map_err(|error| SalamanderError::Serialization(error.to_string()))?;
1224 let event_bytes = generate_id_bytes();
1225 let envelope = RecordEnvelopeV2 {
1226 event_id: EventId::from_bytes(event_bytes),
1227 database_id: self.log.database_id(),
1228 branch_id: id,
1229 stream_id: crate::StreamId::ZERO,
1230 stream_revision: StreamRevision(0),
1231 timestamp_unix_nanos: info.created_at_unix_nanos,
1232 event_type: EventType::new("salamander.branch.created")?,
1233 schema_version: 1,
1234 codec: CodecId::JSON_UTF8,
1235 batch_id: BatchId::from_bytes(event_bytes),
1236 batch_index: 0,
1237 metadata: Metadata::new(),
1238 };
1239 self.log.append_system(&envelope, &payload)?;
1240 if let Err(error) = self.commit() {
1241 self.branches = BranchCatalog::rebuild(self.log.system_records())?;
1242 return Err(error);
1243 }
1244 self.branches = updated_branches;
1245 Ok(info)
1246 }
1247
1248 pub fn archive_branch(&mut self, id: BranchId) -> Result<BranchInfo> {
1251 let mut info = self
1252 .branches
1253 .get(id)
1254 .cloned()
1255 .ok_or_else(|| SalamanderError::BranchNotFound(format!("{id:?}")))?;
1256 if id == BranchId::ZERO {
1257 return Err(SalamanderError::InvalidArgument(
1258 "the default branch cannot be archived".into(),
1259 ));
1260 }
1261 if info.status == BranchStatus::Archived {
1262 return Ok(info);
1263 }
1264 info.status = BranchStatus::Archived;
1265 let mut updated_branches = self.branches.clone();
1266 updated_branches.archive(info.clone())?;
1267 let payload = serde_json::to_vec(&info)
1268 .map_err(|error| SalamanderError::Serialization(error.to_string()))?;
1269 let event_bytes = generate_id_bytes();
1270 let envelope = RecordEnvelopeV2 {
1271 event_id: EventId::from_bytes(event_bytes),
1272 database_id: self.log.database_id(),
1273 branch_id: id,
1274 stream_id: crate::StreamId::ZERO,
1275 stream_revision: StreamRevision(0),
1276 timestamp_unix_nanos: (current_timestamp_ms() as i64).saturating_mul(1_000_000),
1277 event_type: EventType::new("salamander.branch.archived")?,
1278 schema_version: 1,
1279 codec: CodecId::JSON_UTF8,
1280 batch_id: BatchId::from_bytes(event_bytes),
1281 batch_index: 0,
1282 metadata: Metadata::new(),
1283 };
1284 self.log.append_system(&envelope, &payload)?;
1285 if let Err(error) = self.commit() {
1286 self.branches = BranchCatalog::rebuild(self.log.system_records())?;
1287 return Err(error);
1288 }
1289 self.branches = updated_branches;
1290 Ok(info)
1291 }
1292
1293 fn is_batch_boundary(&self, at: u64) -> Result<bool> {
1294 if at == 0 || at == self.head() {
1295 return Ok(true);
1296 }
1297 let mut before = None;
1300 let mut after = None;
1301 for item in self.log.records_from(at - 1) {
1302 let record = item?;
1303 if record.position == at - 1 {
1304 before = Some(record.envelope.batch_id);
1305 } else if record.position >= at {
1306 after = (record.position == at).then_some(record.envelope.batch_id);
1307 break;
1308 }
1309 }
1310 Ok(matches!((before, after), (Some(left), Some(right)) if left != right))
1311 }
1312
1313 fn position_unavailable(&self, requested: u64) -> SalamanderError {
1314 SalamanderError::PositionUnavailable {
1315 requested,
1316 floor: self.retention_floor(),
1317 head: self.head(),
1318 bootstrap_available: false,
1319 }
1320 }
1321
1322 pub fn commit(&mut self) -> Result<u64> {
1326 let head = self.log.commit()?;
1327 self.durable_head = head;
1328 self.pending_bytes = 0;
1329 self.pending_count = 0;
1330 self.last_commit = Instant::now();
1331 persist_core_catalog(
1332 &self.root,
1333 self.log.database_id().into_bytes(),
1334 head,
1335 &self.catalog,
1336 )?;
1337 Ok(head)
1338 }
1339
1340 pub fn uncommitted_bytes(&self) -> u64 {
1343 self.pending_bytes
1344 }
1345
1346 pub fn uncommitted_count(&self) -> u64 {
1348 self.pending_count
1349 }
1350
1351 pub fn projection<P: Projection<Body = B> + Default>(&self) -> Result<P> {
1355 self.require_full_history()?;
1356 let mut p = P::default();
1357 replay_into(&mut p, &self.log, self.log.head())?;
1358 Ok(p)
1359 }
1360
1361 pub fn projection_for<P: NamespaceScoped<Body = B>>(&self, namespace: &str) -> Result<P> {
1367 self.require_full_history()?;
1368 let mut p = P::new_for(namespace);
1369 replay_into(&mut p, &self.log, self.log.head())?;
1370 Ok(p)
1371 }
1372
1373 pub fn view_at<P: Projection<Body = B> + Default>(&self, n: u64) -> Result<P> {
1377 if n > self.log.head() {
1378 return Err(SalamanderError::OffsetBeyondHead(n));
1379 }
1380 self.require_full_history()?;
1381 let mut p = P::default();
1382 replay_into(&mut p, &self.log, n)?;
1383 Ok(p)
1384 }
1385
1386 pub fn register(&mut self, name: &str, mut view: Box<dyn View<B>>) -> Result<()> {
1392 self.require_full_history()?;
1393 catch_up(view.as_mut(), &self.log, self.log.head())?;
1394 self.views.insert(name.to_string(), view);
1395 Ok(())
1396 }
1397
1398 pub fn deregister(&mut self, name: &str) -> Option<Box<dyn View<B>>> {
1402 self.views.remove(name)
1403 }
1404
1405 pub fn view<T: View<B>>(&self, name: &str) -> Option<&T> {
1413 self.views.get(name)?.as_any().downcast_ref::<T>()
1414 }
1415
1416 pub fn replay_to<P: Projection<Body = B>>(&self, mut view: P, n: u64) -> Result<P> {
1422 if n > self.log.head() {
1423 return Err(SalamanderError::OffsetBeyondHead(n));
1424 }
1425 self.require_full_history()?;
1426 replay_into(&mut view, &self.log, n)?;
1427 Ok(view)
1428 }
1429
1430 pub fn head(&self) -> u64 {
1432 self.log.head()
1433 }
1434
1435 pub fn durable_head(&self) -> u64 {
1437 self.durable_head
1438 }
1439
1440 pub(crate) fn stream_id(&self, branch: BranchId, stream: &StreamName) -> Option<StreamId> {
1441 self.catalog.stream_id(branch, stream)
1442 }
1443
1444 pub fn replay(
1446 &self,
1447 namespace: &str,
1448 range: Range<u64>,
1449 f: impl FnMut(&Event<B>),
1450 ) -> Result<()> {
1451 if range.start < self.retention_floor() {
1452 return Err(self.position_unavailable(range.start));
1453 }
1454 crate::introspect::replay(&self.log, namespace, range, f)
1455 }
1456
1457 fn require_full_history(&self) -> Result<()> {
1458 if self.retention_floor() > 0 {
1459 return Err(self.position_unavailable(0));
1460 }
1461 Ok(())
1462 }
1463}
1464
1465#[derive(Debug, Clone, PartialEq, Eq)]
1469pub struct DiffRequest {
1470 pub left: BranchId,
1472 pub right: BranchId,
1474 pub left_until: ReplayEnd,
1476 pub right_until: ReplayEnd,
1478 pub streams: StreamSelector,
1481}
1482
1483impl DiffRequest {
1484 pub fn new(left: BranchId, right: BranchId) -> Self {
1486 Self {
1487 left,
1488 right,
1489 left_until: ReplayEnd::Head,
1490 right_until: ReplayEnd::Head,
1491 streams: StreamSelector::All,
1492 }
1493 }
1494}
1495
1496#[derive(Debug, Clone, PartialEq, Eq)]
1499pub struct DiffSide {
1500 pub branch: BranchInfo,
1502 pub until: u64,
1504 pub suffix: ReplayPlan,
1506}
1507
1508#[derive(Debug, Clone, PartialEq, Eq)]
1513pub struct TimelineDiff {
1514 pub common_ancestor: BranchInfo,
1516 pub divergence: u64,
1518 pub shared: ReplayPlan,
1522 pub left: DiffSide,
1524 pub right: DiffSide,
1526}
1527
1528fn current_timestamp_ms() -> u64 {
1529 SystemTime::now()
1530 .duration_since(UNIX_EPOCH)
1531 .map(|d| d.as_millis() as u64)
1532 .unwrap_or(0)
1533}
1534
1535fn validate_expected(expected: ExpectedRevision, actual: Option<StreamRevision>) -> Result<()> {
1536 let matches = match expected {
1537 ExpectedRevision::Any => true,
1538 ExpectedRevision::NoStream => actual.is_none(),
1539 ExpectedRevision::Exact(expected) => actual == Some(expected),
1540 };
1541 if matches {
1542 return Ok(());
1543 }
1544 Err(SalamanderError::RevisionConflict {
1545 expected: format!("{expected:?}"),
1546 actual: format!("{actual:?}"),
1547 })
1548}
1549
1550fn request_fingerprint<B>(request: &AppendRequest<B>, bodies: &[Vec<u8>]) -> u32 {
1551 let mut bytes = Vec::new();
1552 bytes.extend_from_slice(request.branch.as_bytes());
1553 bytes.extend_from_slice(request.stream.as_str().as_bytes());
1554 bytes.extend_from_slice(format!("{:?}", request.expected).as_bytes());
1555 for (event, body) in request.events.iter().zip(bodies) {
1556 bytes.extend_from_slice(event.event_id.unwrap_or(EventId::ZERO).as_bytes());
1557 bytes.extend_from_slice(event.event_type.as_str().as_bytes());
1558 bytes.extend_from_slice(&event.schema_version.to_le_bytes());
1559 for (key, value) in &event.metadata {
1560 bytes.extend_from_slice(key.as_bytes());
1561 bytes.extend_from_slice(value);
1562 }
1563 bytes.extend_from_slice(body);
1564 }
1565 crc32c::crc32c(&bytes)
1566}