1use crate::envelope::EnvelopeCodecError;
6use crate::sst_blocks::BlockHandle;
7use crate::WriterEpoch;
8use crate::{
9 AttributeRevisionNo, Attributes, ChangeSeq, CommitId, ContentRef, DisplayName, InodeId,
10 InodeKind, ManifestNo, ManifestObjectId, MetadataCompactionId, MetadataSegmentId, NameKey,
11 NamespaceId, RevisionNo, RunNo,
12};
13use serde::{Deserialize, Serialize};
14
15pub const NAMESPACE_MANIFEST_FORMAT_VERSION: u32 = 1;
18
19#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
23#[serde(rename_all = "snake_case")]
24pub enum NamespaceManifestKind {
25 NamespaceManifest,
27}
28
29impl NamespaceManifestKind {
30 pub const fn as_str(self) -> &'static str {
32 match self {
33 Self::NamespaceManifest => "namespace_manifest",
34 }
35 }
36}
37
38#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
42#[serde(rename_all = "snake_case")]
43pub enum MetadataRowFamily {
44 Inodes,
46 DirentryBinds,
48 DirentryChildBinds,
50 DirentryUnbinds,
52 Revisions,
54 RevisionsByInodeDesc,
56 Tombstones,
58 ActiveDeletions,
61 CommitReceipts,
63 Attributes,
68}
69
70#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
74pub struct MetadataSegmentRef {
75 pub owner_namespace_id: NamespaceId,
77 pub segment_id: MetadataSegmentId,
79 #[serde(default, skip_serializing_if = "Option::is_none")]
83 pub compaction_job_id: Option<MetadataCompactionId>,
84 pub run_no: RunNo,
87 pub run_seq: ChangeSeq,
89 pub level: u32,
91 pub family: MetadataRowFamily,
93 pub segment_index: u32,
95 pub row_count: u64,
97 pub min_row_key: String,
99 pub max_row_key: String,
101 pub index_block: BlockHandle,
105 pub filter_block: BlockHandle,
107 #[serde(default, skip_serializing_if = "Option::is_none")]
113 pub filter_inline: Option<String>,
114 pub object_checksum: String,
118}
119
120#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
124#[serde(tag = "kind", rename_all = "snake_case")]
125pub enum MetadataRow {
126 Inode {
128 inode_id: InodeId,
130 inode_kind: InodeKind,
132 created_seq: ChangeSeq,
134 commit_id: CommitId,
136 created_by: crate::ActorRef,
138 created_at_ms: u64,
140 },
141 DirentryBind {
143 parent_inode_id: InodeId,
145 name_key: NameKey,
147 display_name: DisplayName,
149 child_inode_id: InodeId,
151 bind_seq: ChangeSeq,
153 bind_delta_index: u32,
155 },
156 DirentryUnbind {
158 parent_inode_id: InodeId,
160 name_key: NameKey,
162 display_name: DisplayName,
164 child_inode_id: InodeId,
166 bind_seq: ChangeSeq,
168 bind_delta_index: u32,
170 unbind_seq: ChangeSeq,
172 unbind_delta_index: u32,
174 },
175 FileRevision {
177 inode_id: InodeId,
179 revision_no: RevisionNo,
181 committed_seq: ChangeSeq,
183 commit_id: CommitId,
185 committed_at_ms: u64,
189 committed_by: crate::ActorRef,
191 delta_index: u32,
193 content_ref: ContentRef,
195 },
196 Tombstone {
198 root_inode_id: InodeId,
200 generation: TombstoneGeneration,
203 commit_id: CommitId,
205 action: TombstoneRowAction,
208 deleted_at_ms: u64,
211 deleted_by: crate::ActorRef,
213 },
214 ActiveDeletion {
220 root_inode_id: InodeId,
223 deletion_seq: ChangeSeq,
226 action: ActiveDeletionRowAction,
229 },
230 CommitReceipt {
232 commit_id: CommitId,
234 committed_by: crate::ActorRef,
236 semantic_commit_fingerprint: String,
238 committed_seq: ChangeSeq,
240 committed_at_ms: u64,
245 #[serde(default, skip_serializing_if = "Option::is_none")]
247 message: Option<String>,
248 },
249 AttributesRevision {
256 inode_id: InodeId,
258 attributes_revision_no: AttributeRevisionNo,
260 committed_seq: ChangeSeq,
262 commit_id: CommitId,
264 delta_index: u32,
266 updated_by: crate::ActorRef,
268 updated_at_ms: u64,
270 attributes: Attributes,
273 },
274}
275
276#[derive(
284 Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize,
285)]
286pub struct TombstoneGeneration {
287 pub seq: ChangeSeq,
289 pub delta_index: u32,
291}
292
293#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
300pub struct DeletedDirentry {
301 pub parent_inode_id: InodeId,
303 pub name_key: NameKey,
305 pub display_name: DisplayName,
307}
308
309pub(crate) fn required_option<'de, T, D>(deserializer: D) -> Result<Option<T>, D::Error>
316where
317 T: Deserialize<'de>,
318 D: serde::Deserializer<'de>,
319{
320 Option::deserialize(deserializer)
321}
322
323#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
327#[serde(tag = "kind", rename_all = "snake_case")]
328pub enum TombstoneRowAction {
329 Set {
331 #[serde(deserialize_with = "required_option")]
337 deleted_direntry: Option<DeletedDirentry>,
338 },
339 Revoke {
342 target: TombstoneGeneration,
344 },
345}
346
347#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
354#[serde(tag = "kind", rename_all = "snake_case")]
355pub enum ActiveDeletionRowAction {
356 Listed {
359 deleted_at_ms: u64,
362 deleted_by: crate::ActorRef,
364 #[serde(deserialize_with = "required_option")]
368 deleted_direntry: Option<DeletedDirentry>,
369 },
370 Removed {
372 revocation_seq: ChangeSeq,
374 },
375}
376
377impl ActiveDeletionRowAction {
378 fn sort_rank(&self) -> u32 {
381 match self {
382 Self::Removed { .. } => lookup_keys::ACTIVE_DELETION_RANK_REMOVED,
383 Self::Listed { .. } => lookup_keys::ACTIVE_DELETION_RANK_LISTED,
384 }
385 }
386}
387
388impl MetadataRowFamily {
389 pub const fn row_key_prefix(self) -> &'static str {
393 match self {
394 Self::Inodes => lookup_keys::INODE_ROW_PREFIX,
395 Self::DirentryBinds => lookup_keys::DIRENTRY_BIND_ROW_PREFIX,
396 Self::DirentryChildBinds => lookup_keys::DIRENTRY_CHILD_BIND_ROW_PREFIX,
397 Self::DirentryUnbinds => lookup_keys::DIRENTRY_UNBIND_ROW_PREFIX,
398 Self::Revisions => lookup_keys::REVISION_ROW_PREFIX,
399 Self::RevisionsByInodeDesc => lookup_keys::REVISION_BY_INODE_DESC_ROW_PREFIX,
400 Self::Tombstones => lookup_keys::TOMBSTONE_ROW_PREFIX,
401 Self::ActiveDeletions => lookup_keys::ACTIVE_DELETION_ROW_PREFIX,
402 Self::CommitReceipts => lookup_keys::COMMIT_RECEIPT_ROW_PREFIX,
403 Self::Attributes => lookup_keys::ATTRIBUTE_ROW_PREFIX,
404 }
405 }
406}
407
408impl MetadataRow {
409 pub fn row_key(&self) -> String {
413 self.row_key_for_family(match self {
414 Self::Inode { .. } => MetadataRowFamily::Inodes,
415 Self::DirentryBind { .. } => MetadataRowFamily::DirentryBinds,
416 Self::DirentryUnbind { .. } => MetadataRowFamily::DirentryUnbinds,
417 Self::FileRevision { .. } => MetadataRowFamily::Revisions,
418 Self::Tombstone { .. } => MetadataRowFamily::Tombstones,
419 Self::ActiveDeletion { .. } => MetadataRowFamily::ActiveDeletions,
420 Self::CommitReceipt { .. } => MetadataRowFamily::CommitReceipts,
421 Self::AttributesRevision { .. } => MetadataRowFamily::Attributes,
422 })
423 }
424
425 pub fn row_key_for_family(&self, family: MetadataRowFamily) -> String {
429 match self {
430 Self::Inode { inode_id, .. } => lookup_keys::inode_key(*inode_id),
431 Self::DirentryBind {
432 parent_inode_id,
433 name_key,
434 child_inode_id,
435 bind_seq,
436 bind_delta_index,
437 ..
438 } => match family {
439 MetadataRowFamily::DirentryChildBinds => lookup_keys::direntry_child_bind_row_key(
440 *child_inode_id,
441 *bind_seq,
442 *bind_delta_index,
443 *parent_inode_id,
444 name_key.as_str(),
445 ),
446 _ => lookup_keys::direntry_bind_row_key(
447 *parent_inode_id,
448 name_key.as_str(),
449 *bind_seq,
450 *bind_delta_index,
451 ),
452 },
453 Self::DirentryUnbind {
454 parent_inode_id,
455 name_key,
456 bind_seq,
457 bind_delta_index,
458 unbind_seq,
459 unbind_delta_index,
460 ..
461 } => lookup_keys::direntry_unbind_row_key(
462 *parent_inode_id,
463 name_key.as_str(),
464 *bind_seq,
465 *bind_delta_index,
466 *unbind_seq,
467 *unbind_delta_index,
468 ),
469 Self::FileRevision {
470 inode_id,
471 revision_no,
472 committed_seq,
473 delta_index,
474 ..
475 } => match family {
476 MetadataRowFamily::RevisionsByInodeDesc => {
477 lookup_keys::revision_by_inode_desc_row_key(
478 *inode_id,
479 *revision_no,
480 *committed_seq,
481 *delta_index,
482 )
483 }
484 _ => lookup_keys::revision_row_key(*inode_id, *revision_no, *delta_index),
485 },
486 Self::Tombstone {
487 root_inode_id,
488 generation,
489 ..
490 } => lookup_keys::tombstone_row_key(*root_inode_id, *generation),
491 Self::ActiveDeletion {
492 root_inode_id,
493 deletion_seq,
494 action,
495 } => lookup_keys::active_deletion_row_key(
496 *deletion_seq,
497 *root_inode_id,
498 action.sort_rank(),
499 ),
500 Self::CommitReceipt {
501 committed_seq,
502 commit_id,
503 ..
504 } => lookup_keys::commit_receipt_row_key(commit_id.as_str(), *committed_seq),
505 Self::AttributesRevision {
506 inode_id,
507 attributes_revision_no,
508 committed_seq,
509 delta_index,
510 ..
511 } => lookup_keys::attributes_row_key(
512 *inode_id,
513 *attributes_revision_no,
514 *committed_seq,
515 *delta_index,
516 ),
517 }
518 }
519
520 pub fn filter_key_for_family(&self, family: MetadataRowFamily) -> String {
522 match self {
523 Self::Inode { .. } => self.row_key_for_family(family),
524 Self::DirentryBind {
525 parent_inode_id,
526 name_key,
527 child_inode_id,
528 ..
529 } => match family {
530 MetadataRowFamily::DirentryChildBinds => {
531 lookup_keys::direntry_child_probe(*child_inode_id)
532 }
533 _ => lookup_keys::direntry_bind_probe(*parent_inode_id, name_key.as_str()),
534 },
535 Self::DirentryUnbind {
536 parent_inode_id,
537 name_key,
538 ..
539 } => lookup_keys::direntry_unbind_probe(*parent_inode_id, name_key.as_str()),
540 Self::FileRevision { inode_id, .. } => match family {
541 MetadataRowFamily::RevisionsByInodeDesc => {
542 lookup_keys::revision_by_inode_desc_probe(*inode_id)
543 }
544 _ => lookup_keys::revision_probe(*inode_id),
545 },
546 Self::Tombstone { root_inode_id, .. } => lookup_keys::tombstone_probe(*root_inode_id),
547 Self::ActiveDeletion { .. } => self.row_key_for_family(family),
550 Self::CommitReceipt { commit_id, .. } => {
551 lookup_keys::commit_receipt_probe(commit_id.as_str())
552 }
553 Self::AttributesRevision { inode_id, .. } => lookup_keys::attributes_probe(*inode_id),
554 }
555 }
556}
557
558pub fn hex_encode_row_key_component(value: &str) -> String {
562 crate::hex::hex_encode_bytes(value.as_bytes())
563}
564
565pub mod lookup_keys {
569 use super::{hex_encode_row_key_component, TombstoneGeneration};
570 use crate::{AttributeRevisionNo, ChangeSeq, InodeId, RevisionNo};
571
572 pub const INODE_ROW_PREFIX: &str = "inode-";
574
575 pub const REVISION_ROW_PREFIX: &str = "revision-";
577
578 pub(super) const DIRENTRY_BIND_ROW_PREFIX: &str = "direntry-bind-";
579 pub(super) const DIRENTRY_CHILD_BIND_ROW_PREFIX: &str = "direntry-child-bind-";
580 pub(super) const DIRENTRY_UNBIND_ROW_PREFIX: &str = "direntry-unbind-";
581 pub(super) const REVISION_BY_INODE_DESC_ROW_PREFIX: &str = "revision-by-inode-desc-";
582 pub(super) const TOMBSTONE_ROW_PREFIX: &str = "tombstone-";
583 pub(super) const COMMIT_RECEIPT_ROW_PREFIX: &str = "commit-receipt-";
584 pub(super) const ATTRIBUTE_ROW_PREFIX: &str = "attribute-";
585
586 pub fn inode_key(inode_id: InodeId) -> String {
588 format!("{INODE_ROW_PREFIX}{:020}", inode_id.0)
589 }
590
591 pub fn inode_key_after(inode_id: InodeId) -> String {
593 format!("{}\0", inode_key(inode_id))
594 }
595
596 pub fn direntry_parent_prefix(parent_inode_id: InodeId) -> String {
598 format!("{DIRENTRY_BIND_ROW_PREFIX}{:020}-", parent_inode_id.0)
599 }
600
601 pub fn direntry_bind_probe(parent_inode_id: InodeId, name_key: &str) -> String {
603 format!(
604 "{}{}",
605 direntry_parent_prefix(parent_inode_id),
606 hex_encode_row_key_component(name_key)
607 )
608 }
609
610 pub fn direntry_bind_prefix(parent_inode_id: InodeId, name_key: &str) -> String {
612 format!("{}-", direntry_bind_probe(parent_inode_id, name_key))
613 }
614
615 pub fn direntry_bind_row_key(
617 parent_inode_id: InodeId,
618 name_key: &str,
619 bind_seq: ChangeSeq,
620 bind_delta_index: u32,
621 ) -> String {
622 format!(
623 "{}{:020}-{bind_delta_index:010}",
624 direntry_bind_prefix(parent_inode_id, name_key),
625 bind_seq.0
626 )
627 }
628
629 pub fn direntry_child_probe(child_inode_id: InodeId) -> String {
631 format!("{DIRENTRY_CHILD_BIND_ROW_PREFIX}{:020}", child_inode_id.0)
632 }
633
634 pub fn direntry_child_prefix(child_inode_id: InodeId) -> String {
636 format!("{}-", direntry_child_probe(child_inode_id))
637 }
638
639 pub fn direntry_child_bind_row_key(
641 child_inode_id: InodeId,
642 bind_seq: ChangeSeq,
643 bind_delta_index: u32,
644 parent_inode_id: InodeId,
645 name_key: &str,
646 ) -> String {
647 format!(
648 "{}{:020}-{bind_delta_index:010}-{:020}-{}",
649 direntry_child_prefix(child_inode_id),
650 bind_seq.0,
651 parent_inode_id.0,
652 hex_encode_row_key_component(name_key)
653 )
654 }
655
656 pub fn direntry_unbind_probe(parent_inode_id: InodeId, name_key: &str) -> String {
658 format!(
659 "{}{}",
660 direntry_unbind_parent_prefix(parent_inode_id),
661 hex_encode_row_key_component(name_key)
662 )
663 }
664
665 pub fn direntry_unbind_binding_prefix(
667 parent_inode_id: InodeId,
668 name_key: &str,
669 bind_seq: ChangeSeq,
670 bind_delta_index: u32,
671 ) -> String {
672 format!(
673 "{}{:020}-{bind_delta_index:010}-",
674 direntry_unbind_name_prefix(parent_inode_id, name_key),
675 bind_seq.0
676 )
677 }
678
679 pub fn direntry_unbind_row_key(
681 parent_inode_id: InodeId,
682 name_key: &str,
683 bind_seq: ChangeSeq,
684 bind_delta_index: u32,
685 unbind_seq: ChangeSeq,
686 unbind_delta_index: u32,
687 ) -> String {
688 format!(
689 "{}{:020}-{unbind_delta_index:010}",
690 direntry_unbind_binding_prefix(parent_inode_id, name_key, bind_seq, bind_delta_index),
691 unbind_seq.0
692 )
693 }
694
695 pub fn direntry_unbind_parent_prefix(parent_inode_id: InodeId) -> String {
697 format!("{DIRENTRY_UNBIND_ROW_PREFIX}{:020}-", parent_inode_id.0)
698 }
699
700 pub fn direntry_unbind_name_prefix(parent_inode_id: InodeId, name_key: &str) -> String {
702 format!("{}-", direntry_unbind_probe(parent_inode_id, name_key))
703 }
704
705 pub fn tombstone_probe(root_inode_id: InodeId) -> String {
707 format!("{TOMBSTONE_ROW_PREFIX}{:020}", root_inode_id.0)
708 }
709
710 pub fn tombstone_prefix(root_inode_id: InodeId) -> String {
712 format!("{}-", tombstone_probe(root_inode_id))
713 }
714
715 pub fn tombstone_row_key(root_inode_id: InodeId, generation: TombstoneGeneration) -> String {
720 format!(
721 "{}{:020}-{:010}",
722 tombstone_prefix(root_inode_id),
723 generation.seq.0,
724 generation.delta_index
725 )
726 }
727
728 pub const ACTIVE_DELETION_ROW_PREFIX: &str = "active-deletion-";
730
731 pub const ACTIVE_DELETION_RANK_REMOVED: u32 = 0;
736
737 pub const ACTIVE_DELETION_RANK_LISTED: u32 = 1;
740
741 pub fn active_deletion_row_key(
743 deletion_seq: ChangeSeq,
744 root_inode_id: InodeId,
745 sort_rank: u32,
746 ) -> String {
747 format!(
748 "{ACTIVE_DELETION_ROW_PREFIX}{:020}-{:020}-{sort_rank:010}",
749 deletion_seq.0, root_inode_id.0
750 )
751 }
752
753 pub fn active_deletion_key_after(deletion_seq: ChangeSeq, root_inode_id: InodeId) -> String {
755 format!(
756 "{}\0",
757 active_deletion_row_key(deletion_seq, root_inode_id, ACTIVE_DELETION_RANK_LISTED)
758 )
759 }
760
761 pub fn commit_receipt_probe(commit_id: &str) -> String {
763 format!(
764 "{COMMIT_RECEIPT_ROW_PREFIX}{}",
765 hex_encode_row_key_component(commit_id)
766 )
767 }
768
769 pub fn commit_receipt_prefix(commit_id: &str) -> String {
771 format!("{}-", commit_receipt_probe(commit_id))
772 }
773
774 pub fn commit_receipt_row_key(commit_id: &str, committed_seq: ChangeSeq) -> String {
776 format!(
777 "{}{:020}",
778 commit_receipt_prefix(commit_id),
779 committed_seq.0
780 )
781 }
782
783 pub fn revision_probe(inode_id: InodeId) -> String {
785 format!("{REVISION_ROW_PREFIX}{:020}", inode_id.0)
786 }
787
788 pub fn revision_row_key(
790 inode_id: InodeId,
791 revision_no: RevisionNo,
792 delta_index: u32,
793 ) -> String {
794 format!(
795 "{}-{:020}-{delta_index:010}",
796 revision_probe(inode_id),
797 revision_no.0
798 )
799 }
800
801 pub fn revision_by_inode_desc_probe(inode_id: InodeId) -> String {
803 format!("{REVISION_BY_INODE_DESC_ROW_PREFIX}{:020}", inode_id.0)
804 }
805
806 pub fn revision_by_inode_desc_prefix(inode_id: InodeId) -> String {
808 format!("{}-", revision_by_inode_desc_probe(inode_id))
809 }
810
811 pub fn revision_by_inode_desc_revision_prefix(
813 inode_id: InodeId,
814 revision_no: RevisionNo,
815 ) -> String {
816 format!(
817 "{}{:020}-",
818 revision_by_inode_desc_prefix(inode_id),
819 u64::MAX - revision_no.0
820 )
821 }
822
823 pub fn revision_by_inode_desc_row_key(
825 inode_id: InodeId,
826 revision_no: RevisionNo,
827 committed_seq: ChangeSeq,
828 delta_index: u32,
829 ) -> String {
830 format!(
831 "{}{:020}-{:010}",
832 revision_by_inode_desc_revision_prefix(inode_id, revision_no),
833 u64::MAX - committed_seq.0,
834 u32::MAX - delta_index
835 )
836 }
837
838 pub fn attributes_probe(inode_id: InodeId) -> String {
840 format!("{ATTRIBUTE_ROW_PREFIX}{:020}", inode_id.0)
841 }
842
843 pub fn attributes_prefix(inode_id: InodeId) -> String {
845 format!("{}-", attributes_probe(inode_id))
846 }
847
848 pub fn attributes_row_key(
850 inode_id: InodeId,
851 attributes_revision_no: AttributeRevisionNo,
852 committed_seq: ChangeSeq,
853 delta_index: u32,
854 ) -> String {
855 format!(
856 "{}{:020}-{:020}-{:010}",
857 attributes_prefix(inode_id),
858 u64::MAX - attributes_revision_no.0,
859 u64::MAX - committed_seq.0,
860 u32::MAX - delta_index
861 )
862 }
863}
864
865#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
869pub struct NamespaceManifestPayload {
870 pub namespace_id: NamespaceId,
872 pub manifest_no: ManifestNo,
874 pub manifest_object_id: ManifestObjectId,
876 pub head_seq: ChangeSeq,
878 pub head_commit_id: CommitId,
880 pub base_seq: ChangeSeq,
882 pub writer_epoch: WriterEpoch,
884 pub next_inode_id: InodeId,
886 pub next_run_no: RunNo,
889 pub retention_floor_seq: ChangeSeq,
891 pub segments: Vec<MetadataSegmentRef>,
893}
894
895#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
901pub struct NamespaceManifestEnvelope {
902 pub kind: NamespaceManifestKind,
904 pub format_version: u32,
906 pub payload_checksum: String,
909 pub payload: NamespaceManifestPayload,
911}
912
913impl NamespaceManifestEnvelope {
914 pub fn from_payload(payload: NamespaceManifestPayload) -> Result<Self, EnvelopeCodecError> {
918 Ok(Self {
919 kind: NamespaceManifestKind::NamespaceManifest,
920 format_version: NAMESPACE_MANIFEST_FORMAT_VERSION,
921 payload_checksum: namespace_manifest_payload_checksum(&payload)?,
922 payload,
923 })
924 }
925}
926
927fn namespace_manifest_payload_checksum(
928 payload: &NamespaceManifestPayload,
929) -> Result<String, EnvelopeCodecError> {
930 crate::envelope::json_payload_checksum(payload)
931}
932
933pub fn encode_namespace_manifest_json(
939 envelope: &NamespaceManifestEnvelope,
940) -> Result<Vec<u8>, EnvelopeCodecError> {
941 crate::envelope::encode_json_envelope(
942 envelope.kind.as_str(),
943 envelope.format_version,
944 NAMESPACE_MANIFEST_FORMAT_VERSION,
945 &envelope.payload_checksum,
946 &envelope.payload,
947 )
948}
949
950pub fn decode_namespace_manifest_json(
956 bytes: &[u8],
957) -> Result<NamespaceManifestEnvelope, EnvelopeCodecError> {
958 let expected_kind = NamespaceManifestKind::NamespaceManifest;
959 let decoded =
960 crate::envelope::decode_json_envelope(bytes, NAMESPACE_MANIFEST_FORMAT_VERSION, |found| {
961 crate::envelope::verify_kind(expected_kind.as_str(), found)
962 })?;
963
964 Ok(NamespaceManifestEnvelope {
965 kind: expected_kind,
966 format_version: decoded.format_version,
967 payload_checksum: decoded.payload_checksum,
968 payload: decoded.payload,
969 })
970}
971
972#[cfg(test)]
973mod tests {
974 use super::{
975 decode_namespace_manifest_json, encode_namespace_manifest_json, BlockHandle,
976 MetadataRowFamily, MetadataSegmentRef, NamespaceManifestEnvelope, NamespaceManifestPayload,
977 };
978 use crate::{
979 ChangeSeq, CommitId, InodeId, ManifestNo, ManifestObjectId, MetadataCompactionId,
980 MetadataSegmentId, NameKey, NamespaceId, RunNo, WriterEpoch,
981 };
982
983 fn row_commit_id() -> CommitId {
984 CommitId::parse("c_metadata_row").expect("commit id")
985 }
986
987 #[test]
988 fn inode_row_keys_sort_by_ascending_inode_id() {
989 let ids = [9_u64, 1, 100, 10, 2];
992 let key_of = |id: u64| super::lookup_keys::inode_key(InodeId(id));
993 let mut keys: Vec<String> = ids.iter().copied().map(key_of).collect();
994 keys.sort();
995
996 let mut ascending_ids = ids;
997 ascending_ids.sort_unstable();
998 assert_eq!(
999 keys,
1000 ascending_ids
1001 .iter()
1002 .copied()
1003 .map(key_of)
1004 .collect::<Vec<_>>(),
1005 "row-key order must agree with inode-id order"
1006 );
1007 assert!(keys
1008 .iter()
1009 .all(|key| key.starts_with(super::lookup_keys::INODE_ROW_PREFIX)));
1010 }
1011
1012 #[test]
1013 fn the_inode_resume_bound_skips_its_own_row_and_nothing_after_it() {
1014 let resume = super::lookup_keys::inode_key_after(InodeId(7));
1015 assert!(resume > super::lookup_keys::inode_key(InodeId(7)));
1016 assert!(resume < super::lookup_keys::inode_key(InodeId(8)));
1017 }
1018
1019 #[test]
1020 fn namespace_manifest_kind_string_matches_serde() {
1021 let kind = super::NamespaceManifestKind::NamespaceManifest;
1022 let serialized = serde_json::to_value(kind).expect("serialize kind");
1023 assert_eq!(serialized, serde_json::Value::from(kind.as_str()));
1024 }
1025
1026 #[test]
1027 fn namespace_manifest_codec_round_trips_base_only_materialization() {
1028 let envelope = NamespaceManifestEnvelope::from_payload(NamespaceManifestPayload {
1029 namespace_id: NamespaceId::parse("demo").expect("valid namespace id"),
1030 manifest_no: ManifestNo(10),
1031 manifest_object_id: ManifestObjectId::parse(
1032 "man_00000000000000000010-0123456789abcdef",
1033 )
1034 .expect("valid manifest object id"),
1035 head_seq: ChangeSeq(10),
1036 head_commit_id: CommitId::parse("c_00000000000000000000000000000001")
1037 .expect("commit id"),
1038 base_seq: ChangeSeq(10),
1039 writer_epoch: WriterEpoch(2),
1040 next_inode_id: InodeId(42),
1041 next_run_no: RunNo(1),
1042 retention_floor_seq: ChangeSeq(0),
1043 segments: vec![metadata_segment_ref(
1044 "demo",
1045 "seg_00000000000000000000000000000001",
1046 RunNo(0),
1047 ChangeSeq(10),
1048 1,
1049 )],
1050 })
1051 .expect("manifest");
1052
1053 let encoded = encode_namespace_manifest_json(&envelope).expect("encode manifest");
1054 let decoded = decode_namespace_manifest_json(&encoded).expect("decode manifest");
1055
1056 assert_eq!(decoded, envelope);
1057 assert_eq!(decoded.payload.base_seq, ChangeSeq(10));
1058 assert_eq!(decoded.payload.segments.len(), 1);
1059 assert_eq!(decoded.payload.segments[0].run_seq, ChangeSeq(10));
1060 }
1061
1062 #[test]
1063 fn namespace_manifest_codec_round_trips_inherited_source_segments() {
1064 let envelope = NamespaceManifestEnvelope::from_payload(NamespaceManifestPayload {
1065 namespace_id: NamespaceId::parse("demo").expect("valid namespace id"),
1066 manifest_no: ManifestNo(12),
1067 manifest_object_id: ManifestObjectId::parse(
1068 "man_00000000000000000012-0123456789abcdef",
1069 )
1070 .expect("valid manifest object id"),
1071 head_seq: ChangeSeq(12),
1072 head_commit_id: CommitId::parse("c_00000000000000000000000000000002")
1073 .expect("commit id"),
1074 base_seq: ChangeSeq(10),
1075 writer_epoch: WriterEpoch(2),
1076 next_inode_id: InodeId(42),
1077 next_run_no: RunNo(2),
1078 retention_floor_seq: ChangeSeq(0),
1079 segments: vec![
1080 metadata_segment_ref(
1081 "source",
1082 "seg_00000000000000000000000000000001",
1083 RunNo(0),
1084 ChangeSeq(10),
1085 1,
1086 ),
1087 metadata_segment_ref(
1088 "demo",
1089 "seg_00000000000000000000000000000002",
1090 RunNo(1),
1091 ChangeSeq(12),
1092 0,
1093 ),
1094 ],
1095 })
1096 .expect("manifest");
1097
1098 let encoded = encode_namespace_manifest_json(&envelope).expect("encode manifest");
1099 let decoded = decode_namespace_manifest_json(&encoded).expect("decode manifest");
1100
1101 assert_eq!(decoded, envelope);
1102 assert_eq!(decoded.payload.segments[0].level, 1);
1103 assert_eq!(decoded.payload.segments[1].level, 0);
1104 assert_eq!(decoded.payload.segments[1].run_seq, ChangeSeq(12));
1105 assert_eq!(
1106 decoded.payload.segments[0].owner_namespace_id,
1107 NamespaceId::parse("source").expect("valid namespace id")
1108 );
1109 }
1110
1111 #[test]
1112 fn namespace_manifest_codec_round_trips_a_compaction_job_segment() {
1113 let compaction_job_id = MetadataCompactionId::parse("cmp_0123456789abcdef0123456789abcdef")
1114 .expect("valid compaction job id");
1115 let mut staged = metadata_segment_ref(
1116 "demo",
1117 "seg_00000000000000000000000000000001",
1118 RunNo(0),
1119 ChangeSeq(14),
1120 1,
1121 );
1122 staged.compaction_job_id = Some(compaction_job_id.clone());
1123 let flushed = metadata_segment_ref(
1124 "demo",
1125 "seg_00000000000000000000000000000002",
1126 RunNo(1),
1127 ChangeSeq(14),
1128 0,
1129 );
1130 let envelope = NamespaceManifestEnvelope::from_payload(NamespaceManifestPayload {
1131 namespace_id: NamespaceId::parse("demo").expect("valid namespace id"),
1132 manifest_no: ManifestNo(14),
1133 manifest_object_id: ManifestObjectId::parse(
1134 "man_00000000000000000014-0123456789abcdef",
1135 )
1136 .expect("valid manifest object id"),
1137 head_seq: ChangeSeq(14),
1138 head_commit_id: CommitId::parse("c_00000000000000000000000000000003")
1139 .expect("commit id"),
1140 base_seq: ChangeSeq(14),
1141 writer_epoch: WriterEpoch(2),
1142 next_inode_id: InodeId(42),
1143 next_run_no: RunNo(2),
1144 retention_floor_seq: ChangeSeq(0),
1145 segments: vec![staged, flushed],
1146 })
1147 .expect("manifest");
1148
1149 let encoded = encode_namespace_manifest_json(&envelope).expect("encode manifest");
1150 let decoded = decode_namespace_manifest_json(&encoded).expect("decode manifest");
1151
1152 assert_eq!(decoded, envelope);
1153 assert_eq!(
1154 decoded.payload.segments[0].compaction_job_id,
1155 Some(compaction_job_id)
1156 );
1157 assert_eq!(decoded.payload.segments[1].compaction_job_id, None);
1158 let text = String::from_utf8(encoded).expect("manifest json is utf-8");
1159 assert_eq!(
1160 text.matches("\"compaction_job_id\"").count(),
1161 1,
1162 "only the compaction job's segment writes the field, got {text}"
1163 );
1164 }
1165
1166 #[test]
1167 fn direntry_bind_row_key_supports_parent_and_child_indexes() {
1168 let row = super::MetadataRow::DirentryBind {
1169 parent_inode_id: InodeId(9),
1170 name_key: NameKey::parse("report.txt").expect("valid name key"),
1171 display_name: crate::DisplayName::parse("Report.txt").expect("valid display name"),
1172 child_inode_id: InodeId(42),
1173 bind_seq: ChangeSeq(17),
1174 bind_delta_index: 3,
1175 };
1176
1177 assert_eq!(
1178 row.row_key_for_family(MetadataRowFamily::DirentryBinds),
1179 "direntry-bind-00000000000000000009-7265706f72742e747874-00000000000000000017-0000000003"
1180 );
1181 assert_eq!(
1182 row.row_key_for_family(MetadataRowFamily::DirentryChildBinds),
1183 "direntry-child-bind-00000000000000000042-00000000000000000017-0000000003-00000000000000000009-7265706f72742e747874"
1184 );
1185 }
1186
1187 #[test]
1188 fn row_keys_hex_encode_dash_containing_variable_components() {
1189 let row = super::MetadataRow::DirentryBind {
1190 parent_inode_id: InodeId(9),
1191 name_key: NameKey::parse("report-2024").expect("valid name key"),
1192 display_name: crate::DisplayName::parse("report-2024").expect("valid display name"),
1193 child_inode_id: InodeId(42),
1194 bind_seq: ChangeSeq(17),
1195 bind_delta_index: 3,
1196 };
1197
1198 assert_eq!(
1199 row.row_key_for_family(MetadataRowFamily::DirentryBinds),
1200 "direntry-bind-00000000000000000009-7265706f72742d32303234-00000000000000000017-0000000003"
1201 );
1202 }
1203
1204 #[test]
1205 fn revision_row_key_supports_newest_first_inode_index() {
1206 let row = super::MetadataRow::FileRevision {
1207 inode_id: InodeId(42),
1208 revision_no: crate::RevisionNo(7),
1209 committed_seq: ChangeSeq(12),
1210 commit_id: row_commit_id(),
1211 committed_at_ms: 12_000,
1212 committed_by: crate::ActorRef::loonfs_system(),
1213 delta_index: 3,
1214 content_ref: crate::ContentRef::blob_v1(
1215 crate::ContentId::parse("con_0123456789abcdef0123456789abcdef")
1216 .expect("valid content id"),
1217 b"row key sample",
1218 ),
1219 };
1220
1221 assert_eq!(
1222 row.row_key_for_family(MetadataRowFamily::Revisions),
1223 "revision-00000000000000000042-00000000000000000007-0000000003"
1224 );
1225 assert_eq!(
1226 row.row_key_for_family(MetadataRowFamily::RevisionsByInodeDesc),
1227 "revision-by-inode-desc-00000000000000000042-18446744073709551608-18446744073709551603-4294967292"
1228 );
1229 }
1230
1231 #[test]
1232 fn attributes_row_keys_sort_newest_revision_first_under_the_inode_prefix() {
1233 let row_of =
1234 |revision: u64, seq: u64, delta_index: u32| super::MetadataRow::AttributesRevision {
1235 inode_id: InodeId(42),
1236 attributes_revision_no: crate::AttributeRevisionNo(revision),
1237 committed_seq: ChangeSeq(seq),
1238 commit_id: row_commit_id(),
1239 delta_index,
1240 updated_by: crate::ActorRef::loonfs_system(),
1241 updated_at_ms: 12_000 + seq,
1242 attributes: crate::Attributes::default(),
1243 };
1244 let newest = row_of(3, 12, 1);
1245 let older = row_of(2, 11, 0);
1246
1247 assert_eq!(
1248 newest.row_key_for_family(MetadataRowFamily::Attributes),
1249 "attribute-00000000000000000042-18446744073709551612-18446744073709551603-4294967294"
1250 );
1251 assert_eq!(
1252 newest.row_key(),
1253 newest.row_key_for_family(MetadataRowFamily::Attributes)
1254 );
1255 assert!(
1256 newest.row_key() < older.row_key(),
1257 "an ascending scan must reach the newest revision first"
1258 );
1259 let prefix = super::lookup_keys::attributes_prefix(InodeId(42));
1260 assert!(newest.row_key().starts_with(&prefix));
1261 assert!(older.row_key().starts_with(&prefix));
1262 assert_eq!(
1265 newest.filter_key_for_family(MetadataRowFamily::Attributes),
1266 super::lookup_keys::attributes_probe(InodeId(42))
1267 );
1268 assert!(!row_of(3, 12, 1)
1270 .row_key()
1271 .starts_with(&super::lookup_keys::attributes_prefix(InodeId(43))));
1272 }
1273
1274 #[test]
1275 fn row_key_prefixes_match_the_row_keys_they_front() {
1276 let name_key = NameKey::parse("report.txt").expect("valid name key");
1277 let display_name = crate::DisplayName::parse("report.txt").expect("valid display name");
1278 let bind = super::MetadataRow::DirentryBind {
1279 parent_inode_id: InodeId(9),
1280 name_key: name_key.clone(),
1281 display_name: display_name.clone(),
1282 child_inode_id: InodeId(42),
1283 bind_seq: ChangeSeq(17),
1284 bind_delta_index: 3,
1285 };
1286 let revision = super::MetadataRow::FileRevision {
1287 inode_id: InodeId(42),
1288 revision_no: crate::RevisionNo(7),
1289 committed_seq: ChangeSeq(12),
1290 commit_id: row_commit_id(),
1291 committed_at_ms: 12_000,
1292 committed_by: crate::ActorRef::loonfs_system(),
1293 delta_index: 3,
1294 content_ref: crate::ContentRef::blob_v1(
1295 crate::ContentId::parse("con_0123456789abcdef0123456789abcdef")
1296 .expect("valid content id"),
1297 b"row key prefix sample",
1298 ),
1299 };
1300 let rows: [(MetadataRowFamily, super::MetadataRow); 10] = [
1301 (
1302 MetadataRowFamily::Inodes,
1303 super::MetadataRow::Inode {
1304 inode_id: InodeId(42),
1305 inode_kind: crate::InodeKind::File,
1306 created_seq: ChangeSeq(3),
1307 commit_id: row_commit_id(),
1308 created_by: crate::ActorRef::loonfs_system(),
1309 created_at_ms: 3_000,
1310 },
1311 ),
1312 (MetadataRowFamily::DirentryBinds, bind.clone()),
1313 (MetadataRowFamily::DirentryChildBinds, bind),
1314 (
1315 MetadataRowFamily::DirentryUnbinds,
1316 super::MetadataRow::DirentryUnbind {
1317 parent_inode_id: InodeId(9),
1318 name_key,
1319 display_name,
1320 child_inode_id: InodeId(42),
1321 bind_seq: ChangeSeq(17),
1322 bind_delta_index: 3,
1323 unbind_seq: ChangeSeq(19),
1324 unbind_delta_index: 0,
1325 },
1326 ),
1327 (MetadataRowFamily::Revisions, revision.clone()),
1328 (MetadataRowFamily::RevisionsByInodeDesc, revision),
1329 (
1330 MetadataRowFamily::Tombstones,
1331 super::MetadataRow::Tombstone {
1332 root_inode_id: InodeId(42),
1333 generation: super::TombstoneGeneration {
1334 seq: ChangeSeq(12),
1335 delta_index: 0,
1336 },
1337 commit_id: row_commit_id(),
1338 action: super::TombstoneRowAction::Set {
1339 deleted_direntry: None,
1340 },
1341 deleted_at_ms: 12_000,
1342 deleted_by: crate::ActorRef::loonfs_system(),
1343 },
1344 ),
1345 (
1346 MetadataRowFamily::ActiveDeletions,
1347 super::MetadataRow::ActiveDeletion {
1348 root_inode_id: InodeId(42),
1349 deletion_seq: ChangeSeq(12),
1350 action: super::ActiveDeletionRowAction::Removed {
1351 revocation_seq: ChangeSeq(15),
1352 },
1353 },
1354 ),
1355 (
1356 MetadataRowFamily::CommitReceipts,
1357 super::MetadataRow::CommitReceipt {
1358 commit_id: CommitId::parse("c_00000000000000000000000000000001")
1359 .expect("commit id"),
1360 committed_by: crate::ActorRef::loonfs_system(),
1361 semantic_commit_fingerprint: "sha256:unused".to_owned(),
1362 committed_seq: ChangeSeq(12),
1363 committed_at_ms: 12_000,
1364 message: None,
1365 },
1366 ),
1367 (
1368 MetadataRowFamily::Attributes,
1369 super::MetadataRow::AttributesRevision {
1370 inode_id: InodeId(42),
1371 attributes_revision_no: crate::AttributeRevisionNo(3),
1372 committed_seq: ChangeSeq(12),
1373 commit_id: row_commit_id(),
1374 delta_index: 0,
1375 updated_by: crate::ActorRef::loonfs_system(),
1376 updated_at_ms: 12_000,
1377 attributes: crate::Attributes::default(),
1378 },
1379 ),
1380 ];
1381
1382 for (family, row) in rows {
1383 let row_key = row.row_key_for_family(family);
1384 let prefix = family.row_key_prefix();
1385 assert!(
1386 !prefix.is_empty(),
1387 "`{family:?}` declares no row-key prefix"
1388 );
1389 assert!(
1390 row_key.starts_with(prefix),
1391 "row key `{row_key}` for `{family:?}` does not start with `{prefix}`"
1392 );
1393 }
1394 }
1395
1396 #[test]
1397 fn attribution_values_never_change_row_or_index_keys() {
1398 fn rows(actor: crate::ActorRef) -> Vec<(MetadataRowFamily, super::MetadataRow)> {
1399 vec![
1400 (
1401 MetadataRowFamily::Inodes,
1402 super::MetadataRow::Inode {
1403 inode_id: InodeId(42),
1404 inode_kind: crate::InodeKind::File,
1405 created_seq: ChangeSeq(3),
1406 commit_id: row_commit_id(),
1407 created_by: actor.clone(),
1408 created_at_ms: 3_000,
1409 },
1410 ),
1411 (
1412 MetadataRowFamily::RevisionsByInodeDesc,
1413 super::MetadataRow::FileRevision {
1414 inode_id: InodeId(42),
1415 revision_no: crate::RevisionNo(7),
1416 committed_seq: ChangeSeq(12),
1417 commit_id: row_commit_id(),
1418 committed_at_ms: 12_000,
1419 committed_by: actor.clone(),
1420 delta_index: 3,
1421 content_ref: crate::ContentRef::blob_v1(
1422 crate::ContentId::parse("con_0123456789abcdef0123456789abcdef")
1423 .expect("content id"),
1424 b"attribution key test",
1425 ),
1426 },
1427 ),
1428 (
1429 MetadataRowFamily::Tombstones,
1430 super::MetadataRow::Tombstone {
1431 root_inode_id: InodeId(42),
1432 generation: super::TombstoneGeneration {
1433 seq: ChangeSeq(12),
1434 delta_index: 3,
1435 },
1436 commit_id: row_commit_id(),
1437 action: super::TombstoneRowAction::Set {
1438 deleted_direntry: None,
1439 },
1440 deleted_at_ms: 12_000,
1441 deleted_by: actor.clone(),
1442 },
1443 ),
1444 (
1445 MetadataRowFamily::ActiveDeletions,
1446 super::MetadataRow::ActiveDeletion {
1447 root_inode_id: InodeId(42),
1448 deletion_seq: ChangeSeq(12),
1449 action: super::ActiveDeletionRowAction::Listed {
1450 deleted_at_ms: 12_000,
1451 deleted_by: actor.clone(),
1452 deleted_direntry: None,
1453 },
1454 },
1455 ),
1456 (
1457 MetadataRowFamily::Attributes,
1458 super::MetadataRow::AttributesRevision {
1459 inode_id: InodeId(42),
1460 attributes_revision_no: crate::AttributeRevisionNo(2),
1461 committed_seq: ChangeSeq(12),
1462 commit_id: row_commit_id(),
1463 delta_index: 3,
1464 updated_by: actor,
1465 updated_at_ms: 12_000,
1466 attributes: crate::Attributes::default(),
1467 },
1468 ),
1469 ]
1470 }
1471
1472 let actors = [
1473 crate::ActorRef::user(crate::ActorId::parse("auth0|x").expect("actor id")),
1474 crate::ActorRef::service(
1475 crate::ActorId::parse("x".repeat(256)).expect("256-byte actor id"),
1476 ),
1477 crate::ActorRef::system(crate::ActorId::parse("雪-actor").expect("unicode actor id")),
1478 ];
1479 let baseline = rows(actors[0].clone());
1480 for actor in actors.into_iter().skip(1) {
1481 let changed = rows(actor);
1482 for ((family, baseline), (changed_family, changed)) in baseline.iter().zip(&changed) {
1483 assert_eq!(family, changed_family);
1484 assert_eq!(
1485 baseline.row_key_for_family(*family),
1486 changed.row_key_for_family(*family)
1487 );
1488 assert_eq!(
1489 baseline.filter_key_for_family(*family),
1490 changed.filter_key_for_family(*family)
1491 );
1492 }
1493 }
1494 }
1495
1496 fn metadata_segment_ref(
1497 owner_namespace_id: &str,
1498 segment_id: &str,
1499 run_no: RunNo,
1500 run_seq: ChangeSeq,
1501 level: u32,
1502 ) -> MetadataSegmentRef {
1503 MetadataSegmentRef {
1504 owner_namespace_id: NamespaceId::parse(owner_namespace_id).expect("valid namespace id"),
1505 segment_id: MetadataSegmentId::parse(segment_id).expect("valid segment id"),
1506 compaction_job_id: None,
1507 run_no,
1508 run_seq,
1509 level,
1510 family: MetadataRowFamily::Inodes,
1511 segment_index: 0,
1512 row_count: 0,
1513 min_row_key: String::new(),
1514 max_row_key: String::new(),
1515 index_block: BlockHandle {
1516 offset: 0,
1517 stored_len: 0,
1518 decoded_len: 0,
1519 crc32c: 0,
1520 },
1521 filter_block: BlockHandle {
1522 offset: 0,
1523 stored_len: 0,
1524 decoded_len: 0,
1525 crc32c: 0,
1526 },
1527 filter_inline: None,
1528 object_checksum: "sha256:unused".to_owned(),
1529 }
1530 }
1531}