use crate::envelope::EnvelopeCodecError;
use crate::WriterEpoch;
use crate::{
ChangeSeq, CheckpointId, ChecksumAlgorithm, CommitId, ContentId, ContentRef, ContentRefKind,
ContentStoreId, InodeId, ManifestId, ManifestObjectId, NamespaceId, StorageChecksum, UploadId,
WalSegmentId, ROOT_INODE_ID,
};
use serde::de::DeserializeOwned;
use serde::{Deserialize, Deserializer, Serialize};
use std::fmt;
use std::num::NonZeroU64;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ControlObjectKind {
WalHead,
WalFloor,
MetadataRoot,
CheckpointRecord,
UploadSession,
}
impl ControlObjectKind {
pub const ALL: [Self; 5] = [
Self::WalHead,
Self::WalFloor,
Self::MetadataRoot,
Self::CheckpointRecord,
Self::UploadSession,
];
pub const fn format_version(self) -> u32 {
match self {
Self::WalHead => 1,
Self::WalFloor => 1,
Self::MetadataRoot => 1,
Self::CheckpointRecord => 1,
Self::UploadSession => 1,
}
}
pub const fn as_str(self) -> &'static str {
match self {
Self::WalHead => "wal_head",
Self::WalFloor => "wal_floor",
Self::MetadataRoot => "metadata_root",
Self::CheckpointRecord => "checkpoint_record",
Self::UploadSession => "upload_session",
}
}
pub fn parse(value: &str) -> Option<Self> {
Self::ALL.into_iter().find(|kind| kind.as_str() == value)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct WalFloorState {
pub namespace_id: NamespaceId,
pub floor_seq: ChangeSeq,
pub verified_at_ms: u64,
pub updated_at_ms: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MetadataRootState {
pub namespace_id: NamespaceId,
pub manifest_id: ManifestId,
pub manifest_object_id: ManifestObjectId,
pub manifest_head_seq: ChangeSeq,
pub manifest_payload_checksum: String,
pub updated_at_ms: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum CheckpointRecordLifecycle {
Active {},
Released {
released_at_ms: u64,
},
}
impl std::fmt::Display for CheckpointRecordLifecycle {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let state = match self {
Self::Active {} => "active",
Self::Released { .. } => "released",
};
formatter.write_str(state)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum CheckpointOwner {
User {
name: String,
},
Fork {
target_namespace_id: NamespaceId,
},
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CheckpointRecordState {
pub checkpoint_id: CheckpointId,
pub namespace_id: NamespaceId,
pub manifest_id: ManifestId,
pub manifest_object_id: ManifestObjectId,
pub manifest_head_seq: ChangeSeq,
pub manifest_payload_checksum: String,
pub head_commit_id: CommitId,
pub created_at_ms: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub expires_at_ms: Option<u64>,
pub owner: CheckpointOwner,
pub state: CheckpointRecordLifecycle,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct WalSegmentPointer {
pub object_key: String,
pub segment_id: WalSegmentId,
pub start_seq: ChangeSeq,
pub end_seq: ChangeSeq,
pub payload_checksum: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct WriterBlock {
pub writer_id: String,
pub acquired_at_ms: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct AcquiredWriter {
pub writer_id: String,
pub writer_epoch: WriterEpoch,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum NamespaceState {
#[default]
Active,
Deleted,
}
impl NamespaceState {
pub fn is_active(&self) -> bool {
matches!(self, NamespaceState::Active)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ForkBasis {
pub source_namespace_id: NamespaceId,
pub source_manifest_object_id: ManifestObjectId,
pub source_manifest_checksum: String,
pub source_checkpoint_id: CheckpointId,
pub fork_seq: ChangeSeq,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct HeadState {
pub namespace_id: NamespaceId,
pub content_store_id: ContentStoreId,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub fork_basis: Option<ForkBasis>,
pub seq: ChangeSeq,
pub head_commit_id: CommitId,
pub writer_epoch: WriterEpoch,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub writer: Option<WriterBlock>,
pub next_inode_id: InodeId,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub visible_wal_tip: Option<WalSegmentPointer>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub recent_segments: Vec<WalSegmentPointer>,
#[serde(default, skip_serializing_if = "NamespaceState::is_active")]
pub state: NamespaceState,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct StrictHeadState {
namespace_id: NamespaceId,
content_store_id: ContentStoreId,
#[serde(default)]
fork_basis: Option<ForkBasis>,
seq: ChangeSeq,
head_commit_id: CommitId,
writer_epoch: WriterEpoch,
#[serde(default)]
writer: Option<WriterBlock>,
next_inode_id: InodeId,
#[serde(default)]
visible_wal_tip: Option<StrictWalSegmentPointer>,
#[serde(default)]
recent_segments: Vec<StrictWalSegmentPointer>,
#[serde(default)]
state: NamespaceState,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct StrictWalSegmentPointer {
object_key: String,
segment_id: WalSegmentId,
start_seq: ChangeSeq,
end_seq: ChangeSeq,
payload_checksum: String,
}
impl From<StrictWalSegmentPointer> for WalSegmentPointer {
fn from(pointer: StrictWalSegmentPointer) -> Self {
Self {
object_key: pointer.object_key,
segment_id: pointer.segment_id,
start_seq: pointer.start_seq,
end_seq: pointer.end_seq,
payload_checksum: pointer.payload_checksum,
}
}
}
impl<'de> Deserialize<'de> for HeadState {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let state = StrictHeadState::deserialize(deserializer)?;
Ok(Self {
namespace_id: state.namespace_id,
content_store_id: state.content_store_id,
fork_basis: state.fork_basis,
seq: state.seq,
head_commit_id: state.head_commit_id,
writer_epoch: state.writer_epoch,
writer: state.writer,
next_inode_id: state.next_inode_id,
visible_wal_tip: state.visible_wal_tip.map(Into::into),
recent_segments: state.recent_segments.into_iter().map(Into::into).collect(),
state: state.state,
})
}
}
const GENESIS_COMMIT_ID: &str = "c_00000000000000000000000000000000";
pub fn genesis_commit_id() -> CommitId {
CommitId::parse(GENESIS_COMMIT_ID).expect("genesis commit id is valid")
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct HeadIdentityDrift {
pub field: String,
}
impl fmt::Display for HeadIdentityDrift {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
formatter,
"successor head changes the namespace's immutable `{}`",
self.field
)
}
}
impl HeadState {
pub fn initial(namespace_id: NamespaceId, content_store_id: ContentStoreId) -> Self {
Self {
namespace_id,
content_store_id,
fork_basis: None,
seq: ChangeSeq(0),
head_commit_id: CommitId::parse(GENESIS_COMMIT_ID).expect("genesis commit id is valid"),
writer_epoch: WriterEpoch(0),
writer: None,
next_inode_id: InodeId(ROOT_INODE_ID.0 + 1),
visible_wal_tip: None,
recent_segments: Vec::new(),
state: NamespaceState::Active,
}
}
pub fn ensure_successor_identity(
&self,
successor: &HeadState,
) -> Result<(), HeadIdentityDrift> {
let drift = |field: &str| {
Err(HeadIdentityDrift {
field: field.to_owned(),
})
};
if successor.namespace_id != self.namespace_id {
return drift("namespace_id");
}
if successor.content_store_id != self.content_store_id {
return drift("content_store_id");
}
if successor.fork_basis != self.fork_basis {
return drift("fork_basis");
}
Ok(())
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum UploadSessionTransport {
ServiceProxied {},
DirectPut {
promised_content: ContentRef,
},
DirectMultipart {
provider_upload_id: String,
part_size_bytes: NonZeroU64,
},
}
impl UploadSessionTransport {
fn promised_content(&self) -> Option<&ContentRef> {
match self {
Self::DirectPut { promised_content } => Some(promised_content),
Self::ServiceProxied {} | Self::DirectMultipart { .. } => None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum UploadSessionLifecycle {
Open {
expires_at_ms: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
staged_content: Option<ContentRef>,
},
Completed {
completed_at_ms: u64,
content_ref: ContentRef,
},
Aborted {
aborted_at_ms: u64,
},
}
impl UploadSessionLifecycle {
fn content_ref(&self) -> Option<&ContentRef> {
match self {
Self::Open { staged_content, .. } => staged_content.as_ref(),
Self::Completed { content_ref, .. } => Some(content_ref),
Self::Aborted { .. } => None,
}
}
}
impl std::fmt::Display for UploadSessionLifecycle {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let state = match self {
Self::Open { .. } => "open",
Self::Completed { .. } => "completed",
Self::Aborted { .. } => "aborted",
};
formatter.write_str(state)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct UploadSessionState {
pub namespace_id: NamespaceId,
pub upload_id: UploadId,
pub content_id: ContentId,
pub created_at_ms: u64,
pub transport: UploadSessionTransport,
pub state: UploadSessionLifecycle,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct StrictUploadSessionState {
namespace_id: NamespaceId,
upload_id: UploadId,
content_id: ContentId,
created_at_ms: u64,
transport: StrictUploadSessionTransport,
state: StrictUploadSessionLifecycle,
}
#[derive(Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
enum StrictUploadSessionTransport {
ServiceProxied {},
DirectPut {
promised_content: StrictContentRef,
},
DirectMultipart {
provider_upload_id: String,
part_size_bytes: NonZeroU64,
},
}
impl From<StrictUploadSessionTransport> for UploadSessionTransport {
fn from(transport: StrictUploadSessionTransport) -> Self {
match transport {
StrictUploadSessionTransport::ServiceProxied {} => Self::ServiceProxied {},
StrictUploadSessionTransport::DirectPut { promised_content } => Self::DirectPut {
promised_content: promised_content.into(),
},
StrictUploadSessionTransport::DirectMultipart {
provider_upload_id,
part_size_bytes,
} => Self::DirectMultipart {
provider_upload_id,
part_size_bytes,
},
}
}
}
#[derive(Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
enum StrictUploadSessionLifecycle {
Open {
expires_at_ms: u64,
#[serde(default)]
staged_content: Option<StrictContentRef>,
},
Completed {
completed_at_ms: u64,
content_ref: StrictContentRef,
},
Aborted {
aborted_at_ms: u64,
},
}
impl From<StrictUploadSessionLifecycle> for UploadSessionLifecycle {
fn from(state: StrictUploadSessionLifecycle) -> Self {
match state {
StrictUploadSessionLifecycle::Open {
expires_at_ms,
staged_content,
} => Self::Open {
expires_at_ms,
staged_content: staged_content.map(Into::into),
},
StrictUploadSessionLifecycle::Completed {
completed_at_ms,
content_ref,
} => Self::Completed {
completed_at_ms,
content_ref: content_ref.into(),
},
StrictUploadSessionLifecycle::Aborted { aborted_at_ms } => {
Self::Aborted { aborted_at_ms }
}
}
}
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct StrictContentRef {
kind: MutableContentRefKind,
content_id: ContentId,
size_bytes: u64,
storage_checksum: StrictStorageChecksum,
#[serde(default)]
whole_file_sha256: Option<String>,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct StrictStorageChecksum {
algorithm: ChecksumAlgorithm,
value: String,
}
#[derive(Deserialize)]
#[serde(rename_all = "snake_case")]
enum MutableContentRefKind {
BlobV1,
}
impl From<StrictStorageChecksum> for StorageChecksum {
fn from(checksum: StrictStorageChecksum) -> Self {
Self {
algorithm: checksum.algorithm,
value: checksum.value,
}
}
}
impl From<StrictContentRef> for ContentRef {
fn from(content_ref: StrictContentRef) -> Self {
let kind = match content_ref.kind {
MutableContentRefKind::BlobV1 => ContentRefKind::BlobV1,
};
Self {
kind,
content_id: content_ref.content_id,
size_bytes: content_ref.size_bytes,
storage_checksum: content_ref.storage_checksum.into(),
whole_file_sha256: content_ref.whole_file_sha256,
}
}
}
impl<'de> Deserialize<'de> for UploadSessionState {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let state = StrictUploadSessionState::deserialize(deserializer)?;
let transport = UploadSessionTransport::from(state.transport);
let lifecycle = UploadSessionLifecycle::from(state.state);
for content_ref in transport
.promised_content()
.into_iter()
.chain(lifecycle.content_ref())
{
if content_ref.content_id != state.content_id {
return Err(serde::de::Error::custom(format!(
"upload session `{}` owns content `{}` but holds a reference to `{}`",
state.upload_id, state.content_id, content_ref.content_id
)));
}
}
Ok(Self {
namespace_id: state.namespace_id,
upload_id: state.upload_id,
content_id: state.content_id,
created_at_ms: state.created_at_ms,
transport,
state: lifecycle,
})
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ControlObjectEnvelope<T> {
pub kind: ControlObjectKind,
pub format_version: u32,
pub payload_checksum: String,
pub state: T,
}
impl<T> ControlObjectEnvelope<T>
where
T: Serialize,
{
pub fn from_state(kind: ControlObjectKind, state: T) -> Result<Self, EnvelopeCodecError> {
Ok(Self {
kind,
format_version: kind.format_version(),
payload_checksum: control_payload_checksum(&state)?,
state,
})
}
}
pub type HeadStateEnvelope = ControlObjectEnvelope<HeadState>;
pub type UploadSessionEnvelope = ControlObjectEnvelope<UploadSessionState>;
pub type MetadataRootEnvelope = ControlObjectEnvelope<MetadataRootState>;
pub type WalFloorEnvelope = ControlObjectEnvelope<WalFloorState>;
pub type CheckpointRecordEnvelope = ControlObjectEnvelope<CheckpointRecordState>;
pub fn control_payload_checksum<T>(state: &T) -> Result<String, EnvelopeCodecError>
where
T: Serialize,
{
crate::envelope::json_payload_checksum(state)
}
pub fn encode_control_object<T>(
envelope: &ControlObjectEnvelope<T>,
) -> Result<Vec<u8>, EnvelopeCodecError>
where
T: Serialize,
{
crate::envelope::encode_json_envelope(
envelope.kind.as_str(),
envelope.format_version,
envelope.kind.format_version(),
&envelope.payload_checksum,
&envelope.state,
)
}
pub fn decode_control_object<T>(
bytes: &[u8],
expected_kind: ControlObjectKind,
) -> Result<ControlObjectEnvelope<T>, EnvelopeCodecError>
where
T: DeserializeOwned,
{
let decoded = crate::envelope::decode_strict_json_envelope(
bytes,
expected_kind.format_version(),
|found| match ControlObjectKind::parse(found) {
None => Err(EnvelopeCodecError::UnknownKind {
found: found.to_owned(),
}),
Some(kind) if kind != expected_kind => Err(EnvelopeCodecError::KindMismatch {
expected: expected_kind.as_str().to_owned(),
found: found.to_owned(),
}),
Some(_) => Ok(()),
},
)?;
Ok(ControlObjectEnvelope {
kind: expected_kind,
format_version: decoded.format_version,
payload_checksum: decoded.payload_checksum,
state: decoded.payload,
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn control_object_kind_strings_round_trip_and_match_serde() {
for kind in ControlObjectKind::ALL {
assert_eq!(ControlObjectKind::parse(kind.as_str()), Some(kind));
let serialized = serde_json::to_value(kind).expect("serialize kind");
assert_eq!(serialized, serde_json::Value::from(kind.as_str()));
}
assert_eq!(ControlObjectKind::parse("not_a_kind"), None);
}
fn sample_head() -> HeadState {
HeadState::initial(
NamespaceId::parse("demo").expect("valid namespace id"),
ContentStoreId::parse("cs_0123456789abcdef0123456789abcdef")
.expect("valid content store id"),
)
}
#[test]
fn head_without_content_store_is_rejected() {
let missing = serde_json::json!({
"namespace_id": "demo",
"seq": 0,
"head_commit_id": GENESIS_COMMIT_ID,
"writer_epoch": 0,
"next_inode_id": 2
});
serde_json::from_value::<HeadState>(missing)
.expect_err("head without its immutable identity must be rejected");
}
#[test]
fn control_object_codec_round_trips_and_validates() {
let envelope = HeadStateEnvelope::from_state(ControlObjectKind::WalHead, sample_head())
.expect("envelope");
let encoded = encode_control_object(&envelope).expect("encode");
let decoded: HeadStateEnvelope =
decode_control_object(&encoded, ControlObjectKind::WalHead).expect("decode");
assert_eq!(decoded, envelope);
let mismatch =
decode_control_object::<MetadataRootState>(&encoded, ControlObjectKind::MetadataRoot)
.expect_err("kind mismatch");
assert!(matches!(mismatch, EnvelopeCodecError::KindMismatch { .. }));
}
#[test]
fn successor_head_must_carry_the_namespace_identity_forward() {
let head = sample_head();
let mut successor = head.clone();
successor.seq = ChangeSeq(4);
head.ensure_successor_identity(&successor)
.expect("advancing the sequence keeps the identity");
let mut drifted = head.clone();
drifted.content_store_id = ContentStoreId::parse("cs_fedcba9876543210fedcba9876543210")
.expect("valid content store id");
assert_eq!(
head.ensure_successor_identity(&drifted)
.expect_err("content store drift is rejected")
.field,
"content_store_id"
);
let mut forked = head.clone();
forked.fork_basis = Some(ForkBasis {
source_namespace_id: NamespaceId::parse("source").expect("valid namespace id"),
source_manifest_object_id: ManifestObjectId::parse(
"00000000000000000007-0123456789abcdef",
)
.expect("valid manifest object id"),
source_manifest_checksum: "sha256:test".to_owned(),
source_checkpoint_id: CheckpointId::parse("chk_00000000000000000000000000000002")
.expect("valid checkpoint id"),
fork_seq: ChangeSeq(7),
});
assert_eq!(
head.ensure_successor_identity(&forked)
.expect_err("gaining a fork basis is rejected")
.field,
"fork_basis"
);
}
}