use crate::envelope::EnvelopeCodecError;
use crate::{
ChangeSeq, CheckpointId, ChecksumAlgorithm, CommitId, ContentId, ContentRef, ContentStoreId,
ManifestNo, NamespaceId, SubjectId, UploadId,
};
use crate::{WriterEpoch, WriterId};
use serde::de::DeserializeOwned;
use serde::{Deserialize, Deserializer, Serialize};
use std::num::NonZeroU64;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ControlObjectKind {
Hint,
CheckpointRecord,
UploadSession,
ContentStore,
}
impl ControlObjectKind {
pub const ALL: [Self; 4] = [
Self::Hint,
Self::CheckpointRecord,
Self::UploadSession,
Self::ContentStore,
];
pub const fn format_version(self) -> u32 {
match self {
Self::Hint => 1,
Self::CheckpointRecord => 1,
Self::UploadSession => 1,
Self::ContentStore => 1,
}
}
pub const fn as_str(self) -> &'static str {
match self {
Self::Hint => "hint",
Self::CheckpointRecord => "checkpoint_record",
Self::UploadSession => "upload_session",
Self::ContentStore => "content_store",
}
}
pub fn parse(value: &str) -> Option<Self> {
Self::ALL.into_iter().find(|kind| kind.as_str() == value)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ContentStoreState {
pub content_store_id: ContentStoreId,
pub created_at_ms: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct HintState {
pub namespace_id: NamespaceId,
pub manifest_no: ManifestNo,
pub wal_no: crate::WalNo,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ManifestRef {
pub owner_namespace_id: NamespaceId,
pub manifest_no: ManifestNo,
pub manifest_head_seq: ChangeSeq,
pub manifest_payload_checksum: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum CheckpointOwner {
User {
name: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
expires_at_ms: Option<u64>,
},
Fork {
target_namespace_id: NamespaceId,
},
Snapshot {
name: String,
expires_at_ms: u64,
},
}
impl CheckpointOwner {
pub fn expires_at_ms(&self) -> Option<u64> {
match self {
Self::User { expires_at_ms, .. } => *expires_at_ms,
Self::Fork { .. } => None,
Self::Snapshot { expires_at_ms, .. } => Some(*expires_at_ms),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CheckpointRecordState {
pub namespace_id: NamespaceId,
pub pin_id: CheckpointId,
pub manifest_no: ManifestNo,
pub manifest_head_seq: ChangeSeq,
pub manifest_payload_checksum: String,
pub head_commit_id: CommitId,
pub created_at_ms: u64,
pub owner: CheckpointOwner,
}
impl CheckpointRecordState {
pub fn manifest(&self) -> ManifestRef {
ManifestRef {
owner_namespace_id: self.namespace_id.clone(),
manifest_no: self.pin_id.manifest_no(),
manifest_head_seq: self.manifest_head_seq,
manifest_payload_checksum: self.manifest_payload_checksum.clone(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct WriterBlock {
pub writer_id: WriterId,
pub acquired_at_ms: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct AcquiredWriter {
pub writer_id: WriterId,
pub writer_epoch: WriterEpoch,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum NamespaceStatus {
Active {},
Deleted {
#[serde(default, skip_serializing_if = "Option::is_none")]
reclaim_after_ms: Option<u64>,
},
}
impl NamespaceStatus {
pub const fn is_deleted(&self) -> bool {
matches!(self, Self::Deleted { .. })
}
pub const fn reclaim_after_ms(&self) -> Option<u64> {
match self {
Self::Deleted { reclaim_after_ms } => *reclaim_after_ms,
Self::Active {} => None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ForkBasis {
pub manifest: ManifestRef,
pub source_checkpoint_id: CheckpointId,
}
const GENESIS_COMMIT_ID: &str = "c_00000000000000000000000000000000";
pub fn genesis_commit_id() -> CommitId {
CommitId::parse(GENESIS_COMMIT_ID).expect("genesis commit id is valid")
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ProxiedStaging {
Idle,
Claimed,
Staged(ContentRef),
}
impl Serialize for ProxiedStaging {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
#[derive(Serialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
enum Shape<'a> {
Idle {},
Claimed {},
Staged { content_ref: &'a ContentRef },
}
match self {
Self::Idle => Shape::Idle {}.serialize(serializer),
Self::Claimed => Shape::Claimed {}.serialize(serializer),
Self::Staged(content_ref) => Shape::Staged { content_ref }.serialize(serializer),
}
}
}
impl<'de> Deserialize<'de> for ProxiedStaging {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
StrictProxiedStaging::deserialize(deserializer).map(Into::into)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum UploadSessionMode {
ServiceProxied {
staging: ProxiedStaging,
},
DirectPut {
checksum_algorithm: ChecksumAlgorithm,
},
DirectMultipart {
provider_upload_id: String,
part_size_bytes: NonZeroU64,
checksum_algorithm: ChecksumAlgorithm,
},
}
impl UploadSessionMode {
pub fn checksum_algorithm(&self) -> Option<ChecksumAlgorithm> {
match self {
Self::ServiceProxied { .. } => None,
Self::DirectPut { checksum_algorithm }
| Self::DirectMultipart {
checksum_algorithm, ..
} => Some(*checksum_algorithm),
}
}
fn content_ref(&self) -> Option<&ContentRef> {
match self {
Self::ServiceProxied {
staging: ProxiedStaging::Staged(content_ref),
} => Some(content_ref),
Self::ServiceProxied { .. } | Self::DirectPut { .. } | Self::DirectMultipart { .. } => {
None
}
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum UploadSessionRecordStatus {
Open {
expires_at_ms: u64,
},
Completed {
completed_at_ms: u64,
content_ref: ContentRef,
},
Aborted {
aborted_at_ms: u64,
},
}
impl UploadSessionRecordStatus {
fn content_ref(&self) -> Option<&ContentRef> {
match self {
Self::Open { .. } => None,
Self::Completed { content_ref, .. } => Some(content_ref),
Self::Aborted { .. } => None,
}
}
}
impl std::fmt::Display for UploadSessionRecordStatus {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let status = match self {
Self::Open { .. } => "open",
Self::Completed { .. } => "completed",
Self::Aborted { .. } => "aborted",
};
formatter.write_str(status)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct UploadSessionState {
pub namespace_id: NamespaceId,
pub upload_id: UploadId,
pub content_id: ContentId,
pub created_at_ms: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub subject_id: Option<SubjectId>,
pub mode: UploadSessionMode,
pub status: UploadSessionRecordStatus,
}
impl UploadSessionState {
fn validate(&self) -> Result<(), String> {
if !matches!(self.status, UploadSessionRecordStatus::Open { .. })
&& self.mode.content_ref().is_some()
{
return Err(format!(
"upload session `{}` is {} but still holds a staged content reference",
self.upload_id, self.status
));
}
for content_ref in self
.mode
.content_ref()
.into_iter()
.chain(self.status.content_ref())
{
content_ref.validate().map_err(|error| {
format!(
"upload session `{}` holds an invalid content ref: {error}",
self.upload_id
)
})?;
if content_ref.content_id != self.content_id {
return Err(format!(
"upload session `{}` owns content `{}` but holds a reference to `{}`",
self.upload_id, self.content_id, content_ref.content_id
));
}
}
if let (
Some(checksum_algorithm),
UploadSessionRecordStatus::Completed { content_ref, .. },
) = (self.mode.checksum_algorithm(), &self.status)
{
if content_ref.checksum.algorithm != checksum_algorithm {
return Err(format!(
"upload session `{}` requires `{checksum_algorithm}` but its completed \
content uses `{}`",
self.upload_id, content_ref.checksum.algorithm
));
}
}
Ok(())
}
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct StrictUploadSessionState {
namespace_id: NamespaceId,
upload_id: UploadId,
content_id: ContentId,
created_at_ms: u64,
#[serde(default)]
subject_id: Option<SubjectId>,
mode: StrictUploadSessionMode,
status: StrictUploadSessionRecordStatus,
}
#[derive(Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
enum StrictUploadSessionMode {
ServiceProxied {
staging: StrictProxiedStaging,
},
DirectPut {
checksum_algorithm: ChecksumAlgorithm,
},
DirectMultipart {
provider_upload_id: String,
part_size_bytes: NonZeroU64,
checksum_algorithm: ChecksumAlgorithm,
},
}
impl From<StrictUploadSessionMode> for UploadSessionMode {
fn from(mode: StrictUploadSessionMode) -> Self {
match mode {
StrictUploadSessionMode::ServiceProxied { staging } => Self::ServiceProxied {
staging: staging.into(),
},
StrictUploadSessionMode::DirectPut { checksum_algorithm } => {
Self::DirectPut { checksum_algorithm }
}
StrictUploadSessionMode::DirectMultipart {
provider_upload_id,
part_size_bytes,
checksum_algorithm,
} => Self::DirectMultipart {
provider_upload_id,
part_size_bytes,
checksum_algorithm,
},
}
}
}
#[derive(Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
enum StrictProxiedStaging {
Idle {},
Claimed {},
Staged { content_ref: ContentRef },
}
impl From<StrictProxiedStaging> for ProxiedStaging {
fn from(staging: StrictProxiedStaging) -> Self {
match staging {
StrictProxiedStaging::Idle {} => Self::Idle,
StrictProxiedStaging::Claimed {} => Self::Claimed,
StrictProxiedStaging::Staged { content_ref } => Self::Staged(content_ref),
}
}
}
#[derive(Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
enum StrictUploadSessionRecordStatus {
Open {
expires_at_ms: u64,
},
Completed {
completed_at_ms: u64,
content_ref: ContentRef,
},
Aborted {
aborted_at_ms: u64,
},
}
impl From<StrictUploadSessionRecordStatus> for UploadSessionRecordStatus {
fn from(status: StrictUploadSessionRecordStatus) -> Self {
match status {
StrictUploadSessionRecordStatus::Open { expires_at_ms } => Self::Open { expires_at_ms },
StrictUploadSessionRecordStatus::Completed {
completed_at_ms,
content_ref,
} => Self::Completed {
completed_at_ms,
content_ref,
},
StrictUploadSessionRecordStatus::Aborted { aborted_at_ms } => {
Self::Aborted { aborted_at_ms }
}
}
}
}
impl<'de> Deserialize<'de> for UploadSessionState {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let record = StrictUploadSessionState::deserialize(deserializer)?;
let session = Self {
namespace_id: record.namespace_id,
upload_id: record.upload_id,
content_id: record.content_id,
created_at_ms: record.created_at_ms,
subject_id: record.subject_id,
mode: record.mode.into(),
status: record.status.into(),
};
session.validate().map_err(serde::de::Error::custom)?;
Ok(session)
}
}
pub type ControlObjectEnvelope<T> = crate::envelope::VerifiedEnvelope<T>;
pub fn encode_control_state<T: Serialize>(
kind: ControlObjectKind,
state: &T,
) -> Result<Vec<u8>, EnvelopeCodecError> {
crate::envelope::encode_json_envelope(kind.as_str(), kind.format_version(), state)
.map(crate::envelope::EncodedEnvelope::into_bytes)
}
pub fn decode_control_object<T>(
bytes: &[u8],
expected_kind: ControlObjectKind,
) -> Result<ControlObjectEnvelope<T>, EnvelopeCodecError>
where
T: DeserializeOwned,
{
let decoded = crate::envelope::decode_json_envelope(
bytes,
expected_kind.format_version(),
|found| match ControlObjectKind::parse(found) {
None => Err(EnvelopeCodecError::UnknownKind {
found: found.to_owned(),
}),
Some(kind) if kind != expected_kind => Err(EnvelopeCodecError::KindMismatch {
expected: expected_kind.as_str().to_owned(),
found: found.to_owned(),
}),
Some(_) => Ok(()),
},
)?;
Ok(decoded)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{Checksum, ContentRefKind};
#[test]
fn completed_proxied_session_rejects_conflicting_staged_size() {
let content_ref = ContentRef {
kind: ContentRefKind::BlobV1,
owner_namespace_id: crate::NamespaceId::parse("demo").expect("namespace id"),
content_id: ContentId::parse("con_0123456789abcdef0123456789abcdef")
.expect("content id"),
size_bytes: 5,
checksum: Checksum::sha256(b"hello"),
};
let mut staged = content_ref.clone();
staged.size_bytes += 1;
let session = UploadSessionState {
namespace_id: NamespaceId::parse("demo").expect("namespace id"),
upload_id: UploadId::parse("upl_0123456789abcdef0123456789abcdef").expect("upload id"),
content_id: content_ref.content_id.clone(),
created_at_ms: 1_000,
subject_id: None,
mode: UploadSessionMode::ServiceProxied {
staging: ProxiedStaging::Staged(staged),
},
status: UploadSessionRecordStatus::Completed {
completed_at_ms: 2_000,
content_ref,
},
};
let error = session.validate().expect_err("conflicting staged size");
assert_eq!(
error,
format!(
"upload session `{}` is completed but still holds a staged content reference",
session.upload_id
)
);
}
#[test]
fn terminal_upload_modes_reject_only_retained_staged_references() {
let content_ref = ContentRef {
kind: ContentRefKind::BlobV1,
owner_namespace_id: crate::NamespaceId::parse("demo").expect("namespace id"),
content_id: ContentId::parse("con_0123456789abcdef0123456789abcdef")
.expect("content id"),
size_bytes: 5,
checksum: Checksum::sha256(b"hello"),
};
let modes = [
UploadSessionMode::ServiceProxied {
staging: ProxiedStaging::Idle,
},
UploadSessionMode::ServiceProxied {
staging: ProxiedStaging::Claimed,
},
UploadSessionMode::ServiceProxied {
staging: ProxiedStaging::Staged(content_ref.clone()),
},
UploadSessionMode::DirectPut {
checksum_algorithm: ChecksumAlgorithm::Sha256,
},
UploadSessionMode::DirectMultipart {
provider_upload_id: "provider-upload".to_owned(),
part_size_bytes: NonZeroU64::new(8 * 1024 * 1024).expect("part size"),
checksum_algorithm: ChecksumAlgorithm::Sha256,
},
];
for mode in modes {
for status in [
UploadSessionRecordStatus::Completed {
completed_at_ms: 2_000,
content_ref: content_ref.clone(),
},
UploadSessionRecordStatus::Aborted {
aborted_at_ms: 2_000,
},
] {
let session = UploadSessionState {
namespace_id: NamespaceId::parse("demo").expect("namespace id"),
upload_id: UploadId::parse("upl_0123456789abcdef0123456789abcdef")
.expect("upload id"),
content_id: content_ref.content_id.clone(),
created_at_ms: 1_000,
subject_id: None,
mode: mode.clone(),
status,
};
let encoded = serde_json::to_value(&session).expect("encode session");
let decoded = serde_json::from_value::<UploadSessionState>(encoded);
if mode.content_ref().is_some() {
let error = decoded
.expect_err("terminal staging is corrupt")
.to_string();
assert!(error.contains(session.upload_id.as_str()));
assert!(error.contains("still holds a staged content reference"));
} else {
assert_eq!(decoded.expect("valid terminal session"), session);
}
}
}
}
#[test]
fn control_object_kind_strings_round_trip_and_match_serde() {
for kind in ControlObjectKind::ALL {
assert_eq!(ControlObjectKind::parse(kind.as_str()), Some(kind));
let serialized = serde_json::to_value(kind).expect("serialize kind");
assert_eq!(serialized, serde_json::Value::from(kind.as_str()));
}
assert_eq!(ControlObjectKind::parse("not_a_kind"), None);
}
}