#![doc = include_str!("../README.md")]
use std::{fmt, sync::Arc};
use async_trait::async_trait;
use bytes::Bytes;
use futures::stream::BoxStream;
use thiserror::Error;
pub mod conformance;
#[cfg(feature = "grpc")]
pub mod grpc;
#[cfg(feature = "local")]
mod local;
mod memory;
#[cfg(any(feature = "grpc", feature = "local"))]
mod wire_codec;
#[allow(missing_docs, clippy::pedantic, clippy::too_many_lines)]
pub mod wire {
include!(concat!(env!("OUT_DIR"), "/acyclic.stream.v2.rs"));
}
pub const FILE_DESCRIPTOR_SET: &[u8] = include_bytes!("../proto/stream/v2/stream_descriptor.bin");
#[cfg(feature = "local")]
pub use local::{LocalDurability, LocalStream, LocalStreamError, LocalStreamLimits};
pub use memory::{MemoryLimits, MemoryStream};
pub const MAX_RECORD_BYTES: usize = 64 * 1024;
pub const MAX_ITEMS: usize = 1_024;
pub const MAX_COMMAND_BYTES: usize = 1024 * 1024 + 8 * 1024;
pub const MIN_IDEMPOTENCY_RETENTION_SECS: u64 = 24 * 60 * 60;
pub const MAX_IDEMPOTENCY_KEY_BYTES: usize = 256;
pub const MAX_PATH_BYTES: usize = u16::MAX as usize;
#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct StreamPath(Arc<str>);
impl StreamPath {
pub fn new(path: impl AsRef<str>) -> Result<Self, StreamError> {
let path = path.as_ref();
if path.is_empty()
|| path.len() > MAX_PATH_BYTES
|| !path.is_ascii()
|| path.starts_with('/')
|| path.ends_with('/')
{
return Err(StreamError::InvalidPath);
}
let mut count = 0_usize;
for segment in path.split('/') {
count = count.checked_add(1).ok_or(StreamError::LimitExceeded)?;
if segment.is_empty()
|| segment == "."
|| segment == ".."
|| segment
.bytes()
.any(|byte| byte.is_ascii_control() || byte.is_ascii_whitespace())
|| segment.as_bytes().contains(&b'\\')
{
return Err(StreamError::InvalidPath);
}
}
if count > MAX_ITEMS {
return Err(StreamError::LimitExceeded);
}
Ok(Self(Arc::from(path)))
}
#[must_use]
pub fn as_str(&self) -> &str {
&self.0
}
#[must_use]
#[allow(
clippy::string_slice,
reason = "`new` rejects any path that is not `is_ascii()`, so every byte offset from \
`rfind` is a valid char boundary and this can never panic"
)]
pub fn parent(&self) -> Option<Self> {
self.0
.rfind('/')
.map(|index| Self(Arc::from(&self.0[..index])))
}
}
impl fmt::Display for StreamPath {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.as_str())
}
}
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct CommitId([u8; 32]);
impl CommitId {
#[must_use]
pub const fn from_bytes(bytes: [u8; 32]) -> Self {
Self(bytes)
}
#[must_use]
pub const fn as_bytes(&self) -> &[u8; 32] {
&self.0
}
}
#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct IdempotencyKey(Bytes);
impl IdempotencyKey {
pub fn new(value: impl Into<Bytes>) -> Result<Self, StreamError> {
let value = value.into();
if value.is_empty() || value.len() > MAX_IDEMPOTENCY_KEY_BYTES {
return Err(StreamError::InvalidArgument);
}
Ok(Self(value))
}
#[must_use]
pub fn as_bytes(&self) -> &[u8] {
&self.0
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Record {
pub sequence: u64,
pub value: Bytes,
pub commit_id: CommitId,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct AppendReceipt {
pub start: u64,
pub end: u64,
pub tail: u64,
pub commit_id: CommitId,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum AppendOutcome {
Committed(AppendReceipt),
TailConflict {
actual_tail: u64,
},
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct AppendRequest {
pub path: StreamPath,
pub records: Vec<Bytes>,
pub if_tail: Option<u64>,
pub idempotency_key: Option<IdempotencyKey>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ForkReceipt {
pub source: StreamPath,
pub destination: StreamPath,
pub forked_at: u64,
pub tail: u64,
pub commit_id: CommitId,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ForkRequest {
pub source: StreamPath,
pub destination: StreamPath,
pub at_tail: Option<u64>,
pub idempotency_key: Option<IdempotencyKey>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct TrimReceipt {
pub path: StreamPath,
pub trim_point: u64,
pub commit_id: CommitId,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct DeleteReceipt {
pub path: StreamPath,
pub commit_id: CommitId,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ReadRequest {
pub path: StreamPath,
pub from: u64,
pub limit: u32,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Child {
pub path: StreamPath,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ChildrenRequest {
pub parent: Option<StreamPath>,
pub limit: u32,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct CommittedAppend {
pub path: StreamPath,
pub start: u64,
pub end: u64,
pub tail: u64,
pub records: Vec<Record>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct CommittedFork {
pub source: StreamPath,
pub destination: StreamPath,
pub forked_at: u64,
pub tail: u64,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct CommittedTrim {
pub path: StreamPath,
pub trim_point: u64,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct CommittedDelete {
pub path: StreamPath,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum CommittedMutation {
Append(CommittedAppend),
Fork(CommittedFork),
Trim(CommittedTrim),
Delete(CommittedDelete),
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct CommittedEnvelope {
pub commit_id: CommitId,
pub mutations: Vec<CommittedMutation>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum CommitCondition {
Tail {
path: StreamPath,
expected: u64,
},
Absent {
path: StreamPath,
},
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum CommitMutation {
Append {
path: StreamPath,
records: Vec<Bytes>,
},
Fork {
source: StreamPath,
destination: StreamPath,
at_tail: u64,
},
Trim {
path: StreamPath,
before: u64,
},
Delete {
path: StreamPath,
},
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct CommitRequest {
pub conditions: Vec<CommitCondition>,
pub mutations: Vec<CommitMutation>,
pub idempotency_key: IdempotencyKey,
}
pub trait UnixMillisClock: Send + Sync + 'static {
fn now_unix_millis(&self) -> u64;
}
#[derive(Clone, Copy, Debug, Default)]
pub struct SystemUnixMillisClock;
impl UnixMillisClock for SystemUnixMillisClock {
fn now_unix_millis(&self) -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |duration| {
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
})
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum CommitConflict {
Tail {
path: StreamPath,
expected: u64,
actual: Option<u64>,
},
Exists {
path: StreamPath,
},
Retired {
path: StreamPath,
},
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum CommitOutcome {
Committed(CommittedEnvelope),
Conflict(Vec<CommitConflict>),
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum IdempotencyOutcome {
Append(AppendOutcome),
Fork(ForkReceipt),
Trim(TrimReceipt),
Delete(DeleteReceipt),
Commit(CommitOutcome),
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct IdempotencyObservation {
pub idempotency_key: IdempotencyKey,
pub request_digest: [u8; 32],
pub outcome: IdempotencyOutcome,
}
pub type RecordStream = BoxStream<'static, Result<Record, StreamError>>;
pub type ChildStream = BoxStream<'static, Result<Child, StreamError>>;
#[async_trait]
pub trait StreamProvider: Send + Sync + 'static {
async fn inspect_idempotency(
&self,
idempotency_key: IdempotencyKey,
) -> Result<Option<IdempotencyObservation>, StreamError>;
async fn tail(&self, path: StreamPath) -> Result<u64, StreamError>;
async fn append(&self, request: AppendRequest) -> Result<AppendOutcome, StreamError>;
async fn fork(&self, request: ForkRequest) -> Result<ForkReceipt, StreamError>;
async fn trim(
&self,
path: StreamPath,
before: u64,
idempotency_key: IdempotencyKey,
) -> Result<TrimReceipt, StreamError>;
async fn delete(
&self,
path: StreamPath,
idempotency_key: IdempotencyKey,
) -> Result<DeleteReceipt, StreamError>;
async fn read(&self, request: ReadRequest) -> Result<RecordStream, StreamError>;
async fn follow(&self, path: StreamPath, from: u64) -> Result<RecordStream, StreamError>;
async fn children(&self, request: ChildrenRequest) -> Result<ChildStream, StreamError>;
async fn commit(&self, request: CommitRequest) -> Result<CommitOutcome, StreamError>;
async fn commit_before(
&self,
request: CommitRequest,
deadline_unix_millis: u64,
) -> Result<CommitOutcome, StreamError> {
let _ = (request, deadline_unix_millis);
Err(StreamError::Unsupported)
}
async fn read_commit(&self, commit_id: CommitId) -> Result<CommittedEnvelope, StreamError>;
}
pub struct StreamClient<P> {
provider: Arc<P>,
}
impl<P> Clone for StreamClient<P> {
fn clone(&self) -> Self {
Self {
provider: Arc::clone(&self.provider),
}
}
}
impl<P: StreamProvider> StreamClient<P> {
#[must_use]
pub fn new(provider: Arc<P>) -> Self {
Self { provider }
}
pub fn stream(&self, path: impl AsRef<str>) -> Result<Stream<P>, StreamError> {
Ok(Stream {
client: Self::new(Arc::clone(&self.provider)),
path: StreamPath::new(path)?,
})
}
pub async fn inspect_idempotency(
&self,
idempotency_key: IdempotencyKey,
) -> Result<Option<IdempotencyObservation>, StreamError> {
self.provider.inspect_idempotency(idempotency_key).await
}
pub async fn children(
&self,
parent: Option<&str>,
limit: u32,
) -> Result<ChildStream, StreamError> {
self.provider
.children(ChildrenRequest {
parent: parent.map(StreamPath::new).transpose()?,
limit,
})
.await
}
pub async fn commit(&self, request: CommitRequest) -> Result<CommitOutcome, StreamError> {
self.provider.commit(request).await
}
pub async fn commit_before(
&self,
request: CommitRequest,
deadline_unix_millis: u64,
) -> Result<CommitOutcome, StreamError> {
self.provider
.commit_before(request, deadline_unix_millis)
.await
}
pub async fn read_commit(&self, commit_id: CommitId) -> Result<CommittedEnvelope, StreamError> {
self.provider.read_commit(commit_id).await
}
}
#[cfg(feature = "grpc")]
impl StreamClient<grpc::Client> {
pub async fn connect(
endpoint: impl AsRef<str>,
bearer_token: impl AsRef<str>,
) -> Result<Self, grpc::ConnectError> {
Ok(Self::new(Arc::new(
grpc::Client::connect(endpoint, bearer_token).await?,
)))
}
pub async fn connect_with_ca_certificate(
endpoint: impl AsRef<str>,
bearer_token: impl AsRef<str>,
certificate_pem: impl AsRef<[u8]>,
) -> Result<Self, grpc::ConnectError> {
Ok(Self::new(Arc::new(
grpc::Client::connect_with_ca_certificate(endpoint, bearer_token, certificate_pem)
.await?,
)))
}
}
pub struct Stream<P> {
client: StreamClient<P>,
path: StreamPath,
}
impl<P> Clone for Stream<P> {
fn clone(&self) -> Self {
Self {
client: self.client.clone(),
path: self.path.clone(),
}
}
}
impl<P: StreamProvider> Stream<P> {
#[must_use]
pub const fn path(&self) -> &StreamPath {
&self.path
}
pub async fn tail(&self) -> Result<u64, StreamError> {
self.client.provider.tail(self.path.clone()).await
}
pub async fn append(&self, value: impl Into<Bytes>) -> Result<AppendOutcome, StreamError> {
self.append_batch(vec![value.into()], None, None).await
}
pub async fn append_at(
&self,
value: impl Into<Bytes>,
if_tail: u64,
) -> Result<AppendOutcome, StreamError> {
self.append_batch(vec![value.into()], Some(if_tail), None)
.await
}
pub async fn append_batch(
&self,
records: Vec<Bytes>,
if_tail: Option<u64>,
idempotency_key: Option<IdempotencyKey>,
) -> Result<AppendOutcome, StreamError> {
self.client
.provider
.append(AppendRequest {
path: self.path.clone(),
records,
if_tail,
idempotency_key: Some(idempotency_key.unwrap_or_else(new_idempotency_key)),
})
.await
}
pub async fn fork(
&self,
destination: impl AsRef<str>,
at_tail: Option<u64>,
idempotency_key: Option<IdempotencyKey>,
) -> Result<ForkReceipt, StreamError> {
self.client
.provider
.fork(ForkRequest {
source: self.path.clone(),
destination: StreamPath::new(destination)?,
at_tail,
idempotency_key: Some(idempotency_key.unwrap_or_else(new_idempotency_key)),
})
.await
}
pub async fn trim(
&self,
before: u64,
idempotency_key: Option<IdempotencyKey>,
) -> Result<TrimReceipt, StreamError> {
self.client
.provider
.trim(
self.path.clone(),
before,
idempotency_key.unwrap_or_else(new_idempotency_key),
)
.await
}
pub async fn delete(
&self,
idempotency_key: Option<IdempotencyKey>,
) -> Result<DeleteReceipt, StreamError> {
self.client
.provider
.delete(
self.path.clone(),
idempotency_key.unwrap_or_else(new_idempotency_key),
)
.await
}
pub async fn read(&self, from: u64, limit: u32) -> Result<RecordStream, StreamError> {
self.client
.provider
.read(ReadRequest {
path: self.path.clone(),
from,
limit,
})
.await
}
pub async fn follow(&self, from: u64) -> Result<RecordStream, StreamError> {
self.client.provider.follow(self.path.clone(), from).await
}
}
fn new_idempotency_key() -> IdempotencyKey {
IdempotencyKey(Bytes::copy_from_slice(uuid::Uuid::new_v4().as_bytes()))
}
#[derive(Clone, Debug, Eq, Error, PartialEq)]
pub enum StreamError {
#[error("invalid stream path")]
InvalidPath,
#[error("invalid stream argument")]
InvalidArgument,
#[error("stream limit exceeded")]
LimitExceeded,
#[error("stream not found")]
NotFound,
#[error("stream already exists")]
AlreadyExists,
#[error("stream path retired")]
Retired,
#[error("stream prefix not retained")]
PrefixNotRetained,
#[error("stream sequence out of range")]
OutOfRange,
#[error("idempotency mismatch")]
IdempotencyMismatch,
#[error("stream capacity exhausted")]
Capacity,
#[error("stream access denied")]
AccessDenied,
#[error("stream unavailable")]
Unavailable,
#[error("stream commit deadline elapsed")]
DeadlineElapsed,
#[error("stream capability unsupported")]
Unsupported,
}