use std::sync::atomic::AtomicU32;
use std::sync::{Arc, Mutex, OnceLock};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use serde_json::Value;
use crate::causal::CausalContext;
use crate::channel::actor::{ChannelActorCore, PendingPublish, predicate_from};
use crate::channel::admission::{
ChannelPressureConfig, defer_after_append, defer_delay, watermark_reached,
};
use crate::channel::observer::ClusterObserver;
use crate::channel::schema::{Schema, SchemaId, SchemaValidationError};
use crate::channel::subscription::{InboxInstall, SubscriptionHandle, SubscriptionPredicate};
use crate::channel::supervisor::{ChannelSupervisor, shared_supervisor};
use crate::durability::bridge::block_on;
use crate::durability::{DurableChannel, DurableStore, MessageEnvelope, recover_durable_channel};
use crate::envelope::{Envelope, PublisherId};
use crate::error::LiminalError;
use crate::pressure::PressureSignal;
const RUNTIME_DURABLE_PARTITIONS: usize = 1;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ChannelDelivery {
delivered_count: usize,
durable_position: Option<u64>,
signal: PressureSignal,
pressure_config: ChannelPressureConfig,
}
impl ChannelDelivery {
#[must_use]
pub const fn delivered_count(&self) -> usize {
self.delivered_count
}
#[must_use]
pub const fn is_delivered(&self) -> bool {
self.delivered_count > 0
}
#[must_use]
pub const fn durable_position(&self) -> Option<u64> {
self.durable_position
}
#[must_use]
pub const fn pressure(&self) -> &PressureSignal {
&self.signal
}
#[must_use]
pub const fn is_admitted(&self) -> bool {
!matches!(self.signal, PressureSignal::Reject { .. })
}
#[must_use]
pub fn defer_delay(&self) -> Option<Duration> {
defer_delay(&self.signal, self.pressure_config)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ChannelMode {
Ephemeral,
Durable,
}
pub type SchemaRef = Schema;
#[derive(Clone, Debug)]
#[non_exhaustive]
pub struct ChannelConfig {
pub name: String,
pub schema: Schema,
pub mode: ChannelMode,
pub pressure: ChannelPressureConfig,
}
impl ChannelConfig {
#[must_use]
pub const fn new(name: String, schema: Schema, mode: ChannelMode) -> Self {
Self {
name,
schema,
mode,
pressure: ChannelPressureConfig::DEFAULT,
}
}
#[must_use]
pub const fn with_pressure(mut self, pressure: ChannelPressureConfig) -> Self {
self.pressure = pressure;
self
}
}
struct ChannelActorState {
supervisor: Result<ChannelSupervisor, String>,
core: OnceLock<Result<Arc<ChannelActorCore>, String>>,
restarts: AtomicU32,
}
impl ChannelActorState {
const fn new(supervisor: Result<ChannelSupervisor, String>) -> Self {
Self {
supervisor,
core: OnceLock::new(),
restarts: AtomicU32::new(0),
}
}
fn supervisor(&self) -> Result<&ChannelSupervisor, LiminalError> {
self.supervisor
.as_ref()
.map_err(|message| LiminalError::PublishFailed {
message: format!("channel supervisor unavailable: {message}"),
})
}
fn observer(&self) -> Option<Arc<dyn ClusterObserver>> {
self.supervisor
.as_ref()
.ok()
.and_then(|supervisor| supervisor.observer().cloned())
}
fn core(&self, schema: &Schema) -> Result<Arc<ChannelActorCore>, LiminalError> {
let supervisor = self.supervisor()?;
let stored = self.core.get_or_init(|| {
supervisor
.spawn_channel(schema.clone())
.map_err(|error| error.to_string())
});
let core = stored
.as_ref()
.map_err(|message| LiminalError::PublishFailed {
message: format!("channel actor unavailable: {message}"),
})?;
supervisor.ensure_running(core, &self.restarts)?;
Ok(Arc::clone(core))
}
}
impl std::fmt::Debug for ChannelActorState {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("ChannelActorState")
.field("supervisor", &self.supervisor)
.finish_non_exhaustive()
}
}
#[derive(Clone, Debug)]
pub struct ChannelHandle {
config: ChannelConfig,
actor: Arc<ChannelActorState>,
durable: Option<Arc<Mutex<DurableChannel>>>,
durable_store: Option<Arc<dyn DurableStore>>,
}
impl ChannelHandle {
#[must_use]
pub fn new(config: ChannelConfig) -> Self {
let supervisor = shared_supervisor().map_err(|error| error.to_string());
Self {
config,
actor: Arc::new(ChannelActorState::new(supervisor)),
durable: None,
durable_store: None,
}
}
#[must_use]
pub fn with_supervisor(config: ChannelConfig, supervisor: ChannelSupervisor) -> Self {
Self {
config,
actor: Arc::new(ChannelActorState::new(Ok(supervisor))),
durable: None,
durable_store: None,
}
}
pub fn new_durable(
config: ChannelConfig,
store: Arc<dyn DurableStore>,
) -> Result<Self, LiminalError> {
let durable = recover_durable(&config.name, Arc::clone(&store))?;
let supervisor = shared_supervisor()?;
Ok(Self {
config,
actor: Arc::new(ChannelActorState::new(Ok(supervisor))),
durable: Some(Arc::new(Mutex::new(durable))),
durable_store: Some(store),
})
}
pub fn new_durable_with_supervisor(
config: ChannelConfig,
store: Arc<dyn DurableStore>,
supervisor: ChannelSupervisor,
) -> Result<Self, LiminalError> {
let durable = recover_durable(&config.name, Arc::clone(&store))?;
Ok(Self {
config,
actor: Arc::new(ChannelActorState::new(Ok(supervisor))),
durable: Some(Arc::new(Mutex::new(durable))),
durable_store: Some(store),
})
}
#[must_use]
pub const fn config(&self) -> &ChannelConfig {
&self.config
}
pub fn publish<Payload>(&self, payload: Payload) -> Result<(), LiminalError>
where
Payload: AsRef<[u8]>,
{
self.publish_with_context(payload, PublisherId::default(), None)
}
pub fn publish_from<Payload>(
&self,
publisher_id: impl Into<PublisherId>,
payload: Payload,
) -> Result<(), LiminalError>
where
Payload: AsRef<[u8]>,
{
self.publish_with_context(payload, publisher_id.into(), None)
}
pub fn publish_with_context<Payload>(
&self,
payload: Payload,
publisher_id: PublisherId,
causal_context: Option<CausalContext>,
) -> Result<(), LiminalError>
where
Payload: AsRef<[u8]>,
{
self.publish_with_delivery(payload, publisher_id, causal_context)
.map(|_delivery| ())
}
pub fn publish_with_delivery<Payload>(
&self,
payload: Payload,
publisher_id: PublisherId,
causal_context: Option<CausalContext>,
) -> Result<ChannelDelivery, LiminalError>
where
Payload: AsRef<[u8]>,
{
let core = self.core()?;
core.pre_publish_gate(payload.as_ref())?;
if let Some(rejection) = self.watermark_rejection(&core)? {
return Ok(rejection);
}
let (durable_position, pending) = match self.durable.as_ref() {
Some(durable) => self.persist_and_enqueue(
&core,
durable,
payload.as_ref(),
publisher_id,
causal_context,
)?,
None => (
None,
core.enqueue_publish(
payload.as_ref().to_vec(),
publisher_id,
causal_context,
None,
)?,
),
};
let outcome = core.await_publish(pending)?;
if let Some(observer) = self.actor.observer() {
observer.on_publish(&self.config.name, &outcome.envelope);
}
let signal = if durable_position.is_some() && !outcome.every_match_dropped {
defer_after_append(outcome.signal)
} else {
outcome.signal
};
Ok(ChannelDelivery {
delivered_count: outcome.delivered_count,
durable_position,
signal,
pressure_config: self.config.pressure,
})
}
fn watermark_rejection(
&self,
core: &Arc<ChannelActorCore>,
) -> Result<Option<ChannelDelivery>, LiminalError> {
if self.durable.is_none() {
return Ok(None);
}
let (total_queued, total_bound) = core.live_buffer_occupancy()?;
if !watermark_reached(
total_queued,
total_bound,
self.config.pressure.durable_reject_watermark_percent,
) {
return Ok(None);
}
Ok(Some(ChannelDelivery {
delivered_count: 0,
durable_position: None,
signal: PressureSignal::reject(total_queued, total_bound, total_queued, total_bound),
pressure_config: self.config.pressure,
}))
}
fn durable_join_point(
durable: &Arc<Mutex<DurableChannel>>,
) -> Result<(String, u64), LiminalError> {
let channel = durable
.lock()
.map_err(|error| LiminalError::PublishFailed {
message: format!("durable channel state unavailable: {error}"),
})?;
let stream_key = channel.stream_key_for(0);
let head = channel.next_expected_sequence(0).unwrap_or(0);
drop(channel);
Ok((stream_key, head))
}
fn durable_store(&self) -> Result<Arc<dyn DurableStore>, LiminalError> {
self.durable_store
.as_ref()
.map(Arc::clone)
.ok_or_else(|| LiminalError::PublishFailed {
message: format!(
"channel '{}' has no durable store to replay from",
self.config.name
),
})
}
fn persist_and_enqueue(
&self,
core: &Arc<ChannelActorCore>,
durable: &Arc<Mutex<DurableChannel>>,
payload: &[u8],
publisher_id: PublisherId,
causal_context: Option<CausalContext>,
) -> Result<(Option<u64>, PendingPublish), LiminalError> {
let envelope = MessageEnvelope {
payload: payload.to_vec(),
causal_context: None,
timestamp: now_millis(),
publisher_id: publisher_id.as_str().to_owned(),
idempotency_key: None,
};
let mut channel = durable
.lock()
.map_err(|error| LiminalError::PublishFailed {
message: format!("durable channel state unavailable: {error}"),
})?;
let assigned_seq = block_on(channel.publish(&envelope))
.map_err(|error| LiminalError::PublishFailed {
message: format!(
"durable publish bridge for channel '{}' failed: {error}",
self.config.name
),
})?
.map_err(|error| LiminalError::PublishFailed {
message: format!(
"durable publish to channel '{}' failed: {error}",
self.config.name
),
})?;
let pending = core.enqueue_publish(
payload.to_vec(),
publisher_id,
causal_context,
Some(assigned_seq),
)?;
drop(channel);
Ok((Some(assigned_seq), pending))
}
pub fn current_schema_id(&self) -> Result<SchemaId, LiminalError> {
self.core()?.schema_id()
}
pub fn evolve_schema_add_field(
&self,
name: impl Into<String>,
field_schema: Value,
default: Value,
) -> Result<SchemaId, SchemaValidationError> {
let core = self
.core()
.map_err(|error| SchemaValidationError::InvalidSchema {
message: error.to_string(),
})?;
core.evolve(name.into(), field_schema, default)
}
pub fn subscribe(&self) -> Result<SubscriptionHandle, LiminalError> {
self.subscribe_inner(None, None)
}
pub fn subscribe_with_install(
&self,
install: InboxInstall,
) -> Result<SubscriptionHandle, LiminalError> {
self.subscribe_inner(None, Some(install))
}
pub fn subscribe_filtered<F>(&self, predicate: F) -> Result<SubscriptionHandle, LiminalError>
where
F: Fn(&Envelope) -> bool + Send + Sync + 'static,
{
self.subscribe_inner(Some(predicate_from(predicate)), None)
}
fn subscribe_inner(
&self,
predicate: Option<SubscriptionPredicate>,
install: Option<InboxInstall>,
) -> Result<SubscriptionHandle, LiminalError> {
let core = self.core()?;
let (handle, registration) =
SubscriptionHandle::spawn(core.scheduler(), predicate.clone(), install)?;
if let Some(durable) = self.durable.as_ref() {
let (stream_key, head) = Self::durable_join_point(durable)?;
handle.seed_replay_cursor(head);
handle.attach_durable_refill(
self.durable_store()?,
stream_key,
self.config.schema.id(),
predicate,
);
}
let pid = registration.pid();
core.subscribe(registration)?;
if let Some(observer) = self.actor.observer() {
observer.on_subscribe(&self.config.name, pid);
}
Ok(handle)
}
pub fn unsubscribe(&self, subscription: &SubscriptionHandle) -> Result<(), LiminalError> {
let pid = subscription.pid();
self.core()?.unsubscribe(pid)?;
if let Some(observer) = self.actor.observer() {
observer.on_unsubscribe(&self.config.name, pid);
}
Ok(())
}
pub fn flush(&self) -> Result<(), LiminalError> {
drop(self.core()?);
let Some(durable) = self.durable.as_ref() else {
return Ok(());
};
let flush_result = {
let channel = durable
.lock()
.map_err(|error| LiminalError::PublishFailed {
message: format!("durable channel state unavailable: {error}"),
})?;
block_on(channel.flush_store())
};
flush_result
.map_err(|error| LiminalError::PublishFailed {
message: format!(
"durable flush bridge for channel '{}' failed: {error}",
self.config.name
),
})?
.map_err(|error| LiminalError::PublishFailed {
message: format!(
"durable flush for channel '{}' failed: {error}",
self.config.name
),
})?;
Ok(())
}
pub fn subscriber_count(&self) -> Result<usize, LiminalError> {
Ok(self.core()?.list_subscribers()?.len())
}
pub fn close(&self) -> Result<(), LiminalError> {
self.core()?.close()
}
#[must_use]
pub fn is_actor_spawned(&self) -> bool {
self.actor.core.get().is_some_and(Result::is_ok)
}
fn core(&self) -> Result<Arc<ChannelActorCore>, LiminalError> {
self.actor.core(&self.config.schema)
}
#[cfg(test)]
pub(crate) fn actor_pid(&self) -> Result<u64, LiminalError> {
let core = self.core()?;
core.current_pid()?
.ok_or_else(|| LiminalError::DeliveryFailed {
message: "channel actor has no live pid".to_owned(),
})
}
#[cfg(test)]
pub(crate) fn scheduler(&self) -> Result<Arc<beamr::scheduler::Scheduler>, LiminalError> {
Ok(Arc::clone(self.core()?.scheduler()))
}
}
fn recover_durable(
channel_name: &str,
store: Arc<dyn DurableStore>,
) -> Result<DurableChannel, LiminalError> {
block_on(recover_durable_channel(
channel_name.to_owned(),
RUNTIME_DURABLE_PARTITIONS,
store,
))
.map_err(|error| LiminalError::PublishFailed {
message: format!("durable recovery bridge for channel '{channel_name}' failed: {error}"),
})?
.map_err(|error| LiminalError::PublishFailed {
message: format!("failed to recover durable channel '{channel_name}': {error}"),
})
}
fn now_millis() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |duration| {
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
})
}