use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::sync::Arc;
use std::time::Duration;
use rostra_client_db::{
CurrentState, Database, EventContentState, EventRecord, IdsFollowersRecord,
};
use rostra_core::ShortEventId;
use rostra_core::event::{EventContentRaw, EventExt as _, SignedEvent, VerifiedEventContent};
use rostra_core::id::{RostraId, ToShort as _};
use rostra_util_error::{FmtCompact, WhateverResult};
use snafu::ResultExt as _;
use tokio::sync::broadcast;
use tokio::time::Instant;
use tracing::{debug, instrument, trace, warn};
type FollowersMap = Arc<HashMap<RostraId, IdsFollowersRecord>>;
use crate::client::Client;
use crate::task::head_selection::sorted_heads;
use crate::task::outbound_deadline::{PEER_OPERATION_DEADLINE, within};
const LOG_TARGET: &str = "rostra::head_broadcaster";
const BROADCAST_RETRY_INITIAL_DELAY: Duration = Duration::from_secs(1);
const BROADCAST_RETRY_MAX_DELAY: Duration = Duration::from_secs(60);
#[derive(Debug, Clone, Copy)]
struct BroadcastPolicy {
peer_deadline: Duration,
retry_initial_delay: Duration,
retry_max_delay: Duration,
}
const BROADCAST_POLICY: BroadcastPolicy = BroadcastPolicy {
peer_deadline: PEER_OPERATION_DEADLINE,
retry_initial_delay: BROADCAST_RETRY_INITIAL_DELAY,
retry_max_delay: BROADCAST_RETRY_MAX_DELAY,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum BroadcastHeadOutcome {
Delivered,
Retry,
Stop,
}
#[derive(Debug, Clone, Copy)]
struct BroadcastHeadReport {
outcome: BroadcastHeadOutcome,
delivered: usize,
errors: usize,
timeouts: usize,
}
pub struct HeadUpdateBroadcaster {
client: crate::client::ClientHandle,
networking: Arc<crate::net::ClientNetworking>,
db: Arc<Database>,
self_id: RostraId,
self_followers: CurrentState<FollowersMap>,
new_heads_rx: broadcast::Receiver<(RostraId, ShortEventId)>,
new_content_rx: broadcast::Receiver<VerifiedEventContent>,
}
impl HeadUpdateBroadcaster {
pub fn new(client: &Client) -> Self {
debug!(target: LOG_TARGET, "Starting followee head broadcasting task" );
Self {
client: client.handle(),
networking: client.networking().clone(),
db: client.db().to_owned(),
self_id: client.rostra_id(),
self_followers: client.self_followers_subscribe(),
new_heads_rx: client.db().new_heads_subscribe(),
new_content_rx: client.db().new_content_subscribe(),
}
}
#[instrument(name = "head-update-broadcaster", skip(self), fields(self_id = %self.self_id.fmt_short()), ret)]
pub async fn run(self) {
self.run_with_policy(BROADCAST_POLICY).await;
}
async fn run_with_policy(mut self, policy: BroadcastPolicy) {
let mut self_followers = self.self_followers.clone();
let mut pending_heads = BTreeSet::new();
let mut retry_at = BTreeMap::new();
reconcile_pending_heads(&self.db, &mut pending_heads, &mut retry_at).await;
loop {
let followers = self_followers.snapshot();
if let Some((head, event, event_content)) =
take_one_ready_head(&self.db, &mut pending_heads, &retry_at).await
{
let attempt = retry_at
.get(&head)
.map_or(1, |retry| retry.retry_count.saturating_add(1));
let started_at = Instant::now();
let report = self
.broadcast_head(
head,
&event,
&event_content,
&followers,
policy.peer_deadline,
)
.await;
let elapsed = started_at.elapsed();
match report.outcome {
BroadcastHeadOutcome::Delivered => {
pending_heads.remove(&head);
retry_at.remove(&head);
log_broadcast_pass(
head,
followers.len() + 1,
attempt,
report,
elapsed,
None,
);
}
BroadcastHeadOutcome::Retry => {
let retry_count = retry_at
.get(&head)
.map_or(0, |retry_at| retry_at.retry_count);
let retry_delay = broadcast_retry_delay(retry_count, policy);
retry_at.insert(
head,
BroadcastRetry {
retry_count: retry_count.saturating_add(1),
at: Instant::now() + retry_delay,
},
);
log_broadcast_pass(
head,
followers.len() + 1,
attempt,
report,
elapsed,
Some(retry_delay),
);
}
BroadcastHeadOutcome::Stop => {
log_broadcast_pass(
head,
followers.len() + 1,
attempt,
report,
elapsed,
None,
);
return;
}
}
continue;
}
loop {
let next_retry = retry_at
.iter()
.filter(|(head, _)| pending_heads.contains(head))
.map(|(_, retry_at)| retry_at.at)
.min()
.unwrap_or_else(|| Instant::now() + Duration::from_secs(24 * 60 * 60));
let should_retry = tokio::select! {
res = self.new_heads_rx.recv() => {
match res {
Ok((author, head)) if author == self.self_id => {
trace!(target: LOG_TARGET, event_id = %head.to_short(), "Received exact self-head signal");
reconcile_pending_heads(&self.db, &mut pending_heads, &mut retry_at).await;
true
}
Ok(_) => false,
Err(broadcast::error::RecvError::Closed) => return,
Err(broadcast::error::RecvError::Lagged(skipped)) => {
warn!(
target: LOG_TARGET,
skipped,
"Head broadcast receiver lagged; recovering durable heads"
);
reconcile_pending_heads(&self.db, &mut pending_heads, &mut retry_at).await;
true
}
}
}
res = self.new_content_rx.recv() => {
match res {
Ok(content)
if content_completes_pending(
&content,
self.self_id,
&pending_heads,
) =>
{
true
}
Ok(_) => false,
Err(broadcast::error::RecvError::Closed) => return,
Err(broadcast::error::RecvError::Lagged(skipped)) => {
warn!(
target: LOG_TARGET,
skipped,
"Content broadcast receiver lagged; reconciling durable heads"
);
reconcile_pending_heads(&self.db, &mut pending_heads, &mut retry_at).await;
true
}
}
}
res = self_followers.changed() => {
if res.is_err() {
return;
}
reconcile_pending_heads(&self.db, &mut pending_heads, &mut retry_at).await;
true
}
_ = tokio::time::sleep_until(next_retry) => true,
};
if should_retry {
break;
}
}
trace!(target: LOG_TARGET, "Woke up");
}
}
async fn broadcast_head(
&self,
head: ShortEventId,
event: &EventRecord,
event_content: &EventContentRaw,
followers: &FollowersMap,
deadline: Duration,
) -> BroadcastHeadReport {
debug!(
target: LOG_TARGET,
event_id = %head.to_short(),
followers_num = followers.len(),
"Broadcasting new head event to followers"
);
let mut delivered = 0;
let mut errors = 0;
let mut timeouts = 0;
for id in [self.self_id].into_iter().chain(followers.keys().copied()) {
if self.client.app_ref_opt().is_none() {
debug!(target: LOG_TARGET, "Client gone, quitting");
return BroadcastHeadReport {
outcome: BroadcastHeadOutcome::Stop,
delivered,
errors,
timeouts,
};
}
match within(
deadline,
self.broadcast_event(id, &event.signed, event_content),
)
.await
{
Ok(Ok(())) => delivered += 1,
Ok(Err(err)) => {
errors += 1;
debug!(
target: LOG_TARGET,
err = %err.fmt_compact(),
id = %id.to_short(),
"Failed to broadcast new head to node"
);
}
Err(_) => {
timeouts += 1;
warn!(
target: LOG_TARGET,
id = %id.to_short(),
timeout_ms = deadline.as_millis(),
"Timed out broadcasting new head to node"
);
}
}
}
let outcome = if errors != 0 || timeouts != 0 {
BroadcastHeadOutcome::Retry
} else {
BroadcastHeadOutcome::Delivered
};
BroadcastHeadReport {
outcome,
delivered,
errors,
timeouts,
}
}
async fn broadcast_event(
&self,
id: RostraId,
signed_event: &SignedEvent,
event_content: &EventContentRaw,
) -> WhateverResult<()> {
let conn = self
.networking
.connect_cached(id)
.await
.whatever_context("Couldn't connect")?;
conn.feed_event(*signed_event, event_content.clone())
.await
.whatever_context("Failed broadcasting head event")?;
Ok(())
}
}
fn log_broadcast_pass(
head: ShortEventId,
recipients: usize,
attempt: u32,
report: BroadcastHeadReport,
elapsed: Duration,
retry_delay: Option<Duration>,
) {
let outcome = match report.outcome {
BroadcastHeadOutcome::Delivered => "delivered",
BroadcastHeadOutcome::Retry => "retry",
BroadcastHeadOutcome::Stop => "stop",
};
debug!(
target: LOG_TARGET,
parent: None,
head = %head.to_short(),
recipients,
delivered = report.delivered,
errors = report.errors,
timeouts = report.timeouts,
attempt,
elapsed_ms = elapsed.as_millis(),
outcome,
retry_delay_ms = retry_delay.map_or(0, |delay| delay.as_millis()),
"Head broadcast pass completed"
);
}
async fn reconcile_current_heads(db: &Database, pending_heads: &mut BTreeSet<ShortEventId>) {
*pending_heads = sorted_heads(&db.get_heads_self().await)
.into_iter()
.collect();
}
async fn reconcile_pending_heads(
db: &Database,
pending_heads: &mut BTreeSet<ShortEventId>,
retry_at: &mut BTreeMap<ShortEventId, BroadcastRetry>,
) {
reconcile_current_heads(db, pending_heads).await;
retry_at.retain(|head, _| pending_heads.contains(head));
}
#[derive(Debug, Clone, Copy)]
struct BroadcastRetry {
retry_count: u32,
at: Instant,
}
fn broadcast_retry_delay(retry_count: u32, policy: BroadcastPolicy) -> Duration {
policy
.retry_initial_delay
.checked_mul(1u32 << retry_count.min(31))
.unwrap_or(Duration::MAX)
.min(policy.retry_max_delay)
}
fn content_completes_pending(
content: &VerifiedEventContent,
self_id: RostraId,
pending_heads: &BTreeSet<ShortEventId>,
) -> bool {
content.event.event.author == self_id && pending_heads.contains(&content.event_id().to_short())
}
async fn take_one_ready_head(
db: &Database,
pending_heads: &mut BTreeSet<ShortEventId>,
retry_at: &BTreeMap<ShortEventId, BroadcastRetry>,
) -> Option<(ShortEventId, EventRecord, EventContentRaw)> {
for head in pending_heads.iter().copied().collect::<Vec<_>>() {
if retry_at
.get(&head)
.is_some_and(|retry_at| Instant::now() < retry_at.at)
{
continue;
}
let Some(event) = db.get_event(head).await else {
warn!(target: LOG_TARGET, event_id = %head.to_short(), "No head event!?");
pending_heads.remove(&head);
continue;
};
let content = db.get_event_content(head).await;
if let Some(content) = content {
return Some((head, event, content));
}
if event.content_len() == 0 {
return Some((head, event, EventContentRaw::new(Vec::new())));
}
let content_state = db.get_event_content_state(head).await;
if content_is_terminal(content_state) {
debug!(target: LOG_TARGET, event_id = %head.to_short(), "Head content is terminally unavailable");
pending_heads.remove(&head);
} else {
trace!(target: LOG_TARGET, event_id = %head.to_short(), "Head content not ready");
}
continue;
}
None
}
fn content_is_terminal(content_state: Option<EventContentState>) -> bool {
matches!(
content_state,
Some(
EventContentState::Deleted { .. }
| EventContentState::Pruned
| EventContentState::Invalid
)
)
}
#[cfg(test)]
mod tests;