use std::any::type_name;
use std::borrow::Cow;
use std::fmt;
use std::future::{Future, ready};
use std::num::NonZeroU32;
use std::sync::Arc;
use thiserror::Error;
use tokio_util::task::TaskTracker;
use tracing::info;
use crate::runtime::dispatch::Delivery;
use crate::runtime::lifecycle::{BoxError, BoxFuture};
use crate::runtime::metadata::{HandlerMetadata, PublishDescription};
#[cfg(feature = "poll-diagnostics")]
use crate::runtime::poll_diagnostics::PollDiagnostics;
use crate::runtime::publish::{
ForReply, OutPipeline, PublishContext, PublishTransform, PublishTransformIdentity,
};
use crate::{
AddressedCopies, Broker, BrokerMoves, Connected, ConnectedBroker, DeclareRetryError,
DefaultPublish, NamedCopies, OutgoingMessage, PairError, PayloadForm, PublishPolicy, Publisher,
RedeliveryAddress, RedeliveryAddressed, RetryDeclaration, SubscriptionSource,
};
#[cfg(feature = "testing")]
use crate::testing::coordinator::{Origin, TestHooks, paired, publishing_to};
type DefaultPolicy<B> = <Connected<B> as DefaultPublish>::Policy;
type DefaultLive<B> = <DefaultPolicy<B> as PublishPolicy<Connected<B>>>::Live;
pub(crate) struct DeferredRetry<Cx> {
pub(crate) publisher: Arc<dyn ErasedRetryPublisher<Cx>>,
pub(crate) destination: Option<Arc<str>>,
}
pub(crate) trait ErasedRetryPublisher<Cx>: Send + Sync {
fn publish_copy<'a>(
&'a self,
msg: OutgoingMessage<'a>,
cx: &'a PublishContext<'a, Cx>,
) -> BoxFuture<'a, Result<(), BoxError>>;
}
struct RetryLeaf<Live, Stack, Pipeline> {
live: Live,
stack: Stack,
pipeline: Pipeline,
#[cfg(feature = "testing")]
origin: Origin,
}
impl<Cx, Live, Stack, Pipeline> ErasedRetryPublisher<Cx> for RetryLeaf<Live, Stack, Pipeline>
where
Cx: Send + Sync + 'static,
Live: Publisher + Send + Sync + 'static,
Stack: PublishTransform<ForReply<Cx>, Live::Options> + Send + Sync + 'static,
Pipeline: OutPipeline<Live> + 'static,
{
fn publish_copy<'a>(
&'a self,
msg: OutgoingMessage<'a>,
cx: &'a PublishContext<'a, Cx>,
) -> BoxFuture<'a, Result<(), BoxError>> {
Box::pin(async move {
let (name, payload, delivered) = msg.into_parts();
let headers = if let Some(base) = self.live.base_headers() {
let mut headers = base.clone();
for (key, value) in delivered.iter() {
headers.insert(key.to_owned(), value.to_owned());
}
headers
} else {
delivered
};
let mut out = <Live::Payload as PayloadForm>::rebuilt(
name.into(),
<Live::Payload as PayloadForm>::Form::from(payload),
headers,
);
let mut options: Option<Live::Options> = None;
self.stack.apply(&mut out, &mut options, cx);
let mut kept = <Live::Payload as PayloadForm>::Spent::default();
let sent = <Live::Payload as PayloadForm>::handed_on(out, &mut kept);
let publish = self.pipeline.send(&self.live, sent, options.as_ref());
#[cfg(feature = "testing")]
let publish = publishing_to(self.origin, publish);
publish.await.map_err(|err| Box::new(err) as BoxError)
})
}
}
#[cfg(all(test, feature = "memory"))]
pub(crate) fn bare_retry_publisher<Cx, P>(live: P) -> Arc<dyn ErasedRetryPublisher<Cx>>
where
Cx: Send + Sync + 'static,
P: Publisher + Send + Sync + 'static,
{
Arc::new(RetryLeaf {
live,
stack: PublishTransformIdentity,
pipeline: crate::runtime::publish::PublishIdentity,
#[cfg(feature = "testing")]
origin: Origin::default(),
})
}
#[doc(hidden)]
#[diagnostic::on_unimplemented(
message = "subscription descriptor `{Source}` publishes no retry copy in this process",
label = "`out_retry(..)` has no publisher to name on this registration",
note = "`{Source}` declares `Copies = BrokerMoves`: the broker or its client library moves \
the delivery itself (a delivery limit with a dead-letter exchange, a subscription's \
dead-letter policy, a redrive policy), so nothing of this process's is published for \
it and a publish policy has nothing to shape",
note = "declare the cap and the destination instead - `.max_attempts(n).dead_letter(\"dlq\")` \
- and that descriptor maps them onto the broker's own mechanism"
)]
pub trait PublishesCopiesHere<Source> {}
impl<Source> PublishesCopiesHere<Source> for AddressedCopies {}
impl<Source> PublishesCopiesHere<Source> for NamedCopies {}
#[doc(hidden)]
#[diagnostic::on_unimplemented(
message = "this registration does not say where its retry copies go",
label = "subscription descriptor `{Source}` cannot address them",
note = "`{Source}` declares `Copies = NamedCopies`: one such subscription reads many \
destinations (a wildcard subject, a filter, a pattern, a list of topics), so it names \
none of them and the mount site names one",
note = "name a fixed destination - `.out_retry(policy).to(\"orders\")` - or compose a publish \
transform that names one per delivery, which reads the delivery being retried"
)]
pub trait CopiesAddressed<Source> {}
impl<Source> CopiesAddressed<Source> for AddressedCopies {}
impl<Source> CopiesAddressed<Source> for BrokerMoves {}
#[doc(hidden)]
#[diagnostic::on_unimplemented(
message = "broker `{B}` names no default publish policy, so a retry copy has nothing to \
leave through",
label = "this subscription's retry copies are published by the runtime",
note = "the descriptor declares a copy path this process publishes on, so every registration \
on it gets a retry publisher paired from `DefaultPublish`; implement `DefaultPublish` \
on the broker's connected form, or declare `Copies = BrokerMoves` on a descriptor \
whose deliveries the broker moves itself"
)]
pub trait CopyPathPairing<B: Broker, Cx, Pipeline> {
fn pairing(pipeline: Pipeline) -> Option<RetryPairing<B, Cx>>;
}
macro_rules! impl_default_pairing {
($($path:ident),+ $(,)?) => {$(
impl<B, Cx, Pipeline> CopyPathPairing<B, Cx, Pipeline> for $path
where
B: Broker + 'static,
Cx: Send + Sync + 'static,
Connected<B>: DefaultPublish,
DefaultLive<B>: Publisher + Send + Sync + 'static,
Pipeline: OutPipeline<DefaultLive<B>> + 'static,
{
fn pairing(pipeline: Pipeline) -> Option<RetryPairing<B, Cx>> {
Some(RetryPairing::new(
DefaultPolicy::<B>::default(),
PublishTransformIdentity,
pipeline,
))
}
}
)+};
}
impl_default_pairing!(AddressedCopies, NamedCopies);
impl<B: Broker, Cx, Pipeline> CopyPathPairing<B, Cx, Pipeline> for BrokerMoves {
fn pairing(_pipeline: Pipeline) -> Option<RetryPairing<B, Cx>> {
None
}
}
#[doc(hidden)]
pub trait CopyPathAddress<C: ConnectedBroker, Source> {
fn address(
source: &Source,
connected: &C,
) -> impl Future<Output = Result<Option<RedeliveryAddress>, C::Error>> + Send;
}
impl<C, Source> CopyPathAddress<C, Source> for AddressedCopies
where
C: ConnectedBroker,
Source: RedeliveryAddressed<C> + Sync,
{
async fn address(
source: &Source,
connected: &C,
) -> Result<Option<RedeliveryAddress>, C::Error> {
source.redelivery_address(connected).await.map(Some)
}
}
macro_rules! impl_no_address {
($($path:ident),+ $(,)?) => {$(
impl<C: ConnectedBroker, Source> CopyPathAddress<C, Source> for $path {
fn address(
_source: &Source,
_connected: &C,
) -> impl Future<Output = Result<Option<RedeliveryAddress>, C::Error>> + Send {
ready(Ok(None))
}
}
)+};
}
impl_no_address!(NamedCopies, BrokerMoves);
#[doc(hidden)]
pub struct RetrySetup<B: Broker, Cx> {
publisher: Option<RetryPairing<B, Cx>>,
destination: Option<Cow<'static, str>>,
named_per_delivery: bool,
declaration: RetryDeclaration,
}
impl<B: Broker, Cx> fmt::Debug for RetrySetup<B, Cx> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("RetrySetup")
.field("destination", &self.destination)
.field("declaration", &self.declaration)
.finish_non_exhaustive()
}
}
impl<B: Broker, Cx> Default for RetrySetup<B, Cx> {
fn default() -> Self {
Self {
publisher: None,
destination: None,
named_per_delivery: false,
declaration: RetryDeclaration::new(),
}
}
}
impl<B: Broker + 'static, Cx> RetrySetup<B, Cx> {
pub(crate) fn with_publisher(
mut self,
publisher: RetryPairing<B, Cx>,
destination: Option<Cow<'static, str>>,
named_per_delivery: bool,
) -> Self {
self.publisher = Some(publisher);
self.named_per_delivery = named_per_delivery;
self.destination = destination;
self
}
pub(crate) fn with_declaration(mut self, declaration: RetryDeclaration) -> Self {
self.declaration = declaration;
self
}
#[must_use]
pub(crate) fn resolve<Copies, Pipeline>(
mut self,
pipeline: &Pipeline,
meta: &mut HandlerMetadata,
) -> Self
where
Copies: CopyPathPairing<B, Cx, Pipeline>,
Pipeline: Clone,
{
if self.publisher.is_none() {
self.publisher = Copies::pairing(pipeline.clone());
}
if let Some(publisher) = &self.publisher {
meta.describe_copies(|channel| publisher.describe(channel));
}
self
}
}
#[doc(hidden)]
pub struct RetryPairing<B: Broker, Cx>(Box<dyn BoundRetry<B, Cx>>);
impl<B: Broker, Cx> fmt::Debug for RetryPairing<B, Cx> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("RetryPairing").finish_non_exhaustive()
}
}
trait BoundRetry<B: Broker, Cx>: Send {
fn describe(&self, channel: &str) -> PublishDescription;
fn pair(
self: Box<Self>,
connected: &Connected<B>,
) -> BoxFuture<'_, Result<Arc<dyn ErasedRetryPublisher<Cx>>, PairError>>;
}
struct RetryParts<Policy, Stack, Pipe> {
policy: Policy,
stack: Stack,
pipeline: Pipe,
}
impl<B, Cx, Policy, Stack, Pipe> BoundRetry<B, Cx> for RetryParts<Policy, Stack, Pipe>
where
B: Broker + 'static,
Cx: Send + Sync + 'static,
Policy: PublishPolicy<Connected<B>> + Send + 'static,
Policy::Live: Publisher + Send + Sync + 'static,
Stack: PublishTransform<ForReply<Cx>, <Policy::Live as Publisher>::Options>
+ Send
+ Sync
+ 'static,
Pipe: OutPipeline<Policy::Live> + Send + 'static,
{
fn describe(&self, channel: &str) -> PublishDescription {
PublishDescription::of::<Connected<B>, Policy>(&self.policy, None, false, channel)
}
fn pair(
self: Box<Self>,
connected: &Connected<B>,
) -> BoxFuture<'_, Result<Arc<dyn ErasedRetryPublisher<Cx>>, PairError>> {
Box::pin(async move {
let Self {
policy,
stack,
pipeline,
} = *self;
let pairing = policy.pair(connected);
#[cfg(feature = "testing")]
let (live, origin) = paired(connected, pairing).await;
#[cfg(not(feature = "testing"))]
let live = pairing.await;
Ok(Arc::new(RetryLeaf {
live: live?,
stack,
pipeline,
#[cfg(feature = "testing")]
origin,
}) as Arc<dyn ErasedRetryPublisher<Cx>>)
})
}
}
impl<B: Broker, Cx> RetryPairing<B, Cx> {
pub(crate) fn describe(&self, channel: &str) -> PublishDescription {
self.0.describe(channel)
}
}
impl<B: Broker + 'static, Cx: Send + Sync + 'static> RetryPairing<B, Cx> {
pub(crate) fn new<Policy, Stack, Pipe>(policy: Policy, stack: Stack, pipeline: Pipe) -> Self
where
Policy: PublishPolicy<Connected<B>> + Send + 'static,
Policy::Live: Publisher + Send + Sync + 'static,
Stack: PublishTransform<ForReply<Cx>, <Policy::Live as Publisher>::Options>
+ Send
+ Sync
+ 'static,
Pipe: OutPipeline<Policy::Live> + Send + 'static,
{
Self(Box::new(RetryParts {
policy,
stack,
pipeline,
}))
}
async fn pair(
self,
connected: &Connected<B>,
) -> Result<Arc<dyn ErasedRetryPublisher<Cx>>, PairError> {
self.0.pair(connected).await
}
}
pub(crate) struct ScopeDelivery {
tasks: TaskTracker,
#[cfg(feature = "testing")]
hooks: Arc<TestHooks>,
#[cfg(feature = "testing")]
scope_id: usize,
#[cfg(feature = "testing")]
subscription: usize,
#[cfg(feature = "poll-diagnostics")]
poll: PollDiagnostics,
}
impl ScopeDelivery {
pub(crate) fn new(
tasks: TaskTracker,
#[cfg(feature = "testing")] hooks: Arc<TestHooks>,
#[cfg(feature = "testing")] scope_id: usize,
#[cfg(feature = "testing")] subscription: usize,
#[cfg(feature = "poll-diagnostics")] poll: PollDiagnostics,
) -> Self {
Self {
tasks,
#[cfg(feature = "testing")]
hooks,
#[cfg(feature = "testing")]
scope_id,
#[cfg(feature = "testing")]
subscription,
#[cfg(feature = "poll-diagnostics")]
poll,
}
}
#[cfg(feature = "poll-diagnostics")]
pub(crate) const fn poll(&self) -> &PollDiagnostics {
&self.poll
}
pub(crate) fn tasks(&self) -> &TaskTracker {
&self.tasks
}
#[cfg(feature = "testing")]
pub(crate) fn hooks(&self) -> &Arc<TestHooks> {
&self.hooks
}
#[cfg(feature = "testing")]
pub(crate) fn scope_id(&self) -> usize {
self.scope_id
}
#[cfg(feature = "testing")]
pub(crate) fn subscription(&self) -> usize {
self.subscription
}
}
#[derive(Debug, Error)]
#[error(
"subscription `{subscription}`: descriptor `{source_type}` declares `Copies = NamedCopies`, \
so it addresses no retry copies, and this registration names no destination for them. Name \
one at the mount site with `.out_retry(policy).to(\"name\")`, or compose a publish transform \
declaring `Destination = Names` that names one per delivery"
)]
pub(crate) struct RetryDestinationError {
subscription: String,
source_type: &'static str,
}
#[derive(Debug, Error)]
#[error("subscription `{subscription}`: {source}")]
pub(crate) struct RetryDeclareError {
subscription: String,
#[source]
source: DeclareRetryError,
}
#[derive(Debug, Error)]
#[error(
"subscription `{subscription}`: the deferred-retry policy bound with `out_retry` failed to \
pair: {source}"
)]
pub(crate) struct RetryPairError {
subscription: String,
#[source]
source: PairError,
}
pub(crate) async fn open_subscription<B, Source, Cx>(
source: Source,
connected: &Connected<B>,
scope: &ScopeDelivery,
subscription: &str,
setup: RetrySetup<B, Cx>,
) -> Result<(Source::Subscriber, Arc<Delivery<Cx>>), BoxError>
where
B: Broker + 'static,
Cx: Send + Sync + 'static,
Source: SubscriptionSource<Connected<B>>,
Source::Copies: CopyPathAddress<Connected<B>, Source>,
{
let RetrySetup {
publisher,
destination,
named_per_delivery,
declaration,
} = setup;
source
.declare_retry_on(connected, &declaration)
.map_err(|err| {
Box::new(RetryDeclareError {
subscription: subscription.to_owned(),
source: err,
}) as BoxError
})?;
let retry = match publisher {
Some(pairing) => {
let publisher = pairing.pair(connected).await.map_err(|err| {
Box::new(RetryPairError {
subscription: subscription.to_owned(),
source: err,
}) as BoxError
})?;
let destination = match destination {
Some(named) => Some(Arc::from(named.as_ref())),
None => <Source::Copies as CopyPathAddress<Connected<B>, Source>>::address(
&source, connected,
)
.await
.map_err(|err| Box::new(err) as BoxError)?
.map(|address| Arc::from(address.as_str())),
};
if destination.is_none() && !named_per_delivery {
return Err(Box::new(RetryDestinationError {
subscription: subscription.to_owned(),
source_type: type_name::<Source>(),
}));
}
Some(DeferredRetry {
publisher,
destination,
})
}
None => None,
};
announce(subscription, &declaration, retry.as_ref());
let subscriber = source
.declare_retry(&declaration)
.subscribe(connected)
.await
.map_err(|err| Box::new(err) as BoxError)?;
Ok((
subscriber,
Arc::new(Delivery::for_subscription(
scope,
subscription,
retry,
declaration,
)),
))
}
fn announce<Cx>(
subscription: &str,
declaration: &RetryDeclaration,
retry: Option<&DeferredRetry<Cx>>,
) {
if declaration.declares_nothing() {
return;
}
let max_attempts = declaration.max_attempts().map(NonZeroU32::get);
if retry.is_some() {
info!(
target: "ruststream::retry",
subscription = %subscription,
max_attempts,
dead_letter = declaration.dead_letter(),
applied_by = "runtime",
"retry declaration; a cap applies to the copies this process publishes, and to a \
native delayed redelivery through the broker's delivery count where the transport \
keeps one and the framework's retry-count header otherwise, never both",
);
} else {
info!(
target: "ruststream::retry",
subscription = %subscription,
max_attempts,
dead_letter = declaration.dead_letter(),
applied_by = "broker",
"retry declaration; the broker moves a spent delivery itself and applies it",
);
}
}
pub(crate) fn open_mounted_subscriber<Cx>(
scope: &ScopeDelivery,
subscription: &str,
) -> Arc<Delivery<Cx>> {
Arc::new(Delivery::for_subscription(
scope,
subscription,
None,
RetryDeclaration::new(),
))
}