Skip to main content

anycms_event/
bus.rs

1//! The core [`EventBus`] implementation.
2//!
3//! Uses `tokio::broadcast` channels internally for efficient fan-out
4//! pub/sub semantics with back-pressure support.
5
6use std::any::Any;
7use std::sync::Arc;
8use std::sync::RwLock;
9use std::sync::atomic::{AtomicUsize, Ordering};
10use std::time::Duration;
11
12use dashmap::DashMap;
13
14use tokio::sync::broadcast;
15use tokio::task::AbortHandle;
16use tokio::task::JoinHandle;
17
18use crate::error::{EventBusError, PublishErrorReason, Result};
19use crate::event::Event;
20use crate::registry::EventRegistry;
21use crate::execution_log::ExecutionLog;
22use crate::telemetry::Telemetry;
23
24/// Type-erased event payload. Events are wrapped in `Arc<dyn Any + Send + Sync>`
25/// so they can be sent through broadcast channels without serialization.
26type ErasedEvent = Arc<dyn Any + Send + Sync>;
27
28/// Callback invoked after an event is published, receiving the event name
29/// and its JSON representation. Used by observers like `TriggerRuleEngine`.
30type PublishCallback = Arc<dyn Fn(&str, serde_json::Value) + Send + Sync>;
31
32/// Wrapper used on the global dispatch channel.
33/// Carries the event name (used for pattern matching) alongside the type-erased payload.
34#[derive(Clone)]
35struct GlobalEvent {
36    event_name: String,
37    payload: ErasedEvent,
38}
39
40// ── Retry & Dead Letter Types ──────────────────────────────────────
41
42/// 重试退避策略。
43#[derive(Clone, Debug)]
44pub enum RetryBackoff {
45    /// 固定间隔。
46    Fixed(Duration),
47    /// 指数退避(delay = base * 2^attempt,不超过 max)。
48    Exponential { base: Duration, max: Duration },
49}
50
51impl Default for RetryBackoff {
52    fn default() -> Self {
53        Self::Exponential {
54            base: Duration::from_millis(100),
55            max: Duration::from_secs(10),
56        }
57    }
58}
59
60/// Handler 执行重试策略。
61#[derive(Clone, Debug)]
62pub struct RetryPolicy {
63    /// 最大重试次数(0 = 不重试,默认)。
64    pub max_retries: usize,
65    /// 退避策略。
66    pub backoff: RetryBackoff,
67    /// 每次执行超时时间。
68    pub timeout_per_attempt: Duration,
69}
70
71impl Default for RetryPolicy {
72    fn default() -> Self {
73        Self {
74            max_retries: 0, // 默认不重试,保持向后兼容
75            backoff: RetryBackoff::default(),
76            timeout_per_attempt: Duration::from_secs(30),
77        }
78    }
79}
80
81impl RetryPolicy {
82    /// 计算第 N 次重试前的等待时间。
83    pub fn delay_for_attempt(&self, attempt: usize) -> Duration {
84        match &self.backoff {
85            RetryBackoff::Fixed(d) => *d,
86            RetryBackoff::Exponential { base, max } => {
87                let exp = 2u32.saturating_pow(attempt.min(16) as u32);
88                base.saturating_mul(exp as u32).min(*max)
89            }
90        }
91    }
92}
93
94/// 死信处理器 — 当 Handler 重试耗尽后调用。
95pub trait DeadLetterHandler: Send + Sync + 'static {
96    /// 事件处理彻底失败时调用。
97    fn on_dead_letter(&self, event_name: &str, attempts: usize, error: &str);
98}
99
100/// 默认死信处理器 — 使用 tracing::error! 记录。
101pub struct LoggingDeadLetterHandler;
102
103impl DeadLetterHandler for LoggingDeadLetterHandler {
104    fn on_dead_letter(&self, event_name: &str, attempts: usize, error: &str) {
105        tracing::error!(
106            event = event_name,
107            attempts = attempts,
108            error = error,
109            "Event handler failed after all retries (dead letter)"
110        );
111    }
112}
113
114/// A subscription handle that allows unsubscribing or graceful shutdown.
115pub struct Subscription {
116    /// The event name or pattern this subscription is bound to.
117    pub event_name: String,
118    /// Unique subscription identifier.
119    pub id: usize,
120    /// Abort handle for cancelling the subscriber task.
121    abort_handle: AbortHandle,
122    /// Reference to inner state for task cleanup on unsubscribe.
123    inner: Arc<EventBusInner>,
124}
125
126impl std::fmt::Debug for Subscription {
127    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
128        f.debug_struct("Subscription")
129            .field("event_name", &self.event_name)
130            .field("id", &self.id)
131            .field("is_finished", &self.abort_handle.is_finished())
132            .finish()
133    }
134}
135
136impl Subscription {
137    /// Unsubscribe by aborting the background handler task.
138    ///
139    /// After calling this, the handler will no longer receive events.
140    /// This is idempotent — calling it multiple times has no effect.
141    pub fn unsubscribe(&self) {
142        self.abort_handle.abort();
143        self.inner.tasks.remove(&self.id);
144    }
145
146    /// Check if this subscription is still active.
147    pub fn is_finished(&self) -> bool {
148        self.abort_handle.is_finished()
149    }
150}
151
152/// Thread-safe, async event bus backed by tokio broadcast channels.
153///
154/// # Overview
155///
156/// The `EventBus` provides a publish/subscribe pattern where:
157/// - **Publishers** send typed events via [`EventBus::publish`].
158/// - **Subscribers** register async handlers via [`EventBus::subscribe`].
159/// - Events are type-erased via `Arc<dyn Any + Send + Sync>` and sent through
160///   broadcast channels without serialization.
161/// - Each event type gets its own dedicated broadcast channel.
162/// - Pattern subscriptions use a global channel that receives all published events.
163///
164/// # Thread Safety
165///
166/// Channel bookkeeping uses `std::sync::RwLock` for fast, non-blocking
167/// HashMap lookups on the hot path. The bus is wrapped in `Arc`, making it
168/// safe to share across tasks. It implements `Clone` (cheap reference clone)
169/// and `Send + Sync`.
170///
171/// # Example
172///
173/// ```ignore
174/// use anycms_event::prelude::*;
175///
176/// #[derive(Clone, Debug)]
177/// struct UserCreated { name: String }
178///
179/// impl Event for UserCreated {
180///     fn event_name() -> &'static str { "user.created" }
181/// }
182///
183/// let bus = EventBus::new();
184///
185/// bus.subscribe(|event: UserCreated| async move {
186///     println!("New user: {}", event.name);
187///     Ok(())
188/// }).await?;
189///
190/// bus.publish(UserCreated { name: "Alice".into() }).await?;
191/// ```
192pub struct EventBus {
193    inner: Arc<EventBusInner>,
194}
195
196struct EventBusInner {
197    /// Broadcast channels keyed by event name.
198    channels: DashMap<String, broadcast::Sender<ErasedEvent>>,
199    /// Global channel: every published event is also sent here.
200    /// Pattern subscribers listen on this channel and filter with `topic::matches`.
201    global_channel: broadcast::Sender<GlobalEvent>,
202    /// Registered topic patterns and the event names they match.
203    topic_patterns: DashMap<String, Vec<String>>,
204    /// Monotonically increasing subscription ID counter.
205    next_sub_id: AtomicUsize,
206    /// Capacity for new broadcast channels.
207    capacity: usize,
208    /// 可插拔的遥测层,用于监控发布/订阅生命周期。
209    telemetry: Option<Arc<dyn Telemetry>>,
210    /// 事件注册表,跟踪已注册的事件类型及其元数据。
211    registry: Arc<EventRegistry>,
212    /// 执行日志,用于查询事件发布和 Handler 执行的历史记录。
213    execution_log: Option<Arc<ExecutionLog>>,
214    /// Publish callbacks invoked after each event is published.
215    /// Used by observers like `TriggerRuleEngine` that need to see all events.
216    publish_callbacks: RwLock<Vec<PublishCallback>>,
217    /// Track spawned subscriber tasks for graceful shutdown.
218    tasks: DashMap<usize, JoinHandle<()>>,
219    /// 默认重试策略。
220    retry_policy: RetryPolicy,
221    /// 死信处理器。
222    dead_letter: Option<Arc<dyn DeadLetterHandler>>,
223}
224
225impl EventBus {
226    /// Create a new event bus with the default channel capacity (1024).
227    pub fn new() -> Self {
228        Self::from_builder(
229            1024, None, Arc::new(EventRegistry::new()), None,
230            RetryPolicy::default(), None,
231        )
232    }
233
234    /// Create a new event bus with the specified broadcast channel capacity.
235    ///
236    /// The capacity controls how many messages can be buffered before slow
237    /// subscribers start being lagged (dropping old messages).
238    pub fn with_capacity(capacity: usize) -> Self {
239        Self::from_builder(
240            capacity, None, Arc::new(EventRegistry::new()), None,
241            RetryPolicy::default(), None,
242        )
243    }
244
245    /// 从构建器参数创建 EventBus(内部方法)。
246    pub(crate) fn from_builder(
247        capacity: usize,
248        telemetry: Option<Arc<dyn Telemetry>>,
249        registry: Arc<EventRegistry>,
250        execution_log: Option<Arc<ExecutionLog>>,
251        retry_policy: RetryPolicy,
252        dead_letter: Option<Arc<dyn DeadLetterHandler>>,
253    ) -> Self {
254        let (global_tx, _) = broadcast::channel(capacity);
255        Self {
256            inner: Arc::new(EventBusInner {
257                channels: DashMap::new(),
258                global_channel: global_tx,
259                topic_patterns: DashMap::new(),
260                next_sub_id: AtomicUsize::new(0),
261                capacity,
262                telemetry,
263                registry,
264                execution_log,
265                publish_callbacks: RwLock::new(Vec::new()),
266                tasks: DashMap::new(),
267                retry_policy,
268                dead_letter,
269            }),
270        }
271    }
272
273    /// 返回一个 [`EventBusBuilder`] 用于配置 EventBus。
274    pub fn builder() -> crate::builder::EventBusBuilder {
275        crate::builder::EventBusBuilder::new()
276    }
277
278    /// 获取事件注册表引用。
279    ///
280    /// 注册表跟踪所有已发布/订阅的事件类型及其元数据。
281    pub fn registry(&self) -> &Arc<EventRegistry> {
282        &self.inner.registry
283    }
284
285    /// 获取执行日志引用(如果已配置)。
286    ///
287    /// 执行日志记录了事件发布和 Handler 执行的历史。
288    pub fn execution_log(&self) -> Option<&Arc<ExecutionLog>> {
289        self.inner.execution_log.as_ref()
290    }
291
292    /// Register a publish callback that is invoked after every event is published.
293    ///
294    /// The callback receives the event name and its JSON representation.
295    /// Events that return `None` from [`Event::to_json`] will not trigger callbacks.
296    ///
297    /// This is the primary mechanism for `TriggerRuleEngine` to observe all events
298    /// without needing a typed subscription.
299    pub fn register_publish_callback(&self, callback: PublishCallback) {
300        self.inner.publish_callbacks.write().unwrap().push(callback);
301    }
302
303    /// Get or create a broadcast channel for the given event type.
304    ///
305    /// Uses DashMap's atomic entry API for lock-free access on the hot path.
306    fn get_or_create_channel<E: Event>(&self) -> broadcast::Sender<ErasedEvent> {
307        self.inner
308            .channels
309            .entry(E::event_name().to_string())
310            .or_insert_with(|| {
311                // Auto-register in registry when channel is first created
312                if !self.inner.registry.contains(E::event_name()) {
313                    self.inner.registry.register_simple(E::event_name(), E::topic());
314                }
315                broadcast::channel(self.inner.capacity).0
316            })
317            .clone()
318    }
319
320    /// Publish a typed event to the bus.
321    ///
322    /// The event is type-erased via `Arc<dyn Any + Send + Sync>` and sent
323    /// through the broadcast channel associated with its event type. If there
324    /// are no subscribers, the publish is a no-op (not an error).
325    ///
326    /// The event is also sent to the global channel for pattern subscribers.
327    ///
328    /// # Errors
329    ///
330    /// Returns [`EventBusError::PublishFailed`] if the channel returns an
331    /// unexpected error.
332    pub async fn publish<E: Event>(&self, event: E) -> Result<()> {
333        let start = std::time::Instant::now();
334
335        // Auto-register event in registry if not yet registered
336        if !self.inner.registry.contains(E::event_name()) {
337            self.inner.registry.register_simple(E::event_name(), E::topic());
338        }
339        self.inner.registry.increment_publish_count(E::event_name());
340
341        // Extract JSON for publish callbacks (before moving event)
342        let event_json = event.to_json();
343
344        // Fast path: check if channel exists with DashMap
345        let sender = {
346            match self.inner.channels.get(E::event_name()) {
347                Some(sender) if sender.receiver_count() > 0 => Some(sender.clone()),
348                Some(_) => {
349                    // Channel exists but no subscribers
350                    tracing::debug!(
351                        event = E::event_name(),
352                        receivers = 0,
353                        "Event published (no subscribers)"
354                    );
355                    // Still send to global channel below
356                    None
357                }
358                None => {
359                    // Channel doesn't exist, no subscribers ever registered
360                    tracing::debug!(
361                        event = E::event_name(),
362                        receivers = 0,
363                        "Event published (no channel)"
364                    );
365                    // Still send to global channel below
366                    None
367                }
368            }
369        };
370
371        if let Some(sender) = sender {
372            // Type-erase the event
373            let payload: ErasedEvent = Arc::new(event.clone());
374            let receiver_count = sender.receiver_count();
375
376            // Telemetry: publish started
377            if let Some(ref tel) = self.inner.telemetry {
378                tel.on_publish(E::event_name(), receiver_count);
379            }
380
381            sender
382                .send(payload)
383                .map_err(|e| EventBusError::PublishFailed {
384                    event_name: E::event_name(),
385                    reason: PublishErrorReason::ChannelError(e.to_string()),
386                })?;
387
388            tracing::debug!(
389                event = E::event_name(),
390                receivers = receiver_count,
391                "Event published"
392            );
393
394            // Telemetry: publish complete
395            if let Some(ref tel) = self.inner.telemetry {
396                tel.on_publish_complete(E::event_name(), start.elapsed());
397            }
398
399            // Also publish to the global channel for pattern subscribers
400            let global_event = GlobalEvent {
401                event_name: E::event_name().to_string(),
402                payload: Arc::new(event) as ErasedEvent,
403            };
404            let _ = self.inner.global_channel.send(global_event);
405        } else {
406            // No per-event-type subscribers, but still publish to global channel
407            let global_event = GlobalEvent {
408                event_name: E::event_name().to_string(),
409                payload: Arc::new(event) as ErasedEvent,
410            };
411            let _ = self.inner.global_channel.send(global_event);
412        }
413
414        // Notify publish callbacks (for trigger engine etc.)
415        if let Some(json) = event_json {
416            let callbacks = self.inner.publish_callbacks.read().unwrap();
417            for cb in callbacks.iter() {
418                cb(E::event_name(), json.clone());
419            }
420        }
421
422        Ok(())
423    }
424
425    /// Subscribe to a specific event type with an async handler.
426    ///
427    /// The handler is spawned as a background tokio task that listens for
428    /// events on the broadcast channel. If the handler falls behind,
429    /// lagged messages are logged as warnings but the subscriber continues.
430    ///
431    /// Uses the bus-level [`RetryPolicy`] (default: no retry).
432    /// For custom retry, use [`EventBus::subscribe_with_retry`].
433    ///
434    /// # Type Parameters
435    ///
436    /// - `E`: The event type to subscribe to (must implement [`Event`]).
437    /// - `F`: The handler closure type.
438    /// - `Fut`: The future returned by the handler closure.
439    ///
440    /// # Returns
441    ///
442    /// A [`Subscription`] handle with the event name and a unique ID.
443    /// The subscription can be used to unsubscribe via [`Subscription::unsubscribe`].
444    pub async fn subscribe<E, F, Fut>(&self, handler: F) -> Result<Subscription>
445    where
446        E: Event,
447        F: Fn(E) -> Fut + Send + Sync + 'static,
448        Fut: std::future::Future<Output = Result<()>> + Send + 'static,
449    {
450        self.subscribe_internal(handler, self.inner.retry_policy.clone()).await
451    }
452
453    /// Subscribe with a custom retry policy (overrides the bus default).
454    pub async fn subscribe_with_retry<E, F, Fut>(
455        &self,
456        handler: F,
457        retry_policy: RetryPolicy,
458    ) -> Result<Subscription>
459    where
460        E: Event,
461        F: Fn(E) -> Fut + Send + Sync + 'static,
462        Fut: std::future::Future<Output = Result<()>> + Send + 'static,
463    {
464        self.subscribe_internal(handler, retry_policy).await
465    }
466
467    /// Internal subscribe implementation with configurable retry.
468    async fn subscribe_internal<E, F, Fut>(
469        &self,
470        handler: F,
471        retry_policy: RetryPolicy,
472    ) -> Result<Subscription>
473    where
474        E: Event,
475        F: Fn(E) -> Fut + Send + Sync + 'static,
476        Fut: std::future::Future<Output = Result<()>> + Send + 'static,
477    {
478        let sender = self.get_or_create_channel::<E>();
479        let mut rx = sender.subscribe();
480
481        let event_name = E::event_name().to_string();
482        let id = self.inner.next_sub_id.fetch_add(1, Ordering::Relaxed);
483
484        // Update registry subscriber count
485        let sub_count = sender.receiver_count();
486        self.inner.registry.set_subscriber_count(&event_name, sub_count);
487
488        // Clone telemetry Arc for use inside the spawned task
489        let telemetry = self.inner.telemetry.clone();
490        let dead_letter = self.inner.dead_letter.clone();
491
492        let handler_event_name = event_name.clone();
493        let handle: JoinHandle<()> = tokio::spawn(async move {
494            loop {
495                match rx.recv().await {
496                    Ok(payload) => {
497                        match payload.downcast_ref::<E>() {
498                            Some(event) => {
499                                // Telemetry: handler start
500                                if let Some(ref tel) = telemetry {
501                                    tel.on_handler_start(&handler_event_name, id);
502                                }
503                                let handler_start = std::time::Instant::now();
504
505                                // Retry loop
506                                let mut last_error_str = None;
507                                let mut success = false;
508                                for attempt in 0..=retry_policy.max_retries {
509                                    if attempt > 0 {
510                                        let delay = retry_policy.delay_for_attempt(attempt - 1);
511                                        tracing::debug!(
512                                            event = %handler_event_name,
513                                            attempt = attempt + 1,
514                                            delay_ms = delay.as_millis(),
515                                            "Retrying handler"
516                                        );
517                                        tokio::time::sleep(delay).await;
518                                    }
519
520                                    match tokio::time::timeout(
521                                        retry_policy.timeout_per_attempt,
522                                        handler(event.clone()),
523                                    ).await {
524                                        Ok(Ok(())) => {
525                                            success = true;
526                                            break;
527                                        }
528                                        Ok(Err(e)) => {
529                                            last_error_str = Some(e.to_string());
530                                            tracing::warn!(
531                                                event = %handler_event_name,
532                                                attempt = attempt + 1,
533                                                max_retries = retry_policy.max_retries,
534                                                error = %e,
535                                                "Handler failed"
536                                            );
537                                        }
538                                        Err(_) => {
539                                            last_error_str = Some("Handler timeout".to_string());
540                                            tracing::warn!(
541                                                event = %handler_event_name,
542                                                attempt = attempt + 1,
543                                                "Handler timed out"
544                                            );
545                                        }
546                                    }
547                                }
548
549                                let handler_elapsed = handler_start.elapsed();
550
551                                // Dead letter if all retries exhausted
552                                if !success {
553                                    if let Some(ref dl) = dead_letter {
554                                        dl.on_dead_letter(
555                                            &handler_event_name,
556                                            retry_policy.max_retries + 1,
557                                            last_error_str.as_deref().unwrap_or("unknown"),
558                                        );
559                                    }
560                                }
561
562                                // Telemetry: handler complete
563                                if let Some(ref tel) = telemetry {
564                                    let err_str = if success {
565                                        None
566                                    } else {
567                                        last_error_str
568                                    };
569                                    tel.on_handler_complete(
570                                        &handler_event_name,
571                                        id,
572                                        handler_elapsed,
573                                        err_str.as_deref(),
574                                    );
575                                }
576                            }
577                            None => {
578                                tracing::error!(
579                                    event = %handler_event_name,
580                                    "Failed to downcast event"
581                                );
582                            }
583                        }
584                    }
585                    Err(broadcast::error::RecvError::Lagged(n)) => {
586                        tracing::warn!(
587                            event = %handler_event_name,
588                            lagged = n,
589                            "Subscriber lagged behind"
590                        );
591                        // Telemetry: handler lagged
592                        if let Some(ref tel) = telemetry {
593                            tel.on_handler_lagged(&handler_event_name, id, n as usize);
594                        }
595                    }
596                    Err(broadcast::error::RecvError::Closed) => {
597                        tracing::debug!(
598                            event = %handler_event_name,
599                            "Channel closed, stopping subscriber"
600                        );
601                        break;
602                    }
603                }
604            }
605        });
606
607        let abort_handle = handle.abort_handle();
608
609        // Store JoinHandle for graceful shutdown
610        self.inner.tasks.insert(id, handle);
611
612        // Telemetry: subscriber registered
613        if let Some(ref tel) = self.inner.telemetry {
614            tel.on_subscribe(&event_name, id);
615        }
616
617        Ok(Subscription {
618            event_name,
619            id,
620            abort_handle,
621            inner: self.inner.clone(),
622        })
623    }
624
625    /// Subscribe to a topic pattern with wildcard support.
626    ///
627    /// Patterns support:
628    /// - `*` matches a single segment (e.g., `"user.*"` matches `"user.created"`)
629    /// - `**` matches multiple segments (e.g., `"user.**"` matches `"user.foo.bar"`)
630    /// - Exact match when no wildcards are present
631    ///
632    /// Pattern subscribers listen on the global channel and filter events
633    /// using [`crate::topic::matches`]. This allows a single subscriber to
634    /// receive events of different types that share a topic namespace.
635    ///
636    /// Uses the bus-level [`RetryPolicy`] (default: no retry).
637    /// For custom retry, use [`EventBus::subscribe_pattern_with_retry`].
638    pub async fn subscribe_pattern<E, F, Fut>(
639        &self,
640        pattern: &str,
641        handler: F,
642    ) -> Result<Subscription>
643    where
644        E: Event,
645        F: Fn(E) -> Fut + Send + Sync + 'static,
646        Fut: std::future::Future<Output = Result<()>> + Send + 'static,
647    {
648        self.subscribe_pattern_internal(pattern, handler, self.inner.retry_policy.clone()).await
649    }
650
651    /// Subscribe to a pattern with a custom retry policy.
652    pub async fn subscribe_pattern_with_retry<E, F, Fut>(
653        &self,
654        pattern: &str,
655        handler: F,
656        retry_policy: RetryPolicy,
657    ) -> Result<Subscription>
658    where
659        E: Event,
660        F: Fn(E) -> Fut + Send + Sync + 'static,
661        Fut: std::future::Future<Output = Result<()>> + Send + 'static,
662    {
663        self.subscribe_pattern_internal(pattern, handler, retry_policy).await
664    }
665
666    /// Internal pattern subscribe implementation with configurable retry.
667    async fn subscribe_pattern_internal<E, F, Fut>(
668        &self,
669        pattern: &str,
670        handler: F,
671        retry_policy: RetryPolicy,
672    ) -> Result<Subscription>
673    where
674        E: Event,
675        F: Fn(E) -> Fut + Send + Sync + 'static,
676        Fut: std::future::Future<Output = Result<()>> + Send + 'static,
677    {
678        // Register the pattern for bookkeeping
679        self.inner
680            .topic_patterns
681            .entry(pattern.to_string())
682            .or_default()
683            .push(E::event_name().to_string());
684
685        let mut rx = self.inner.global_channel.subscribe();
686        let event_name = E::event_name().to_string();
687        let id = self.inner.next_sub_id.fetch_add(1, Ordering::Relaxed);
688        let pattern_owned = pattern.to_string();
689        let handler_event_name = event_name.clone();
690
691        // Clone telemetry Arc for use inside the spawned task
692        let telemetry = self.inner.telemetry.clone();
693        let dead_letter = self.inner.dead_letter.clone();
694
695        let handle: JoinHandle<()> = tokio::spawn(async move {
696            loop {
697                match rx.recv().await {
698                    Ok(global_event) => {
699                        // Filter: only process if pattern matches
700                        if !crate::topic::matches(&pattern_owned, &global_event.event_name) {
701                            continue;
702                        }
703                        match global_event.payload.downcast_ref::<E>() {
704                            Some(event) => {
705                                // Telemetry: handler start
706                                if let Some(ref tel) = telemetry {
707                                    tel.on_handler_start(&handler_event_name, id);
708                                }
709                                let handler_start = std::time::Instant::now();
710
711                                // Retry loop
712                                let mut last_error_str = None;
713                                let mut success = false;
714                                for attempt in 0..=retry_policy.max_retries {
715                                    if attempt > 0 {
716                                        let delay = retry_policy.delay_for_attempt(attempt - 1);
717                                        tracing::debug!(
718                                            event = %handler_event_name,
719                                            pattern = %pattern_owned,
720                                            attempt = attempt + 1,
721                                            "Retrying pattern handler"
722                                        );
723                                        tokio::time::sleep(delay).await;
724                                    }
725
726                                    match tokio::time::timeout(
727                                        retry_policy.timeout_per_attempt,
728                                        handler(event.clone()),
729                                    ).await {
730                                        Ok(Ok(())) => {
731                                            success = true;
732                                            break;
733                                        }
734                                        Ok(Err(e)) => {
735                                            last_error_str = Some(e.to_string());
736                                            tracing::warn!(
737                                                event = %handler_event_name,
738                                                pattern = %pattern_owned,
739                                                attempt = attempt + 1,
740                                                error = %e,
741                                                "Pattern handler failed"
742                                            );
743                                        }
744                                        Err(_) => {
745                                            last_error_str = Some("Handler timeout".to_string());
746                                        }
747                                    }
748                                }
749
750                                let handler_elapsed = handler_start.elapsed();
751
752                                if !success {
753                                    if let Some(ref dl) = dead_letter {
754                                        dl.on_dead_letter(
755                                            &handler_event_name,
756                                            retry_policy.max_retries + 1,
757                                            last_error_str.as_deref().unwrap_or("unknown"),
758                                        );
759                                    }
760                                }
761
762                                // Telemetry: handler complete
763                                if let Some(ref tel) = telemetry {
764                                    let err_str = if success { None } else { last_error_str };
765                                    tel.on_handler_complete(
766                                        &handler_event_name,
767                                        id,
768                                        handler_elapsed,
769                                        err_str.as_deref(),
770                                    );
771                                }
772                            }
773                            None => {
774                                // Event matched the pattern name but is a different type.
775                                // This is expected when multiple event types share a topic.
776                            }
777                        }
778                    }
779                    Err(broadcast::error::RecvError::Lagged(n)) => {
780                        tracing::warn!(
781                            pattern = %pattern_owned,
782                            lagged = n,
783                            "Pattern subscriber lagged behind"
784                        );
785                        // Telemetry: handler lagged
786                        if let Some(ref tel) = telemetry {
787                            tel.on_handler_lagged(&handler_event_name, id, n as usize);
788                        }
789                    }
790                    Err(broadcast::error::RecvError::Closed) => {
791                        tracing::debug!(
792                            pattern = %pattern_owned,
793                            "Global channel closed, stopping pattern subscriber"
794                        );
795                        break;
796                    }
797                }
798            }
799        });
800
801        let abort_handle = handle.abort_handle();
802
803        // Store JoinHandle for graceful shutdown
804        self.inner.tasks.insert(id, handle);
805
806        // Telemetry: subscriber registered
807        if let Some(ref tel) = self.inner.telemetry {
808            tel.on_subscribe(&event_name, id);
809        }
810
811        Ok(Subscription {
812            event_name,
813            id,
814            abort_handle,
815            inner: self.inner.clone(),
816        })
817    }
818
819    /// Shut down the event bus by aborting all subscriber tasks and clearing channels.
820    ///
821    /// All active subscriber tasks are immediately aborted. This is useful
822    /// for a fast shutdown during application termination.
823    pub fn shutdown(&self) {
824        // Abort all subscriber tasks
825        let task_ids: Vec<usize> = self.inner.tasks.iter().map(|e| *e.key()).collect();
826        for id in task_ids {
827            if let Some((_, handle)) = self.inner.tasks.remove(&id) {
828                handle.abort();
829            }
830        }
831        // Clear per-event channels so any remaining receivers get Closed
832        self.inner.channels.clear();
833    }
834
835    /// Gracefully shut down the event bus, waiting for subscriber tasks to complete.
836    ///
837    /// Clears channels first so no new events arrive, then waits for all
838    /// subscriber tasks to finish. If the timeout elapses before all tasks
839    /// complete, returns the number of tasks that may still be running.
840    ///
841    /// Returns `0` on success (all tasks completed within the timeout).
842    pub async fn shutdown_graceful(&self, timeout: std::time::Duration) -> usize {
843        // Clear channels first so no new events arrive
844        self.inner.channels.clear();
845
846        // Collect all task handles
847        let tasks: Vec<(usize, JoinHandle<()>)> = {
848            let mut handles = Vec::new();
849            let keys: Vec<usize> = self.inner.tasks.iter().map(|e| *e.key()).collect();
850            for key in keys {
851                if let Some(entry) = self.inner.tasks.remove(&key) {
852                    handles.push(entry);
853                }
854            }
855            handles
856        };
857
858        let total = tasks.len();
859        let result = tokio::time::timeout(timeout, async {
860            for (_, handle) in tasks {
861                let _ = handle.await;
862            }
863        })
864        .await;
865
866        if result.is_ok() {
867            0
868        } else {
869            // Timeout — some tasks may still be running
870            total
871        }
872    }
873}
874
875impl Default for EventBus {
876    fn default() -> Self {
877        Self::new()
878    }
879}
880
881impl Clone for EventBus {
882    fn clone(&self) -> Self {
883        Self {
884            inner: self.inner.clone(),
885        }
886    }
887}
888
889// Note: EventBus is Send + Sync because Arc<EventBusInner> is Send + Sync.
890// EventBusInner is Sync because RwLock<HashMap<...>> and AtomicUsize are Sync.