Skip to main content

outbox_core/
manager.rs

1//! Top-level worker that owns the outbox event loop.
2//!
3//! [`OutboxManager`] ties together storage, transport, configuration and a
4//! shutdown signal. It drives the main processing loop — waiting for database
5//! notifications, polling on a fixed interval, draining pending events via
6//! [`OutboxProcessor`], and running a periodic [`GarbageCollector`] task on
7//! the side. Construct one via [`OutboxManagerBuilder`](crate::builder::OutboxManagerBuilder).
8
9use crate::config::OutboxConfig;
10#[cfg(feature = "dlq")]
11use crate::dlq::processor::DlqProcessor;
12use crate::error::OutboxError;
13use crate::gc::GarbageCollector;
14use crate::processor::OutboxProcessor;
15use crate::publisher::Transport;
16use crate::storage::OutboxStorage;
17use serde::Serialize;
18use std::fmt::Debug;
19use std::sync::Arc;
20use std::time::Duration;
21use tokio::sync::watch::Receiver;
22use tracing::{debug, error, info, trace};
23
24/// Long-running worker that publishes pending outbox events to the broker.
25///
26/// The manager owns one concrete [`OutboxStorage`] implementation (`S`), one
27/// [`Transport`] implementation (`P`), and the user's domain event payload type
28/// (`PT`). After [`run`](Self::run) is invoked, it will keep processing events
29/// until the watched shutdown channel flips to `true`.
30///
31/// Construct one through
32/// [`OutboxManagerBuilder`](crate::builder::OutboxManagerBuilder): it reports
33/// missing dependencies with a structured [`OutboxError::ConfigError`] and
34/// shields callers from the manager's `cfg`-dependent constructor signature.
35pub struct OutboxManager<S, P, PT>
36where
37    PT: Debug + Clone + Serialize,
38{
39    storage: Arc<S>,
40    publisher: Arc<P>,
41    config: Arc<OutboxConfig<PT>>,
42    shutdown_rx: Receiver<bool>,
43    #[cfg(feature = "dlq")]
44    dlq_heap: Arc<dyn crate::dlq::storage::DlqHeap>,
45}
46
47impl<S, P, PT> OutboxManager<S, P, PT>
48where
49    S: OutboxStorage<PT> + Send + Sync + 'static,
50    P: Transport<PT> + Send + Sync + 'static,
51    PT: Debug + Clone + Serialize + Send + Sync + 'static,
52{
53    /// Direct constructor used by
54    /// [`OutboxManagerBuilder`](crate::builder::OutboxManagerBuilder).
55    ///
56    /// Crate-private because the signature changes with the `dlq` feature flag,
57    /// which makes it unstable as a public entry point. Application code goes
58    /// through the builder.
59    #[cfg(feature = "dlq")]
60    pub(crate) fn new(
61        storage: Arc<S>,
62        publisher: Arc<P>,
63        config: Arc<OutboxConfig<PT>>,
64        dlq_heap: Arc<dyn crate::dlq::storage::DlqHeap>,
65        shutdown_rx: Receiver<bool>,
66    ) -> Self {
67        Self {
68            storage,
69            publisher,
70            config,
71            shutdown_rx,
72            dlq_heap,
73        }
74    }
75
76    /// Direct constructor used by
77    /// [`OutboxManagerBuilder`](crate::builder::OutboxManagerBuilder).
78    ///
79    /// Crate-private; see the `dlq`-enabled variant for the rationale.
80    #[cfg(not(feature = "dlq"))]
81    pub(crate) fn new(
82        storage: Arc<S>,
83        publisher: Arc<P>,
84        config: Arc<OutboxConfig<PT>>,
85        shutdown_rx: Receiver<bool>,
86    ) -> Self {
87        Self {
88            storage,
89            publisher,
90            config,
91            shutdown_rx,
92        }
93    }
94
95    /// Starts the main outbox worker loop.
96    ///
97    /// This method will run until a shutdown signal is received via the
98    /// `shutdown_rx` channel. It coordinates three concerns:
99    ///
100    /// - **Event processing** — on each wake-up it drives
101    ///   [`OutboxProcessor`] in an inner drain loop until the fetched batch is
102    ///   empty, at which point it returns to waiting.
103    /// - **Wake-up sources** — a `tokio::select!` races a storage-level
104    ///   `LISTEN`/notify call, a poll interval (`config.poll_interval_secs`),
105    ///   and the shutdown receiver. A notification error is logged and the
106    ///   loop sleeps for 5 seconds before retrying.
107    /// - **Garbage collection** — a background task is spawned that ticks on
108    ///   `config.gc_interval_secs` and calls [`GarbageCollector::collect_garbage`],
109    ///   exiting when the shutdown signal fires.
110    ///
111    /// # Errors
112    ///
113    /// The `Result` is reserved for forward compatibility. In the current
114    /// implementation `run` only returns `Ok(())` — it returns when the
115    /// shutdown signal flips to `true`. Transient storage and transport
116    /// failures are logged via `tracing::error!` and the loop continues after
117    /// a short back-off. A future revision may surface terminal errors (for
118    /// example, an unrecoverable storage configuration) through the `Err`
119    /// arm; treat `Err` as a hard stop if and when it appears.
120    ///
121    /// # Example
122    ///
123    /// ```ignore
124    /// use std::sync::Arc;
125    /// use tokio::sync::watch;
126    /// use outbox_core::prelude::*;
127    ///
128    /// # async fn start(
129    /// #     storage: Arc<impl OutboxStorage<MyEvent> + Send + Sync + 'static>,
130    /// #     publisher: Arc<impl Transport<MyEvent> + Send + Sync + 'static>,
131    /// # ) -> Result<(), OutboxError>
132    /// # where MyEvent: std::fmt::Debug + Clone + serde::Serialize + Send + Sync + 'static,
133    /// # {
134    /// let (shutdown_tx, shutdown_rx) = watch::channel(false);
135    /// let manager = OutboxManagerBuilder::new()
136    ///     .storage(storage)
137    ///     .publisher(publisher)
138    ///     .config(Arc::new(OutboxConfig::default()))
139    ///     .shutdown_rx(shutdown_rx)
140    ///     .build()?;
141    ///
142    /// let handle = tokio::spawn(async move { manager.run().await });
143    ///
144    /// // ... later, on a signal or process exit:
145    /// let _ = shutdown_tx.send(true);
146    /// handle.await.expect("worker panicked")?;
147    /// # Ok(()) }
148    /// ```
149    pub async fn run(self) -> Result<(), OutboxError> {
150        let storage_for_listen = self.storage.clone();
151        let processor = OutboxProcessor::new(
152            self.storage.clone(),
153            self.publisher.clone(),
154            self.config.clone(),
155        );
156
157        let gc = GarbageCollector::new(self.storage.clone());
158        let mut rx_gc = self.shutdown_rx.clone();
159        let gc_interval_secs = self.config.gc_interval_secs;
160        tokio::spawn(async move {
161            gc.run(Duration::from_secs(gc_interval_secs), &mut rx_gc)
162                .await
163        });
164
165        #[cfg(feature = "dlq")]
166        {
167            let dlq_processor = DlqProcessor::new(
168                self.dlq_heap.clone(),
169                self.storage.clone(),
170                self.config.clone(),
171                self.shutdown_rx.clone(),
172            );
173            tokio::spawn(async move { dlq_processor.run().await });
174        }
175
176        let mut rx_listen = self.shutdown_rx.clone();
177        let poll_interval = self.config.poll_interval_secs;
178        let mut interval = tokio::time::interval(Duration::from_secs(poll_interval));
179
180        info!("Outbox worker loop started");
181
182        loop {
183            tokio::select! {
184                signal = storage_for_listen.wait_for_notification("outbox_event") => {
185                    if let Err(e) = signal {
186                        error!("Listen error: {}", e);
187                        tokio::time::sleep(Duration::from_secs(5)).await;
188                        continue;
189                    }
190                }
191                _ = interval.tick() => {
192                    trace!("Checking for stale or pending events via interval");
193                }
194                _ = rx_listen.changed() => {
195                    if rx_listen.has_changed().is_err(){
196                        break;
197                    }
198                    if *rx_listen.borrow() {
199                        break
200                    }
201                }
202            }
203            loop {
204                if *rx_listen.borrow() {
205                    return Ok(());
206                }
207                #[cfg(feature = "dlq")]
208                match processor
209                    .process_pending_events(self.dlq_heap.clone())
210                    .await
211                {
212                    Ok(0) => break,
213                    Ok(count) => debug!("Processed {} events", count),
214                    Err(e) => {
215                        error!("Processing error: {}", e);
216                        tokio::time::sleep(Duration::from_secs(5)).await;
217                        break;
218                    }
219                }
220                #[cfg(not(feature = "dlq"))]
221                match processor.process_pending_events().await {
222                    Ok(0) => break,
223                    Ok(count) => debug!("Processed {} events", count),
224                    Err(e) => {
225                        error!("Processing error: {}", e);
226                        tokio::time::sleep(Duration::from_secs(5)).await;
227                        break;
228                    }
229                }
230            }
231        }
232        debug!("Outbox worker loop stopped");
233        Ok(())
234    }
235}
236
237#[cfg(test)]
238#[allow(clippy::unwrap_used)]
239mod tests {
240    use crate::builder::OutboxManagerBuilder;
241    use crate::config::{IdempotencyStrategy, OutboxConfig};
242    #[cfg(feature = "dlq")]
243    use crate::dlq::storage::MockDlqHeap;
244    use crate::error::OutboxError;
245    use crate::model::{Event, EventStatus};
246    use crate::object::EventType;
247    use crate::prelude::Payload;
248    use crate::publisher::MockTransport;
249    use crate::storage::MockOutboxStorage;
250    use mockall::Sequence;
251    use rstest::rstest;
252    use serde::{Deserialize, Serialize};
253    use std::sync::Arc;
254    use tokio::sync::watch;
255
256    #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
257    enum SomeDomainEvent {
258        SomeEvent(String),
259    }
260
261    #[rstest]
262    #[tokio::test]
263    async fn test_event_send_success() {
264        let config = OutboxConfig {
265            batch_size: 100,
266            retention_days: 7,
267            gc_interval_secs: 3600,
268            poll_interval_secs: 5,
269            lock_timeout_mins: 5,
270            idempotency_strategy: IdempotencyStrategy::None,
271            dlq_threshold: 10,
272            dlq_interval_secs: 1,
273        };
274
275        let mut storage_mock = MockOutboxStorage::<SomeDomainEvent>::new();
276        let mut transport_mock = MockTransport::<SomeDomainEvent>::new();
277
278        #[cfg(feature = "dlq")]
279        let mut dlq_heap_mock: MockDlqHeap = MockDlqHeap::new();
280
281        #[cfg(feature = "dlq")]
282        dlq_heap_mock.expect_record_success().returning(|_| Ok(()));
283
284        let (shutdown_tx, shutdown_rx) = watch::channel(false);
285
286        storage_mock
287            .expect_wait_for_notification()
288            .returning(|_| Ok(()));
289
290        storage_mock
291            .expect_fetch_next_to_process()
292            .withf(move |l| l == &config.batch_size)
293            .times(1)
294            .returning(move |_| {
295                let _ = shutdown_tx.send(true);
296                Ok(vec![
297                    Event::new(
298                        EventType::new("1"),
299                        Payload::new(SomeDomainEvent::SomeEvent("test1".to_string())),
300                        None,
301                    ),
302                    Event::new(
303                        EventType::new("2"),
304                        Payload::new(SomeDomainEvent::SomeEvent("test2".to_string())),
305                        None,
306                    ),
307                    Event::new(
308                        EventType::new("3"),
309                        Payload::new(SomeDomainEvent::SomeEvent("test3".to_string())),
310                        None,
311                    ),
312                    Event::new(
313                        EventType::new("4"),
314                        Payload::new(SomeDomainEvent::SomeEvent("test4".to_string())),
315                        None,
316                    ),
317                ])
318            });
319
320        storage_mock
321            .expect_fetch_next_to_process()
322            .withf(move |l| l == &config.batch_size)
323            .returning(move |_| Ok(vec![]));
324
325        storage_mock
326            .expect_update_status()
327            .withf(|ids, s| ids.len() == 4 && s == &EventStatus::Sent)
328            .returning(|_, _| Ok(()));
329
330        storage_mock.expect_delete_garbage().returning(|| Ok(()));
331
332        let mut seq = Sequence::new();
333
334        for i in 1..=4 {
335            let expected_type = i.to_string();
336            let expected_val = SomeDomainEvent::SomeEvent(format!("test{}", i));
337
338            transport_mock
339                .expect_publish()
340                .withf(move |event| {
341                    let type_matches = event.event_type.as_str() == expected_type;
342                    let payload_matches = event.payload.as_value() == &expected_val;
343                    type_matches && payload_matches
344                })
345                .times(1)
346                .in_sequence(&mut seq)
347                .returning(|_| Ok(()));
348        }
349
350        #[cfg(feature = "dlq")]
351        let manager = OutboxManagerBuilder::new()
352            .storage(Arc::new(storage_mock))
353            .publisher(Arc::new(transport_mock))
354            .config(Arc::new(config))
355            .shutdown_rx(shutdown_rx)
356            .dlq_heap(Arc::new(dlq_heap_mock))
357            .build()
358            .unwrap();
359
360        #[cfg(not(feature = "dlq"))]
361        let manager = OutboxManagerBuilder::new()
362            .storage(Arc::new(storage_mock))
363            .publisher(Arc::new(transport_mock))
364            .config(Arc::new(config))
365            .shutdown_rx(shutdown_rx)
366            .build()
367            .unwrap();
368
369        let handle = tokio::spawn(async move {
370            manager.run().await.unwrap();
371        });
372
373        tokio::time::timeout(tokio::time::Duration::from_secs(1), handle)
374            .await
375            .expect("Manager did not stop in time")
376            .unwrap();
377    }
378
379    #[rstest]
380    #[tokio::test]
381    async fn test_recovery_after_storage_error() {
382        let mut storage_mock = MockOutboxStorage::<SomeDomainEvent>::new();
383        let (shutdown_tx, shutdown_rx) = watch::channel(false);
384
385        storage_mock
386            .expect_wait_for_notification()
387            .times(1)
388            .returning(|_| Err(OutboxError::InfrastructureError("Connection lost".into())));
389
390        storage_mock
391            .expect_wait_for_notification()
392            .times(1)
393            .returning(move |_| {
394                let _ = shutdown_tx.send(true);
395                Ok(())
396            });
397
398        storage_mock
399            .expect_fetch_next_to_process()
400            .returning(|_| Ok(vec![]));
401
402        storage_mock
403            .expect_delete_garbage()
404            .times(1)
405            .returning(|| Ok(()));
406
407        let transport_mock = MockTransport::<SomeDomainEvent>::new();
408
409        let config = OutboxConfig::<SomeDomainEvent> {
410            batch_size: 100,
411            retention_days: 7,
412            gc_interval_secs: 3600,
413            poll_interval_secs: 5,
414            lock_timeout_mins: 5,
415            idempotency_strategy: IdempotencyStrategy::None,
416            dlq_threshold: 10,
417            dlq_interval_secs: 1,
418        };
419
420        #[cfg(feature = "dlq")]
421        let manager = OutboxManagerBuilder::new()
422            .storage(Arc::new(storage_mock))
423            .publisher(Arc::new(transport_mock))
424            .config(Arc::new(config))
425            .dlq_heap(Arc::new(MockDlqHeap::new()))
426            .shutdown_rx(shutdown_rx)
427            .build()
428            .unwrap();
429
430        #[cfg(not(feature = "dlq"))]
431        let manager = OutboxManagerBuilder::new()
432            .storage(Arc::new(storage_mock))
433            .publisher(Arc::new(transport_mock))
434            .config(Arc::new(config))
435            .shutdown_rx(shutdown_rx)
436            .build()
437            .unwrap();
438
439        let result = manager.run().await;
440        assert!(result.is_ok());
441    }
442
443    #[rstest]
444    #[tokio::test]
445    async fn test_publish_failure() {
446        let mut storage_mock = MockOutboxStorage::<SomeDomainEvent>::new();
447        let (shutdown_tx, shutdown_rx) = watch::channel(false);
448
449        let e1 = Event::new(
450            EventType::new("1"),
451            Payload::new(SomeDomainEvent::SomeEvent("test1".to_string())),
452            None,
453        );
454        let e2 = Event::new(
455            EventType::new("2"),
456            Payload::new(SomeDomainEvent::SomeEvent("test2".to_string())),
457            None,
458        );
459        let e3 = Event::new(
460            EventType::new("3"),
461            Payload::new(SomeDomainEvent::SomeEvent("test3".to_string())),
462            None,
463        );
464        let e4 = Event::new(
465            EventType::new("4"),
466            Payload::new(SomeDomainEvent::SomeEvent("test4".to_string())),
467            None,
468        );
469
470        let id1 = e1.id.clone();
471        let id2 = e2.id.clone();
472        let id3 = e3.id.clone();
473        let id4 = e4.id.clone();
474
475        storage_mock
476            .expect_wait_for_notification()
477            .returning(|_| Ok(()));
478
479        storage_mock
480            .expect_fetch_next_to_process()
481            .times(1)
482            .returning(move |_| Ok(vec![e1.clone(), e2.clone(), e3.clone(), e4.clone()]));
483
484        storage_mock
485            .expect_fetch_next_to_process()
486            .returning(|_| Ok(vec![]));
487
488        storage_mock.expect_delete_garbage().returning(|| Ok(()));
489
490        storage_mock
491            .expect_update_status()
492            .withf(move |ids, status| {
493                if status != &EventStatus::Sent {
494                    return false;
495                }
496
497                let ids_set: std::collections::HashSet<_> = ids.iter().cloned().collect();
498
499                ids_set.len() == 3
500                    && ids_set.contains(&id1)
501                    && ids_set.contains(&id2)
502                    && ids_set.contains(&id4)
503                    && !ids_set.contains(&id3)
504            })
505            .returning(move |_, _| {
506                let _ = shutdown_tx.send(true);
507                Ok(())
508            });
509
510        let mut transport_mock = MockTransport::<SomeDomainEvent>::new();
511
512        let mut seq = Sequence::new();
513
514        transport_mock
515            .expect_publish()
516            .times(1)
517            .in_sequence(&mut seq)
518            .returning(|_| Ok(()));
519
520        transport_mock
521            .expect_publish()
522            .times(1)
523            .in_sequence(&mut seq)
524            .returning(|_| Ok(()));
525
526        transport_mock
527            .expect_publish()
528            .times(1)
529            .in_sequence(&mut seq)
530            .returning(|_| Err(OutboxError::InfrastructureError("Connection lost".into())));
531
532        transport_mock
533            .expect_publish()
534            .times(1)
535            .in_sequence(&mut seq)
536            .returning(|_| Ok(()));
537
538        let config = OutboxConfig::<SomeDomainEvent> {
539            batch_size: 100,
540            retention_days: 7,
541            gc_interval_secs: 3600,
542            poll_interval_secs: 5,
543            lock_timeout_mins: 5,
544            idempotency_strategy: IdempotencyStrategy::None,
545            dlq_threshold: 10,
546            dlq_interval_secs: 1,
547        };
548
549        #[cfg(feature = "dlq")]
550        let dlq_heap_mock = {
551            let mut m = MockDlqHeap::new();
552            m.expect_record_success().returning(|_| Ok(()));
553            m.expect_record_failure().returning(|_| Ok(()));
554            m
555        };
556
557        #[cfg(feature = "dlq")]
558        let manager = OutboxManagerBuilder::new()
559            .storage(Arc::new(storage_mock))
560            .publisher(Arc::new(transport_mock))
561            .config(Arc::new(config))
562            .dlq_heap(Arc::new(dlq_heap_mock))
563            .shutdown_rx(shutdown_rx)
564            .build()
565            .unwrap();
566
567        #[cfg(not(feature = "dlq"))]
568        let manager = OutboxManagerBuilder::new()
569            .storage(Arc::new(storage_mock))
570            .publisher(Arc::new(transport_mock))
571            .config(Arc::new(config))
572            .shutdown_rx(shutdown_rx)
573            .build()
574            .unwrap();
575
576        let result = manager.run().await;
577        assert!(result.is_ok());
578    }
579
580    fn default_config() -> OutboxConfig<SomeDomainEvent> {
581        OutboxConfig {
582            batch_size: 100,
583            retention_days: 7,
584            gc_interval_secs: 3600,
585            poll_interval_secs: 5,
586            lock_timeout_mins: 5,
587            idempotency_strategy: IdempotencyStrategy::None,
588            dlq_threshold: 10,
589            dlq_interval_secs: 1,
590        }
591    }
592
593    #[rstest]
594    #[tokio::test]
595    async fn shutdown_set_before_run_exits_without_fetch() {
596        let mut storage_mock = MockOutboxStorage::<SomeDomainEvent>::new();
597        let transport_mock = MockTransport::<SomeDomainEvent>::new();
598        let (shutdown_tx, shutdown_rx) = watch::channel(false);
599        let _ = shutdown_tx.send(true);
600
601        storage_mock
602            .expect_wait_for_notification()
603            .returning(|_| Ok(()));
604        storage_mock.expect_delete_garbage().returning(|| Ok(()));
605        storage_mock.expect_fetch_next_to_process().times(0);
606        storage_mock.expect_update_status().times(0);
607
608        #[cfg(feature = "dlq")]
609        let manager = OutboxManagerBuilder::new()
610            .storage(Arc::new(storage_mock))
611            .publisher(Arc::new(transport_mock))
612            .config(Arc::new(default_config()))
613            .dlq_heap(Arc::new(MockDlqHeap::new()))
614            .shutdown_rx(shutdown_rx)
615            .build()
616            .unwrap();
617        #[cfg(not(feature = "dlq"))]
618        let manager = OutboxManagerBuilder::new()
619            .storage(Arc::new(storage_mock))
620            .publisher(Arc::new(transport_mock))
621            .config(Arc::new(default_config()))
622            .shutdown_rx(shutdown_rx)
623            .build()
624            .unwrap();
625
626        let result = tokio::time::timeout(tokio::time::Duration::from_secs(1), manager.run())
627            .await
628            .expect("manager did not stop in time");
629        assert!(result.is_ok());
630        drop(shutdown_tx);
631    }
632
633    #[rstest]
634    #[tokio::test]
635    async fn inner_loop_drains_until_empty_batch() {
636        let mut storage_mock = MockOutboxStorage::<SomeDomainEvent>::new();
637        let mut transport_mock = MockTransport::<SomeDomainEvent>::new();
638        let (shutdown_tx, shutdown_rx) = watch::channel(false);
639
640        storage_mock
641            .expect_wait_for_notification()
642            .returning(|_| Ok(()));
643        storage_mock.expect_delete_garbage().returning(|| Ok(()));
644
645        let mut seq = Sequence::new();
646        storage_mock
647            .expect_fetch_next_to_process()
648            .times(1)
649            .in_sequence(&mut seq)
650            .returning(|_| {
651                Ok(vec![
652                    Event::new(
653                        EventType::new("a1"),
654                        Payload::new(SomeDomainEvent::SomeEvent("a1".into())),
655                        None,
656                    ),
657                    Event::new(
658                        EventType::new("a2"),
659                        Payload::new(SomeDomainEvent::SomeEvent("a2".into())),
660                        None,
661                    ),
662                ])
663            });
664        storage_mock
665            .expect_fetch_next_to_process()
666            .times(1)
667            .in_sequence(&mut seq)
668            .returning(|_| {
669                Ok(vec![Event::new(
670                    EventType::new("b1"),
671                    Payload::new(SomeDomainEvent::SomeEvent("b1".into())),
672                    None,
673                )])
674            });
675        storage_mock
676            .expect_fetch_next_to_process()
677            .times(1)
678            .in_sequence(&mut seq)
679            .returning(move |_| {
680                let _ = shutdown_tx.send(true);
681                Ok(vec![])
682            });
683        storage_mock
684            .expect_fetch_next_to_process()
685            .returning(|_| Ok(vec![]));
686
687        storage_mock
688            .expect_update_status()
689            .times(2)
690            .returning(|_, _| Ok(()));
691
692        transport_mock
693            .expect_publish()
694            .times(3)
695            .returning(|_| Ok(()));
696
697        #[cfg(feature = "dlq")]
698        let manager = {
699            let mut dlq = MockDlqHeap::new();
700            dlq.expect_record_success().returning(|_| Ok(()));
701            dlq.expect_record_failure().returning(|_| Ok(()));
702            OutboxManagerBuilder::new()
703                .storage(Arc::new(storage_mock))
704                .publisher(Arc::new(transport_mock))
705                .config(Arc::new(default_config()))
706                .dlq_heap(Arc::new(dlq))
707                .shutdown_rx(shutdown_rx)
708                .build()
709                .unwrap()
710        };
711        #[cfg(not(feature = "dlq"))]
712        let manager = OutboxManagerBuilder::new()
713            .storage(Arc::new(storage_mock))
714            .publisher(Arc::new(transport_mock))
715            .config(Arc::new(default_config()))
716            .shutdown_rx(shutdown_rx)
717            .build()
718            .unwrap();
719
720        let handle = tokio::spawn(async move { manager.run().await.unwrap() });
721        tokio::time::timeout(tokio::time::Duration::from_secs(1), handle)
722            .await
723            .expect("manager did not stop in time")
724            .unwrap();
725    }
726
727    #[rstest]
728    #[tokio::test(start_paused = true)]
729    async fn fetch_error_inside_loop_is_recoverable() {
730        let mut storage_mock = MockOutboxStorage::<SomeDomainEvent>::new();
731        let transport_mock = MockTransport::<SomeDomainEvent>::new();
732        let (shutdown_tx, shutdown_rx) = watch::channel(false);
733
734        storage_mock
735            .expect_wait_for_notification()
736            .returning(|_| Ok(()));
737        storage_mock.expect_delete_garbage().returning(|| Ok(()));
738
739        let mut seq = Sequence::new();
740        storage_mock
741            .expect_fetch_next_to_process()
742            .times(1)
743            .in_sequence(&mut seq)
744            .returning(|_| Err(OutboxError::DatabaseError("transient".into())));
745        storage_mock
746            .expect_fetch_next_to_process()
747            .in_sequence(&mut seq)
748            .returning(move |_| {
749                let _ = shutdown_tx.send(true);
750                Ok(vec![])
751            });
752        storage_mock
753            .expect_fetch_next_to_process()
754            .returning(|_| Ok(vec![]));
755
756        storage_mock.expect_update_status().times(0);
757
758        #[cfg(feature = "dlq")]
759        let manager = OutboxManagerBuilder::new()
760            .storage(Arc::new(storage_mock))
761            .publisher(Arc::new(transport_mock))
762            .config(Arc::new(default_config()))
763            .dlq_heap(Arc::new(MockDlqHeap::new()))
764            .shutdown_rx(shutdown_rx)
765            .build()
766            .unwrap();
767        #[cfg(not(feature = "dlq"))]
768        let manager = OutboxManagerBuilder::new()
769            .storage(Arc::new(storage_mock))
770            .publisher(Arc::new(transport_mock))
771            .config(Arc::new(default_config()))
772            .shutdown_rx(shutdown_rx)
773            .build()
774            .unwrap();
775
776        let result = tokio::time::timeout(tokio::time::Duration::from_secs(30), manager.run())
777            .await
778            .expect("manager did not stop in time");
779        assert!(result.is_ok());
780    }
781}