1use crate::config::OutboxConfig;
10#[cfg(feature = "dlq")]
11use crate::dlq::processor::DlqProcessor;
12use crate::error::OutboxError;
13use crate::gc::GarbageCollector;
14use crate::processor::OutboxProcessor;
15use crate::publisher::Transport;
16use crate::storage::OutboxStorage;
17use serde::Serialize;
18use std::fmt::Debug;
19use std::sync::Arc;
20use std::time::Duration;
21use tokio::sync::watch::Receiver;
22use tracing::{debug, error, info, trace};
23
24pub struct OutboxManager<S, P, PT>
36where
37 PT: Debug + Clone + Serialize,
38{
39 storage: Arc<S>,
40 publisher: Arc<P>,
41 config: Arc<OutboxConfig<PT>>,
42 shutdown_rx: Receiver<bool>,
43 #[cfg(feature = "dlq")]
44 dlq_heap: Arc<dyn crate::dlq::storage::DlqHeap>,
45}
46
47impl<S, P, PT> OutboxManager<S, P, PT>
48where
49 S: OutboxStorage<PT> + Send + Sync + 'static,
50 P: Transport<PT> + Send + Sync + 'static,
51 PT: Debug + Clone + Serialize + Send + Sync + 'static,
52{
53 #[cfg(feature = "dlq")]
60 pub(crate) fn new(
61 storage: Arc<S>,
62 publisher: Arc<P>,
63 config: Arc<OutboxConfig<PT>>,
64 dlq_heap: Arc<dyn crate::dlq::storage::DlqHeap>,
65 shutdown_rx: Receiver<bool>,
66 ) -> Self {
67 Self {
68 storage,
69 publisher,
70 config,
71 shutdown_rx,
72 dlq_heap,
73 }
74 }
75
76 #[cfg(not(feature = "dlq"))]
81 pub(crate) fn new(
82 storage: Arc<S>,
83 publisher: Arc<P>,
84 config: Arc<OutboxConfig<PT>>,
85 shutdown_rx: Receiver<bool>,
86 ) -> Self {
87 Self {
88 storage,
89 publisher,
90 config,
91 shutdown_rx,
92 }
93 }
94
95 pub async fn run(self) -> Result<(), OutboxError> {
150 let storage_for_listen = self.storage.clone();
151 let processor = OutboxProcessor::new(
152 self.storage.clone(),
153 self.publisher.clone(),
154 self.config.clone(),
155 );
156
157 let gc = GarbageCollector::new(self.storage.clone());
158 let mut rx_gc = self.shutdown_rx.clone();
159 let gc_interval_secs = self.config.gc_interval_secs;
160 tokio::spawn(async move {
161 gc.run(Duration::from_secs(gc_interval_secs), &mut rx_gc)
162 .await
163 });
164
165 #[cfg(feature = "dlq")]
166 {
167 let dlq_processor = DlqProcessor::new(
168 self.dlq_heap.clone(),
169 self.storage.clone(),
170 self.config.clone(),
171 self.shutdown_rx.clone(),
172 );
173 tokio::spawn(async move { dlq_processor.run().await });
174 }
175
176 let mut rx_listen = self.shutdown_rx.clone();
177 let poll_interval = self.config.poll_interval_secs;
178 let mut interval = tokio::time::interval(Duration::from_secs(poll_interval));
179
180 info!("Outbox worker loop started");
181
182 loop {
183 tokio::select! {
184 signal = storage_for_listen.wait_for_notification("outbox_event") => {
185 if let Err(e) = signal {
186 error!("Listen error: {}", e);
187 tokio::time::sleep(Duration::from_secs(5)).await;
188 continue;
189 }
190 }
191 _ = interval.tick() => {
192 trace!("Checking for stale or pending events via interval");
193 }
194 _ = rx_listen.changed() => {
195 if rx_listen.has_changed().is_err(){
196 break;
197 }
198 if *rx_listen.borrow() {
199 break
200 }
201 }
202 }
203 loop {
204 if *rx_listen.borrow() {
205 return Ok(());
206 }
207 #[cfg(feature = "dlq")]
208 match processor
209 .process_pending_events(self.dlq_heap.clone())
210 .await
211 {
212 Ok(0) => break,
213 Ok(count) => debug!("Processed {} events", count),
214 Err(e) => {
215 error!("Processing error: {}", e);
216 tokio::time::sleep(Duration::from_secs(5)).await;
217 break;
218 }
219 }
220 #[cfg(not(feature = "dlq"))]
221 match processor.process_pending_events().await {
222 Ok(0) => break,
223 Ok(count) => debug!("Processed {} events", count),
224 Err(e) => {
225 error!("Processing error: {}", e);
226 tokio::time::sleep(Duration::from_secs(5)).await;
227 break;
228 }
229 }
230 }
231 }
232 debug!("Outbox worker loop stopped");
233 Ok(())
234 }
235}
236
237#[cfg(test)]
238#[allow(clippy::unwrap_used)]
239mod tests {
240 use crate::builder::OutboxManagerBuilder;
241 use crate::config::{IdempotencyStrategy, OutboxConfig};
242 #[cfg(feature = "dlq")]
243 use crate::dlq::storage::MockDlqHeap;
244 use crate::error::OutboxError;
245 use crate::model::{Event, EventStatus};
246 use crate::object::EventType;
247 use crate::prelude::Payload;
248 use crate::publisher::MockTransport;
249 use crate::storage::MockOutboxStorage;
250 use mockall::Sequence;
251 use rstest::rstest;
252 use serde::{Deserialize, Serialize};
253 use std::sync::Arc;
254 use tokio::sync::watch;
255
256 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
257 enum SomeDomainEvent {
258 SomeEvent(String),
259 }
260
261 #[rstest]
262 #[tokio::test]
263 async fn test_event_send_success() {
264 let config = OutboxConfig {
265 batch_size: 100,
266 retention_days: 7,
267 gc_interval_secs: 3600,
268 poll_interval_secs: 5,
269 lock_timeout_mins: 5,
270 idempotency_strategy: IdempotencyStrategy::None,
271 dlq_threshold: 10,
272 dlq_interval_secs: 1,
273 };
274
275 let mut storage_mock = MockOutboxStorage::<SomeDomainEvent>::new();
276 let mut transport_mock = MockTransport::<SomeDomainEvent>::new();
277
278 #[cfg(feature = "dlq")]
279 let mut dlq_heap_mock: MockDlqHeap = MockDlqHeap::new();
280
281 #[cfg(feature = "dlq")]
282 dlq_heap_mock.expect_record_success().returning(|_| Ok(()));
283
284 let (shutdown_tx, shutdown_rx) = watch::channel(false);
285
286 storage_mock
287 .expect_wait_for_notification()
288 .returning(|_| Ok(()));
289
290 storage_mock
291 .expect_fetch_next_to_process()
292 .withf(move |l| l == &config.batch_size)
293 .times(1)
294 .returning(move |_| {
295 let _ = shutdown_tx.send(true);
296 Ok(vec![
297 Event::new(
298 EventType::new("1"),
299 Payload::new(SomeDomainEvent::SomeEvent("test1".to_string())),
300 None,
301 ),
302 Event::new(
303 EventType::new("2"),
304 Payload::new(SomeDomainEvent::SomeEvent("test2".to_string())),
305 None,
306 ),
307 Event::new(
308 EventType::new("3"),
309 Payload::new(SomeDomainEvent::SomeEvent("test3".to_string())),
310 None,
311 ),
312 Event::new(
313 EventType::new("4"),
314 Payload::new(SomeDomainEvent::SomeEvent("test4".to_string())),
315 None,
316 ),
317 ])
318 });
319
320 storage_mock
321 .expect_fetch_next_to_process()
322 .withf(move |l| l == &config.batch_size)
323 .returning(move |_| Ok(vec![]));
324
325 storage_mock
326 .expect_update_status()
327 .withf(|ids, s| ids.len() == 4 && s == &EventStatus::Sent)
328 .returning(|_, _| Ok(()));
329
330 storage_mock.expect_delete_garbage().returning(|| Ok(()));
331
332 let mut seq = Sequence::new();
333
334 for i in 1..=4 {
335 let expected_type = i.to_string();
336 let expected_val = SomeDomainEvent::SomeEvent(format!("test{}", i));
337
338 transport_mock
339 .expect_publish()
340 .withf(move |event| {
341 let type_matches = event.event_type.as_str() == expected_type;
342 let payload_matches = event.payload.as_value() == &expected_val;
343 type_matches && payload_matches
344 })
345 .times(1)
346 .in_sequence(&mut seq)
347 .returning(|_| Ok(()));
348 }
349
350 #[cfg(feature = "dlq")]
351 let manager = OutboxManagerBuilder::new()
352 .storage(Arc::new(storage_mock))
353 .publisher(Arc::new(transport_mock))
354 .config(Arc::new(config))
355 .shutdown_rx(shutdown_rx)
356 .dlq_heap(Arc::new(dlq_heap_mock))
357 .build()
358 .unwrap();
359
360 #[cfg(not(feature = "dlq"))]
361 let manager = OutboxManagerBuilder::new()
362 .storage(Arc::new(storage_mock))
363 .publisher(Arc::new(transport_mock))
364 .config(Arc::new(config))
365 .shutdown_rx(shutdown_rx)
366 .build()
367 .unwrap();
368
369 let handle = tokio::spawn(async move {
370 manager.run().await.unwrap();
371 });
372
373 tokio::time::timeout(tokio::time::Duration::from_secs(1), handle)
374 .await
375 .expect("Manager did not stop in time")
376 .unwrap();
377 }
378
379 #[rstest]
380 #[tokio::test]
381 async fn test_recovery_after_storage_error() {
382 let mut storage_mock = MockOutboxStorage::<SomeDomainEvent>::new();
383 let (shutdown_tx, shutdown_rx) = watch::channel(false);
384
385 storage_mock
386 .expect_wait_for_notification()
387 .times(1)
388 .returning(|_| Err(OutboxError::InfrastructureError("Connection lost".into())));
389
390 storage_mock
391 .expect_wait_for_notification()
392 .times(1)
393 .returning(move |_| {
394 let _ = shutdown_tx.send(true);
395 Ok(())
396 });
397
398 storage_mock
399 .expect_fetch_next_to_process()
400 .returning(|_| Ok(vec![]));
401
402 storage_mock
403 .expect_delete_garbage()
404 .times(1)
405 .returning(|| Ok(()));
406
407 let transport_mock = MockTransport::<SomeDomainEvent>::new();
408
409 let config = OutboxConfig::<SomeDomainEvent> {
410 batch_size: 100,
411 retention_days: 7,
412 gc_interval_secs: 3600,
413 poll_interval_secs: 5,
414 lock_timeout_mins: 5,
415 idempotency_strategy: IdempotencyStrategy::None,
416 dlq_threshold: 10,
417 dlq_interval_secs: 1,
418 };
419
420 #[cfg(feature = "dlq")]
421 let manager = OutboxManagerBuilder::new()
422 .storage(Arc::new(storage_mock))
423 .publisher(Arc::new(transport_mock))
424 .config(Arc::new(config))
425 .dlq_heap(Arc::new(MockDlqHeap::new()))
426 .shutdown_rx(shutdown_rx)
427 .build()
428 .unwrap();
429
430 #[cfg(not(feature = "dlq"))]
431 let manager = OutboxManagerBuilder::new()
432 .storage(Arc::new(storage_mock))
433 .publisher(Arc::new(transport_mock))
434 .config(Arc::new(config))
435 .shutdown_rx(shutdown_rx)
436 .build()
437 .unwrap();
438
439 let result = manager.run().await;
440 assert!(result.is_ok());
441 }
442
443 #[rstest]
444 #[tokio::test]
445 async fn test_publish_failure() {
446 let mut storage_mock = MockOutboxStorage::<SomeDomainEvent>::new();
447 let (shutdown_tx, shutdown_rx) = watch::channel(false);
448
449 let e1 = Event::new(
450 EventType::new("1"),
451 Payload::new(SomeDomainEvent::SomeEvent("test1".to_string())),
452 None,
453 );
454 let e2 = Event::new(
455 EventType::new("2"),
456 Payload::new(SomeDomainEvent::SomeEvent("test2".to_string())),
457 None,
458 );
459 let e3 = Event::new(
460 EventType::new("3"),
461 Payload::new(SomeDomainEvent::SomeEvent("test3".to_string())),
462 None,
463 );
464 let e4 = Event::new(
465 EventType::new("4"),
466 Payload::new(SomeDomainEvent::SomeEvent("test4".to_string())),
467 None,
468 );
469
470 let id1 = e1.id.clone();
471 let id2 = e2.id.clone();
472 let id3 = e3.id.clone();
473 let id4 = e4.id.clone();
474
475 storage_mock
476 .expect_wait_for_notification()
477 .returning(|_| Ok(()));
478
479 storage_mock
480 .expect_fetch_next_to_process()
481 .times(1)
482 .returning(move |_| Ok(vec![e1.clone(), e2.clone(), e3.clone(), e4.clone()]));
483
484 storage_mock
485 .expect_fetch_next_to_process()
486 .returning(|_| Ok(vec![]));
487
488 storage_mock.expect_delete_garbage().returning(|| Ok(()));
489
490 storage_mock
491 .expect_update_status()
492 .withf(move |ids, status| {
493 if status != &EventStatus::Sent {
494 return false;
495 }
496
497 let ids_set: std::collections::HashSet<_> = ids.iter().cloned().collect();
498
499 ids_set.len() == 3
500 && ids_set.contains(&id1)
501 && ids_set.contains(&id2)
502 && ids_set.contains(&id4)
503 && !ids_set.contains(&id3)
504 })
505 .returning(move |_, _| {
506 let _ = shutdown_tx.send(true);
507 Ok(())
508 });
509
510 let mut transport_mock = MockTransport::<SomeDomainEvent>::new();
511
512 let mut seq = Sequence::new();
513
514 transport_mock
515 .expect_publish()
516 .times(1)
517 .in_sequence(&mut seq)
518 .returning(|_| Ok(()));
519
520 transport_mock
521 .expect_publish()
522 .times(1)
523 .in_sequence(&mut seq)
524 .returning(|_| Ok(()));
525
526 transport_mock
527 .expect_publish()
528 .times(1)
529 .in_sequence(&mut seq)
530 .returning(|_| Err(OutboxError::InfrastructureError("Connection lost".into())));
531
532 transport_mock
533 .expect_publish()
534 .times(1)
535 .in_sequence(&mut seq)
536 .returning(|_| Ok(()));
537
538 let config = OutboxConfig::<SomeDomainEvent> {
539 batch_size: 100,
540 retention_days: 7,
541 gc_interval_secs: 3600,
542 poll_interval_secs: 5,
543 lock_timeout_mins: 5,
544 idempotency_strategy: IdempotencyStrategy::None,
545 dlq_threshold: 10,
546 dlq_interval_secs: 1,
547 };
548
549 #[cfg(feature = "dlq")]
550 let dlq_heap_mock = {
551 let mut m = MockDlqHeap::new();
552 m.expect_record_success().returning(|_| Ok(()));
553 m.expect_record_failure().returning(|_| Ok(()));
554 m
555 };
556
557 #[cfg(feature = "dlq")]
558 let manager = OutboxManagerBuilder::new()
559 .storage(Arc::new(storage_mock))
560 .publisher(Arc::new(transport_mock))
561 .config(Arc::new(config))
562 .dlq_heap(Arc::new(dlq_heap_mock))
563 .shutdown_rx(shutdown_rx)
564 .build()
565 .unwrap();
566
567 #[cfg(not(feature = "dlq"))]
568 let manager = OutboxManagerBuilder::new()
569 .storage(Arc::new(storage_mock))
570 .publisher(Arc::new(transport_mock))
571 .config(Arc::new(config))
572 .shutdown_rx(shutdown_rx)
573 .build()
574 .unwrap();
575
576 let result = manager.run().await;
577 assert!(result.is_ok());
578 }
579
580 fn default_config() -> OutboxConfig<SomeDomainEvent> {
581 OutboxConfig {
582 batch_size: 100,
583 retention_days: 7,
584 gc_interval_secs: 3600,
585 poll_interval_secs: 5,
586 lock_timeout_mins: 5,
587 idempotency_strategy: IdempotencyStrategy::None,
588 dlq_threshold: 10,
589 dlq_interval_secs: 1,
590 }
591 }
592
593 #[rstest]
594 #[tokio::test]
595 async fn shutdown_set_before_run_exits_without_fetch() {
596 let mut storage_mock = MockOutboxStorage::<SomeDomainEvent>::new();
597 let transport_mock = MockTransport::<SomeDomainEvent>::new();
598 let (shutdown_tx, shutdown_rx) = watch::channel(false);
599 let _ = shutdown_tx.send(true);
600
601 storage_mock
602 .expect_wait_for_notification()
603 .returning(|_| Ok(()));
604 storage_mock.expect_delete_garbage().returning(|| Ok(()));
605 storage_mock.expect_fetch_next_to_process().times(0);
606 storage_mock.expect_update_status().times(0);
607
608 #[cfg(feature = "dlq")]
609 let manager = OutboxManagerBuilder::new()
610 .storage(Arc::new(storage_mock))
611 .publisher(Arc::new(transport_mock))
612 .config(Arc::new(default_config()))
613 .dlq_heap(Arc::new(MockDlqHeap::new()))
614 .shutdown_rx(shutdown_rx)
615 .build()
616 .unwrap();
617 #[cfg(not(feature = "dlq"))]
618 let manager = OutboxManagerBuilder::new()
619 .storage(Arc::new(storage_mock))
620 .publisher(Arc::new(transport_mock))
621 .config(Arc::new(default_config()))
622 .shutdown_rx(shutdown_rx)
623 .build()
624 .unwrap();
625
626 let result = tokio::time::timeout(tokio::time::Duration::from_secs(1), manager.run())
627 .await
628 .expect("manager did not stop in time");
629 assert!(result.is_ok());
630 drop(shutdown_tx);
631 }
632
633 #[rstest]
634 #[tokio::test]
635 async fn inner_loop_drains_until_empty_batch() {
636 let mut storage_mock = MockOutboxStorage::<SomeDomainEvent>::new();
637 let mut transport_mock = MockTransport::<SomeDomainEvent>::new();
638 let (shutdown_tx, shutdown_rx) = watch::channel(false);
639
640 storage_mock
641 .expect_wait_for_notification()
642 .returning(|_| Ok(()));
643 storage_mock.expect_delete_garbage().returning(|| Ok(()));
644
645 let mut seq = Sequence::new();
646 storage_mock
647 .expect_fetch_next_to_process()
648 .times(1)
649 .in_sequence(&mut seq)
650 .returning(|_| {
651 Ok(vec![
652 Event::new(
653 EventType::new("a1"),
654 Payload::new(SomeDomainEvent::SomeEvent("a1".into())),
655 None,
656 ),
657 Event::new(
658 EventType::new("a2"),
659 Payload::new(SomeDomainEvent::SomeEvent("a2".into())),
660 None,
661 ),
662 ])
663 });
664 storage_mock
665 .expect_fetch_next_to_process()
666 .times(1)
667 .in_sequence(&mut seq)
668 .returning(|_| {
669 Ok(vec![Event::new(
670 EventType::new("b1"),
671 Payload::new(SomeDomainEvent::SomeEvent("b1".into())),
672 None,
673 )])
674 });
675 storage_mock
676 .expect_fetch_next_to_process()
677 .times(1)
678 .in_sequence(&mut seq)
679 .returning(move |_| {
680 let _ = shutdown_tx.send(true);
681 Ok(vec![])
682 });
683 storage_mock
684 .expect_fetch_next_to_process()
685 .returning(|_| Ok(vec![]));
686
687 storage_mock
688 .expect_update_status()
689 .times(2)
690 .returning(|_, _| Ok(()));
691
692 transport_mock
693 .expect_publish()
694 .times(3)
695 .returning(|_| Ok(()));
696
697 #[cfg(feature = "dlq")]
698 let manager = {
699 let mut dlq = MockDlqHeap::new();
700 dlq.expect_record_success().returning(|_| Ok(()));
701 dlq.expect_record_failure().returning(|_| Ok(()));
702 OutboxManagerBuilder::new()
703 .storage(Arc::new(storage_mock))
704 .publisher(Arc::new(transport_mock))
705 .config(Arc::new(default_config()))
706 .dlq_heap(Arc::new(dlq))
707 .shutdown_rx(shutdown_rx)
708 .build()
709 .unwrap()
710 };
711 #[cfg(not(feature = "dlq"))]
712 let manager = OutboxManagerBuilder::new()
713 .storage(Arc::new(storage_mock))
714 .publisher(Arc::new(transport_mock))
715 .config(Arc::new(default_config()))
716 .shutdown_rx(shutdown_rx)
717 .build()
718 .unwrap();
719
720 let handle = tokio::spawn(async move { manager.run().await.unwrap() });
721 tokio::time::timeout(tokio::time::Duration::from_secs(1), handle)
722 .await
723 .expect("manager did not stop in time")
724 .unwrap();
725 }
726
727 #[rstest]
728 #[tokio::test(start_paused = true)]
729 async fn fetch_error_inside_loop_is_recoverable() {
730 let mut storage_mock = MockOutboxStorage::<SomeDomainEvent>::new();
731 let transport_mock = MockTransport::<SomeDomainEvent>::new();
732 let (shutdown_tx, shutdown_rx) = watch::channel(false);
733
734 storage_mock
735 .expect_wait_for_notification()
736 .returning(|_| Ok(()));
737 storage_mock.expect_delete_garbage().returning(|| Ok(()));
738
739 let mut seq = Sequence::new();
740 storage_mock
741 .expect_fetch_next_to_process()
742 .times(1)
743 .in_sequence(&mut seq)
744 .returning(|_| Err(OutboxError::DatabaseError("transient".into())));
745 storage_mock
746 .expect_fetch_next_to_process()
747 .in_sequence(&mut seq)
748 .returning(move |_| {
749 let _ = shutdown_tx.send(true);
750 Ok(vec![])
751 });
752 storage_mock
753 .expect_fetch_next_to_process()
754 .returning(|_| Ok(vec![]));
755
756 storage_mock.expect_update_status().times(0);
757
758 #[cfg(feature = "dlq")]
759 let manager = OutboxManagerBuilder::new()
760 .storage(Arc::new(storage_mock))
761 .publisher(Arc::new(transport_mock))
762 .config(Arc::new(default_config()))
763 .dlq_heap(Arc::new(MockDlqHeap::new()))
764 .shutdown_rx(shutdown_rx)
765 .build()
766 .unwrap();
767 #[cfg(not(feature = "dlq"))]
768 let manager = OutboxManagerBuilder::new()
769 .storage(Arc::new(storage_mock))
770 .publisher(Arc::new(transport_mock))
771 .config(Arc::new(default_config()))
772 .shutdown_rx(shutdown_rx)
773 .build()
774 .unwrap();
775
776 let result = tokio::time::timeout(tokio::time::Duration::from_secs(30), manager.run())
777 .await
778 .expect("manager did not stop in time");
779 assert!(result.is_ok());
780 }
781}