Skip to main content

mako_engine/
builder.rs

1//! [`EngineModule`] trait, [`EngineBuilder`], and [`EngineContext`].
2//!
3// Allow using deprecated Noop stores as *type-level defaults* in EngineBuilder / EngineContext
4// generic parameters.  The types are deprecated to prevent instantiation in production code,
5// but using them as default type parameters in struct definitions (not instantiating them) is
6// the intended pattern for the type-state builder API.
7#![allow(deprecated)]
8//!
9//! # Summary
10//!
11//! `EngineBuilder` assembles all engine infrastructure into a single
12//! [`EngineContext`] value. Domain modules (GPKE, WiM, GeLi Gas, …) register
13//! themselves at startup via the [`EngineModule`] trait, making their names
14//! visible in diagnostics and health checks.
15//!
16//! # Type-state guarantee
17//!
18//! [`EngineBuilder::build`] is only available when the event store type
19//! parameter `ES` implements [`EventStore`]. Forgetting to call
20//! [`with_event_store`] is a **compile-time error**, not a runtime panic.
21//!
22//! All other stores default to their respective `Noop` implementations:
23//!
24//! | Store | Default |
25//! |-------|---------|
26//! | Snapshot store | [`NoopSnapshotStore`] |
27//! | Outbox store | [`NoopOutboxStore`] |
28//! | Deadline store | [`NoopDeadlineStore`] |
29//! | Process registry | [`NoopProcessRegistry`] |
30//!
31//! # Assembly example
32//!
33//! ```rust,ignore
34//! use mako_engine::builder::{EngineBuilder, EngineModule};
35//! use mako_engine::event_store::InMemoryEventStore;
36//! use mako_engine::outbox::InMemoryOutboxStore;
37//! use mako_engine::deadline::InMemoryDeadlineStore;
38//! use mako_engine::registry::InMemoryProcessRegistry;
39//! use mako_engine::snapshot::InMemorySnapshotStore;
40//!
41//! struct GpkeModule;
42//! impl EngineModule for GpkeModule { fn name(&self) -> &'static str { "gpke" } }
43//!
44//! let ctx = EngineBuilder::new()
45//!     .with_event_store(InMemoryEventStore::new())
46//!     .with_snapshot_store(InMemorySnapshotStore::new())
47//!     .with_outbox_store(InMemoryOutboxStore::new())
48//!     .with_deadline_store(InMemoryDeadlineStore::new())
49//!     .with_registry(InMemoryProcessRegistry::new())
50//!     .register(Box::new(GpkeModule))
51//!     .build();
52//!
53//! // Spawn a fresh process:
54//! let p = ctx.spawn::<SupplierChangeWorkflow>(tenant_id, workflow_id);
55//! p.execute(ReceiveUtilmd { .. }).await?;
56//!
57//! // Resume an existing process from a persisted identity:
58//! let identity = ctx.registry.lookup(&conv_id.to_string()).await?.unwrap();
59//! let p = ctx.resume::<SupplierChangeWorkflow>(identity);
60//!
61//! // Access stores for delivery workers / schedulers:
62//! let pending = ctx.outbox_store.pending_now(50).await?;
63//! let overdue = ctx.deadline_store.due_now(50).await?;
64//! ```
65//!
66//! [`with_event_store`]: EngineBuilder::with_event_store
67
68// Type-state generics can produce long signatures that trip up the
69// `type_complexity` lint; suppress it for this module only.
70#![allow(clippy::type_complexity)]
71
72// The Noop* types are marked #[deprecated] to guard against accidental
73// production use.  The builder is the only place they're instantiated as
74// defaults; suppress the lint here explicitly.
75#[allow(deprecated)]
76use crate::{
77    dead_letter::{DeadLetterSink, LogDeadLetterSink},
78    deadline::{Deadline, DeadlineStore, NoopDeadlineStore},
79    error::EngineError,
80    event_store::EventStore,
81    ids::{ProcessIdentity, TenantId},
82    marktrolle::DeploymentRoles,
83    outbox::{NoopOutboxStore, OutboxMessage, OutboxStore},
84    pid_router::PidRouter,
85    process::Process,
86    registry::{NoopProcessRegistry, ProcessRegistry},
87    snapshot::{NoopSnapshotStore, SnapshotStore},
88    version::WorkflowId,
89    workflow::Workflow,
90};
91
92use std::sync::Arc;
93
94// ── EngineModule ──────────────────────────────────────────────────────────────
95
96/// A self-contained domain module that registers with the engine at startup.
97///
98/// Domain crates implement this trait to declare their presence in the engine.
99/// The module name is surfaced in [`EngineContext::registered_modules`] for
100/// diagnostics, health checks, and log output.
101///
102/// ## Startup validation
103///
104/// Override [`configure`] to perform adapter coverage checks at engine startup
105/// time. The engine calls [`configure`] for every registered module during
106/// [`EngineBuilder::build`] and panics with an actionable message if any
107/// module returns `Err`. This surfaces missing adapter registrations as a
108/// startup failure rather than a silent runtime error.
109///
110/// ## Example
111///
112/// ```rust,ignore
113/// pub struct GpkeModule;
114///
115/// impl EngineModule for GpkeModule {
116///     fn name(&self) -> &'static str { "gpke" }
117///
118///     fn configure(&self) -> Result<(), String> {
119///         // Validate that every known BDEW format version has an adapter:
120///         GPKE_ADAPTER_REGISTRY
121///             .validate_policy(&GpkeWorkflow::version_policy(), &KNOWN_FVS)
122///             .map_err(|uncovered| format!(
123///                 "gpke: missing adapters for format versions: {:?}",
124///                 uncovered
125///             ))
126///     }
127/// }
128///
129/// let ctx = EngineBuilder::new()
130///     .with_event_store(my_store)
131///     .register(Box::new(GpkeModule))
132///     .build(); // panics if GpkeModule::configure returns Err
133///
134/// assert_eq!(ctx.registered_modules(), &["gpke"]);
135/// ```
136///
137/// [`configure`]: EngineModule::configure
138pub trait EngineModule: Send + 'static {
139    /// Stable, unique name for this domain module.
140    ///
141    /// Used in diagnostics, health checks, and structured log output.
142    /// Choose a short lowercase identifier (e.g. `"gpke"`, `"wim"`,
143    /// `"geli"`).
144    fn name(&self) -> &'static str;
145
146    /// Register all PIDs this module handles into the shared [`PidRouter`].
147    ///
148    /// # Mutability contract
149    ///
150    /// This method is called **exactly once** by [`EngineBuilder::build`],
151    /// before the resulting [`EngineContext`] is handed to the caller. The
152    /// `&mut PidRouter` reference is only available here, at build time.
153    /// After `build` returns the router is **sealed** — the engine provides
154    /// only a shared `&PidRouter` reference, with no mutation path at runtime.
155    ///
156    /// Consequence: **all PIDs a module will ever need must be registered
157    /// here**. Do not attempt to register PIDs lazily from async handlers or
158    /// after the engine has started — there is no API for that by design.
159    ///
160    /// Duplicate registrations (same PID from two modules) silently overwrite
161    /// the previous mapping; the last module to register wins. Use
162    /// `cargo xtask validate-pruefids` to catch accidental PID conflicts
163    /// between modules before they reach production.
164    ///
165    /// For role-conditional registration (PIDs that should only be active for
166    /// specific BDEW Marktrollen), override [`register_pids_with_roles`] instead.
167    ///
168    /// # Example
169    ///
170    /// ```rust,ignore
171    /// fn register_pids(&self, router: &mut PidRouter) {
172    ///     // GPKE Lieferantenwechsel / Lieferbeginn (BK6-22-024, PIDs 55001, 55002, 55017)
173    ///     for &pid in &[55001_u32, 55002, 55017] {
174    ///         router.register(pid, "gpke-supplier-change");
175    ///     }
176    /// }
177    /// ```
178    ///
179    /// [`register_pids_with_roles`]: EngineModule::register_pids_with_roles
180    fn register_pids(&self, _router: &mut PidRouter) {}
181
182    /// Register PIDs with role-context awareness.
183    ///
184    /// This is the **preferred override** for modules that have role-conditional
185    /// PID registrations — PIDs that should only be active when this `makod`
186    /// instance holds a specific [`Marktrolle`].
187    ///
188    /// The default implementation calls [`register_pids`] (role-agnostic) so
189    /// existing modules that override `register_pids` continue to work without
190    /// changes.
191    ///
192    /// Override this method instead of `register_pids` when any PID registration
193    /// should be conditional on the deployment role:
194    ///
195    /// ```rust,ignore
196    /// use mako_engine::marktrolle::Marktrolle;
197    ///
198    /// fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
199    ///     // Always register: 55001, 55002 (not role-specific)
200    ///     for pid in [55001_u32, 55002] { router.register_with_module(pid, "gpke-supplier-change", self.name()); }
201    ///
202    ///     // Only when NB role: 19001/19002 inbound ORDRSP from MSB
203    ///     if roles.contains(Marktrolle::Nb) {
204    ///         for pid in [19001_u32, 19002] { router.register_with_module(pid, "gpke-konfiguration", self.name()); }
205    ///     }
206    /// }
207    /// ```
208    ///
209    /// # Conflict guard
210    ///
211    /// Use [`PidRouter::register_with_module`] (not `register`) inside this
212    /// method. The conflict guard panics at build time if two modules register
213    /// the same PID to different workflows — this makes role misconfigurations
214    /// visible at startup rather than silently misrouting messages.
215    ///
216    /// [`Marktrolle`]: crate::marktrolle::Marktrolle
217    /// [`register_pids`]: EngineModule::register_pids
218    fn register_pids_with_roles(&self, router: &mut PidRouter, _roles: &DeploymentRoles) {
219        self.register_pids(router);
220    }
221
222    /// Workflow names this module handles for deadline dispatch.
223    ///
224    /// Return the same name strings that [`register_pids`] maps PIDs to.
225    /// These names are stored in [`EngineContext::registered_workflows`] and
226    /// used to validate that every workflow that has deadlines scheduled is
227    /// covered by the deadline scheduler dispatch function at runtime.
228    ///
229    /// The default implementation returns an empty slice. Override it to
230    /// declare all workflow names that may fire deadlines:
231    ///
232    /// ```rust,ignore
233    /// fn workflow_names(&self) -> &'static [&'static str] {
234    ///     &["gpke-supplier-change", "gpke-abrechnung"]
235    /// }
236    /// ```
237    ///
238    /// [`register_pids`]: EngineModule::register_pids
239    /// [`EngineContext::registered_workflows`]: crate::builder::EngineContext::registered_workflows
240    fn workflow_names(&self) -> &'static [&'static str] {
241        &[]
242    }
243
244    /// Declare the EDIFACT profile types this module requires at runtime.
245    ///
246    /// Returning a non-empty slice causes [`EngineBuilder::build`] to call the
247    /// registered profile validator for each requirement.  If no active profile
248    /// exists for a required message type, `build` panics with an actionable
249    /// error so deployment fails fast rather than silently.
250    ///
251    /// Domain crates declare their format requirements here rather than
252    /// calling `edi_energy::registry::ReleaseRegistry::global()` inside
253    /// `configure()`, so `edi-energy` stays out of their production
254    /// `[dependencies]`.
255    ///
256    /// ```rust,ignore
257    /// fn profile_requirements(&self) -> &'static [ProfileRequirement] {
258    ///     &[
259    ///         ProfileRequirement { message_type: "UTILMD", label: "UTILMD Strom (GPKE)" },
260    ///         ProfileRequirement { message_type: "INVOIC", label: "INVOIC Abrechnung (GPKE)" },
261    ///     ]
262    /// }
263    /// ```
264    ///
265    /// [`ProfileRequirement`]: crate::profile::ProfileRequirement
266    fn profile_requirements(&self) -> &'static [crate::profile::ProfileRequirement] {
267        &[]
268    }
269
270    /// Validate adapter coverage and configuration at engine startup.
271    ///
272    /// Called by [`EngineBuilder::build`] after all modules are registered.
273    /// Return `Ok(())` when the module is fully configured. Return `Err(msg)`
274    /// with an actionable description when an adapter or configuration is
275    /// missing — the engine will panic with that message so the deployment
276    /// fails early rather than silently.
277    ///
278    /// The default implementation is a no-op (always returns `Ok(())`).
279    /// Override it in domain crates to call
280    /// [`AdapterRegistry::validate_policy`] and emit structured errors.
281    ///
282    /// Note: if your validation needs access to the edi-energy profile
283    /// registry, use [`profile_requirements`] instead — it does not require
284    /// importing `edi-energy` in domain crates.
285    ///
286    /// [`AdapterRegistry::validate_policy`]: crate::message_adapter::AdapterRegistry::validate_policy
287    /// [`profile_requirements`]: EngineModule::profile_requirements
288    ///
289    /// # Errors
290    ///
291    /// Returns a descriptive error string when the module's configuration is invalid.
292    fn configure(&self) -> Result<(), String> {
293        Ok(())
294    }
295}
296
297// ── EngineContext ─────────────────────────────────────────────────────────────
298
299/// Assembled engine infrastructure returned by [`EngineBuilder::build`].
300///
301/// `EngineContext` bundles all stores and the process registry into a single
302/// value. It is the root dependency for:
303///
304/// - Spawning new processes ([`spawn`])
305/// - Resuming existing processes ([`resume`])
306/// - Running outbox delivery workers (`outbox_store.pending_now(…)`)
307/// - Driving the deadline scheduler (`deadline_store.due_now(…)`)
308///
309/// ## Generic parameters
310///
311/// | Param | Role | Default |
312/// |-------|------|---------|
313/// | `ES`  | [`EventStore`] backend | — (required) |
314/// | `SS`  | [`SnapshotStore`] backend | [`NoopSnapshotStore`] |
315/// | `OS`  | [`OutboxStore`] backend  | [`NoopOutboxStore`]   |
316/// | `DS`  | [`DeadlineStore`] backend | [`NoopDeadlineStore`] |
317/// | `PR`  | [`ProcessRegistry`] backend | [`NoopProcessRegistry`] |
318///
319/// In most codebases all type parameters are inferred from the builder calls.
320///
321/// [`spawn`]: EngineContext::spawn
322/// [`resume`]: EngineContext::resume
323pub struct EngineContext<
324    ES,
325    SS = NoopSnapshotStore,
326    OS = NoopOutboxStore,
327    DS = NoopDeadlineStore,
328    PR = NoopProcessRegistry,
329> {
330    event_store: Arc<ES>,
331    snapshot_store: SS,
332    outbox_store: OS,
333    deadline_store: DS,
334    registry: PR,
335    /// Dead-letter sink for unroutable or unprocessable inbound messages.
336    ///
337    /// Stored as `Arc<dyn DeadLetterSink>` so callers can share it across
338    /// tasks without an extra type parameter on `EngineContext`.
339    pub dead_letter_sink: Arc<dyn DeadLetterSink>,
340    /// PID-to-workflow routing table, populated from all registered modules.
341    pid_router: PidRouter,
342    registered_modules: Vec<&'static str>,
343    /// Workflow names declared by all registered modules via
344    /// [`EngineModule::workflow_names`]. Used to validate deadline scheduler
345    /// coverage at runtime (see [`EngineContext::registered_workflows`]).
346    registered_workflows: Vec<&'static str>,
347}
348
349// ── Type aliases ──────────────────────────────────────────────────────────────
350
351/// An [`EngineContext`] with all optional subsystems disabled.
352///
353/// Uses `NoopSnapshotStore` and, in `testing`-enabled builds, Noop
354/// implementations for outbox, deadline, and process registry. Suitable for
355/// tests and minimal deployments where only a durable event store is required.
356///
357/// All five type parameters are inferred from context when used with
358/// [`EngineBuilder`]:
359///
360/// ```rust,ignore
361/// // Only available in test / testing-feature builds:
362/// use mako_engine::builder::{EngineBuilder, MinimalEngine};
363/// use mako_engine::event_store::InMemoryEventStore;
364///
365/// let ctx: MinimalEngine<InMemoryEventStore> = EngineBuilder::new()
366///     .with_event_store(InMemoryEventStore::new())
367///     .build();
368/// ```
369pub type MinimalEngine<ES> = EngineContext<ES>;
370
371impl<ES, SS, OS, DS, PR> EngineContext<ES, SS, OS, DS, PR>
372where
373    ES: EventStore,
374{
375    /// Spawn a new process and return a typed `Process<W, Arc<ES>>` handle.
376    ///
377    /// No `ES: Clone` bound is required — the engine stores the event store
378    /// behind an `Arc` so spawning is always a cheap pointer clone.
379    ///
380    /// ```rust,ignore
381    /// let p = ctx.spawn::<SupplierChangeWorkflow>(tenant_id, workflow_id);
382    /// p.execute(ReceiveUtilmd { .. }).await?;
383    /// ```
384    #[must_use]
385    pub fn spawn<W: Workflow>(
386        &self,
387        tenant_id: TenantId,
388        workflow_id: WorkflowId,
389    ) -> Process<W, Arc<ES>> {
390        Process::new(Arc::clone(&self.event_store), tenant_id, workflow_id)
391    }
392
393    /// Resume an existing process from a [`ProcessIdentity`].
394    ///
395    /// ```rust,ignore
396    /// let identity = ctx.registry()
397    ///     .lookup(tenant_id, &conv_id.to_string())
398    ///     .await?
399    ///     .ok_or(EngineError::Registry("unknown conversation".into()))?;
400    /// let p = ctx.resume::<SupplierChangeWorkflow>(identity);
401    /// p.execute(HandleAperak { .. }).await?;
402    /// ```
403    #[must_use]
404    pub fn resume<W: Workflow>(&self, identity: ProcessIdentity) -> Process<W, Arc<ES>> {
405        Process::from_identity(Arc::clone(&self.event_store), identity)
406    }
407
408    /// Names of all domain modules registered with the builder, in
409    /// registration order.
410    #[must_use]
411    pub fn registered_modules(&self) -> &[&'static str] {
412        &self.registered_modules
413    }
414
415    /// Workflow names declared by all registered modules, in registration order.
416    ///
417    /// Use this in the deadline scheduler dispatch function to detect unknown
418    /// workflow names at startup. If a deadline fires for a workflow name that
419    /// is not in this list, the scheduler's dispatch function should emit an
420    /// error rather than silently dropping the deadline:
421    ///
422    /// ```rust,ignore
423    /// let known = ctx.registered_workflows().iter().copied().collect::<HashSet<_>>();
424    /// let scheduler = ctx.run_deadline_scheduler(
425    ///     move |deadline| {
426    ///         let wf = deadline.workflow_id().name.as_ref();
427    ///         if !known.contains(wf) {
428    ///             tracing::error!(workflow = %wf, "deadline fired for unregistered workflow");
429    ///             return Box::pin(async { Ok(()) });
430    ///         }
431    ///         // dispatch by workflow name …
432    ///         Box::pin(async { Ok(()) })
433    ///     },
434    ///     100,
435    ///     Duration::from_secs(30),
436    /// );
437    /// ```
438    #[must_use]
439    pub fn registered_workflows(&self) -> &[&'static str] {
440        &self.registered_workflows
441    }
442
443    /// The event store backend (behind an `Arc`).
444    #[must_use]
445    pub fn event_store(&self) -> &Arc<ES> {
446        &self.event_store
447    }
448
449    /// The snapshot store backend.
450    #[must_use]
451    pub fn snapshot_store(&self) -> &SS {
452        &self.snapshot_store
453    }
454
455    /// The outbox store backend.
456    ///
457    /// Poll `outbox_store().pending_now(limit)` in a background task to drain
458    /// the delivery queue.
459    #[must_use]
460    pub fn outbox_store(&self) -> &OS {
461        &self.outbox_store
462    }
463
464    /// The deadline store backend.
465    ///
466    /// Poll `deadline_store().due_now(limit)` in a background scheduler to
467    /// fire overdue process timers.
468    #[must_use]
469    pub fn deadline_store(&self) -> &DS {
470        &self.deadline_store
471    }
472
473    /// The process routing registry.
474    ///
475    /// Register a [`ProcessIdentity`] under a `(tenant_id, key)` pair at
476    /// process creation, then `lookup` it when routing inbound messages.
477    #[must_use]
478    pub fn registry(&self) -> &PR {
479        &self.registry
480    }
481
482    /// The dead-letter sink for unroutable or unprocessable messages.
483    ///
484    /// Call [`DeadLetterSink::reject`] when an inbound message cannot be
485    /// dispatched to any workflow. The default sink emits `tracing::warn!`
486    /// so rejections are always visible in the log output.
487    #[must_use]
488    pub fn dead_letter_sink(&self) -> &Arc<dyn DeadLetterSink> {
489        &self.dead_letter_sink
490    }
491
492    /// Assert that no Noop store is active — call this during production startup.
493    ///
494    /// Checks the type names of `OS`, `DS`, and `PR` against the string `"Noop"`.
495    /// Panics with a human-readable message if any match, directing the operator
496    /// to configure a persistent backend.
497    ///
498    /// # When to call
499    ///
500    /// Call this early in `makod`'s startup path (and `--check` mode) to catch
501    /// deployments where a Noop store was accidentally wired — e.g. the
502    /// `[outbox]`, `[deadline]`, or `[registry]` configuration section was
503    /// omitted from `makod.toml`.  The check is defence-in-depth: in release
504    /// builds without the `testing` feature, Noop stores cannot implement the
505    /// required traits at all and the compiler would have already rejected them.
506    ///
507    /// # Panics
508    ///
509    /// Panics when any of `OS`, `DS`, or `PR` is a Noop implementation.
510    pub fn assert_production_stores(&self) {
511        let checks: &[(&str, &str)] = &[
512            ("OutboxStore", std::any::type_name::<OS>()),
513            ("DeadlineStore", std::any::type_name::<DS>()),
514            ("ProcessRegistry", std::any::type_name::<PR>()),
515        ];
516        for (trait_name, type_name) in checks {
517            assert!(
518                !type_name.contains("Noop"),
519                "makod: Noop{trait_name} is active — \
520                 configure a persistent {trait_name} backend in makod.toml. \
521                 Type resolved to: {type_name}"
522            );
523        }
524    }
525
526    /// The PID-to-workflow routing table.
527    ///
528    /// Populated **once** during [`EngineBuilder::build`] by calling
529    /// [`EngineModule::register_pids`] on every registered module in
530    /// registration order. After `build` returns the table is **sealed** —
531    /// it is read-only for the lifetime of the `EngineContext` and may be
532    /// freely shared across async tasks without synchronisation.
533    ///
534    /// # Mutability contract
535    ///
536    /// There is intentionally no `pid_router_mut()` accessor. Adding PIDs
537    /// after the engine is built would create a TOCTOU race between the
538    /// dispatch path (which calls `route(pid)`) and any hypothetical
539    /// concurrent mutator. Instead, register all PIDs during the build phase
540    /// via `EngineModule::register_pids`.
541    ///
542    /// If a new process family needs to be added without restarting the
543    /// binary, rebuild and restart `makod` — hot-swap of PID routing is not
544    /// supported.
545    ///
546    /// # Example — dispatch at the AS4 reception boundary
547    ///
548    /// ```rust,ignore
549    /// let workflow_name = ctx.pid_router().route(pid)
550    ///     .ok_or_else(|| EngineError::Workflow(WorkflowError::InvalidCommand(
551    ///         format!("no workflow registered for PID {pid}").into()
552    ///     )))?;
553    ///
554    /// match workflow_name {
555    ///     "gpke-supplier-change" => dispatch::<GpkeSupplierChangeWorkflow>(&ctx, pid, payload).await,
556    ///     "wim-device-change"    => dispatch::<WimDeviceChangeWorkflow>(&ctx, pid, payload).await,
557    ///     other => Err(EngineError::Workflow(WorkflowError::InvalidCommand(
558    ///         format!("unhandled workflow name: {other}").into()
559    ///     ))),
560    /// }
561    /// ```
562    #[must_use]
563    pub fn pid_router(&self) -> &PidRouter {
564        &self.pid_router
565    }
566}
567
568// ── As4Sender ─────────────────────────────────────────────────────────────────
569
570/// Sends a single AS4 / EDIINT-over-HTTP outbound message.
571///
572/// Implement this trait for your AS4 gateway client and pass it to
573/// [`EngineContext::run_outbox_worker`].
574///
575/// # Contract
576///
577/// Return `Ok(())` only after the message has been **durably accepted** by the
578/// receiving MSH.  Return `Err(…)` on transient or permanent failure — the
579/// outbox worker calls [`OutboxStore::reschedule`] so the message is retried.
580pub trait As4Sender: Send + Sync + 'static {
581    /// Transmit `msg` and return when the remote MSH has accepted it.
582    fn send(
583        &self,
584        msg: &OutboxMessage,
585    ) -> impl std::future::Future<Output = Result<(), EngineError>> + Send;
586}
587
588// ── OutboxWorker ──────────────────────────────────────────────────────────────
589
590/// A background worker that drains the outbox by polling pending
591/// [`OutboxMessage`]s and dispatching them via an [`As4Sender`].
592///
593/// Obtain via [`EngineContext::run_outbox_worker`] and drive by spawning
594/// [`OutboxWorker::run`] in a Tokio task.
595///
596/// # Polling behaviour
597///
598/// When the poll returns an empty batch the worker sleeps for `poll_interval`
599/// before polling again.  Non-empty batches are processed immediately.
600///
601/// # Error handling
602///
603/// Successful sends are acknowledged via [`OutboxStore::acknowledge`].
604/// Failed sends are rescheduled via [`OutboxStore::reschedule`] using
605/// **full-jitter exponential backoff**: `delay = rand(0, min(MAX, BASE * 2^n))`
606/// where `n = attempt_count`. This avoids thundering-herd when multiple
607/// `makod` instances restart simultaneously after a receiver outage.
608///
609/// When `attempt_count >= max_attempts`, the message is **acknowledged** (removed
610/// from the outbox) and a [`DeadLetterReason::OutboxExhausted`] record is written
611/// to the dead-letter sink. This prevents permanently-undeliverable messages
612/// from clogging the outbox forever.
613///
614/// All errors are emitted as structured `tracing` events at `warn` / `error`
615/// level rather than `eprintln!`, so they appear in the application's log
616/// pipeline with full context (message_id, error).
617///
618/// # Example
619///
620/// ```rust,ignore
621/// use std::time::Duration;
622///
623/// let worker = ctx.run_outbox_worker(my_sender, 50, Duration::from_secs(1));
624/// tokio::spawn(async move { worker.run().await });
625/// ```
626///
627/// [`DeadLetterReason::OutboxExhausted`]: crate::dead_letter::DeadLetterReason::OutboxExhausted
628pub struct OutboxWorker<OS: OutboxStore, S: As4Sender, DS: DeadlineStore> {
629    store: OS,
630    sender: S,
631    /// Used to discharge a delivery-window deadline once the message it was
632    /// watching has actually been sent — see [`OutboxWorker::run`].
633    deadline_store: DS,
634    batch_size: usize,
635    poll_interval: std::time::Duration,
636    /// Maximum total delivery attempts before a message is dead-lettered — a
637    /// runaway belt, not the budget. The budget is [`Self::max_retry_window`]:
638    /// the backoff is full-jitter, so an attempt *count* cannot promise a
639    /// retry *duration*, and the BDEW retry duty is stated in hours.
640    max_attempts: u32,
641    /// Maximum age (from `created_at`) a message is retried for before it is
642    /// dead-lettered. This is what honours a time-stated retry duty (BDEW AS4
643    /// Kommunikationshandbuch: 72 h for unacknowledged messages) — see
644    /// `mako_as4::constants::MAX_RETRY_DURATION_SECS`.
645    ///
646    /// Checked only after at least one attempt: a message that aged in a
647    /// stopped worker still gets its first try rather than being buried
648    /// unsent.
649    max_retry_window: std::time::Duration,
650    /// Sink for messages that exceed `max_attempts` or `max_retry_window`.
651    dead_letter_sink: std::sync::Arc<dyn crate::dead_letter::DeadLetterSink>,
652    /// Optional liveness heartbeat — stores the current UTC Unix timestamp
653    /// (seconds) after each poll cycle so health probes can detect stale workers.
654    heartbeat: Option<std::sync::Arc<std::sync::atomic::AtomicI64>>,
655    /// Graceful-shutdown signal. When cancelled the worker finishes the message
656    /// it is delivering, then returns from [`OutboxWorker::run`] — see
657    /// [`OutboxWorker::with_shutdown`].
658    shutdown: Option<tokio_util::sync::CancellationToken>,
659}
660
661/// Sleep for `dur`, returning early if `token` is cancelled.
662///
663/// Returns `true` when the sleep completed and the caller should keep looping,
664/// `false` when the token was cancelled and the caller must return.
665///
666/// A worker that sleeps on a bare `tokio::time::sleep` cannot observe a
667/// shutdown until its poll interval elapses. For the deadline scheduler that is
668/// 30 seconds by default — longer than a typical container termination grace
669/// period, which turns a graceful drain into a SIGKILL.
670///
671/// Public so that binaries running their own poll-loop workers alongside the
672/// engine's (projection catch-up, webhook delivery, retention purges) can honour
673/// the same token and stop before the store is closed.
674pub async fn sleep_or_cancel(
675    dur: std::time::Duration,
676    token: Option<&tokio_util::sync::CancellationToken>,
677) -> bool {
678    let Some(t) = token else {
679        tokio::time::sleep(dur).await;
680        return true;
681    };
682    tokio::select! {
683        () = tokio::time::sleep(dur) => true,
684        () = t.cancelled() => false,
685    }
686}
687
688/// Compute a full-jitter exponential backoff delay.
689///
690/// `attempt` is the number of prior attempts (0 = first retry).
691/// `entropy` provides randomness; derive from a stable message identifier
692/// (e.g. hash of `message_id`) rather than the current timestamp — a
693/// timestamp-derived value is deterministic within a single batch, which
694/// defeats jitter when multiple messages fail simultaneously.
695///
696/// | attempt | window (s) | expected delay (s) |
697/// |---------|------------|-------------------|
698/// | 0       | 5          | 2.5               |
699/// | 1       | 10         | 5                 |
700/// | 2       | 20         | 10                |
701/// | 3       | 40         | 20                |
702/// | 4       | 80         | 40                |
703/// | 5+      | 300 (cap)  | 150               |
704fn backoff_delay(attempt: u32, entropy: u64) -> std::time::Duration {
705    const BASE_SECS: u64 = 5;
706    const MAX_SECS: u64 = 300;
707    // Exponential window: BASE * 2^attempt, capped at MAX.
708    let window = BASE_SECS
709        .saturating_mul(1u64.wrapping_shl(attempt.min(5)))
710        .min(MAX_SECS);
711    // Full jitter: uniform random in [0, window).
712    let jitter_secs = if window == 0 { 0 } else { entropy % window };
713    std::time::Duration::from_secs(jitter_secs)
714}
715
716impl<OS: OutboxStore, S: As4Sender, DS: DeadlineStore> OutboxWorker<OS, S, DS> {
717    /// Run the outbox drain loop until the shutdown token is cancelled.
718    ///
719    /// Without a token (see [`OutboxWorker::with_shutdown`]) the loop runs until
720    /// the task is aborted or the process exits. With one, cancellation is
721    /// observed between messages and during the idle sleep, so an in-flight
722    /// delivery is always finished and acknowledged before the worker returns —
723    /// dropping it mid-`send` would risk a duplicate AS4 delivery on restart.
724    ///
725    /// # Panics
726    ///
727    /// Panics if `time::Duration::try_from(delay)` overflows (unreachable for
728    /// the delay values produced by `backoff_delay`).
729    #[allow(clippy::too_many_lines)]
730    pub async fn run(self) {
731        loop {
732            if self
733                .shutdown
734                .as_ref()
735                .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
736            {
737                tracing::info!("outbox worker: shutdown signalled; stopping");
738                return;
739            }
740            // Tick liveness at the *start* of every poll cycle, ahead of the
741            // early-`continue` paths below.  An idle worker (empty outbox) and
742            // one retrying after a store error are both alive and must keep
743            // ticking; only a worker genuinely hung inside an `.await` stops.
744            if let Some(ref hb) = self.heartbeat {
745                hb.store(
746                    time::OffsetDateTime::now_utc().unix_timestamp(),
747                    std::sync::atomic::Ordering::Relaxed,
748                );
749            }
750
751            let batch = match self.store.pending_now(self.batch_size).await {
752                Ok(b) => b,
753                Err(e) => {
754                    tracing::warn!(error = %e, "outbox worker: store error polling pending messages (will retry)");
755                    if !sleep_or_cancel(self.poll_interval, self.shutdown.as_ref()).await {
756                        return;
757                    }
758                    continue;
759                }
760            };
761
762            if batch.is_empty() {
763                if !sleep_or_cancel(self.poll_interval, self.shutdown.as_ref()).await {
764                    return;
765                }
766                continue;
767            }
768
769            for msg in batch {
770                // Between messages, not inside one: a `send` that is already in
771                // flight must run to its `acknowledge`, or the counterparty
772                // receives a message the outbox still believes is pending and
773                // redelivers it after the restart.
774                if self
775                    .shutdown
776                    .as_ref()
777                    .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
778                {
779                    tracing::info!(
780                        "outbox worker: shutdown signalled mid-batch; \
781                         remaining messages stay queued for the next start"
782                    );
783                    return;
784                }
785                // ── Retry budget ──────────────────────────────────────
786                // `attempt_count` starts at 0 and is incremented on each
787                // `reschedule` call. The message is permanently undeliverable
788                // when the retry *window* has elapsed (the BDEW duty is stated
789                // in hours, and full-jitter backoff makes a count no proxy for
790                // a duration) or the attempt belt is exhausted: acknowledge it
791                // (remove from outbox) and dead-letter it so the regulatory
792                // audit trail is preserved. The window is only consulted after
793                // a first attempt, so a message that aged while the worker was
794                // down still gets tried once.
795                let age = time::OffsetDateTime::now_utc() - msg.created_at;
796                let window_elapsed = msg.attempt_count > 0
797                    && age
798                        >= time::Duration::try_from(self.max_retry_window)
799                            .unwrap_or(time::Duration::hours(72));
800                if msg.attempt_count >= self.max_attempts || window_elapsed {
801                    tracing::error!(
802                        message_id   = %msg.message_id,
803                        message_type = %msg.message_type,
804                        recipient    = %msg.recipient,
805                        attempts     = msg.attempt_count,
806                        max_attempts = self.max_attempts,
807                        age_secs     = age.whole_seconds(),
808                        window_secs  = self.max_retry_window.as_secs(),
809                        "outbox worker: retry budget exhausted; dead-lettering message",
810                    );
811                    self.dead_letter_sink.reject(
812                        &crate::dead_letter::DeadLetterReason::OutboxExhausted {
813                            message_id: msg.message_id,
814                            message_type: msg.message_type.to_string(),
815                            recipient: msg.recipient.to_string(),
816                            last_error: format!(
817                                "delivery exhausted after {} attempts",
818                                msg.attempt_count
819                            ),
820                            attempts: msg.attempt_count,
821                        },
822                    );
823                    if let Err(e) = self.store.acknowledge(msg.message_id).await {
824                        tracing::error!(
825                            message_id = %msg.message_id,
826                            error = %e,
827                            "outbox worker: acknowledge after exhaust failed; message may reappear",
828                        );
829                    }
830                    continue;
831                }
832
833                match self.sender.send(&msg).await {
834                    Ok(()) => {
835                        if let Err(e) = self.store.acknowledge(msg.message_id).await {
836                            tracing::warn!(
837                                message_id = %msg.message_id,
838                                error = %e,
839                                "outbox worker: acknowledge failed",
840                            );
841                        }
842                        // CONTRL AHB 1.0 §1.2: the CONTRL must be delivered
843                        // within 6 wall-clock hours of interchange receipt.
844                        // `msg.created_at` is when the PendingOutbox was
845                        // materialised (which should equal the ingest timestamp
846                        // for transport-layer CONTRL obligations).
847                        if msg.message_type.as_ref() == "CONTRL" {
848                            let elapsed = time::OffsetDateTime::now_utc() - msg.created_at;
849                            if elapsed > time::Duration::hours(mako_fristen::CONTRL_FRIST_HOURS) {
850                                tracing::warn!(
851                                    message_id   = %msg.message_id,
852                                    elapsed_secs = elapsed.whole_seconds(),
853                                    max_secs     = mako_fristen::CONTRL_FRIST_HOURS * 3600,
854                                    "outbox worker: CONTRL delivered OUTSIDE the 6h Übertragungsfrist \
855                                     (CONTRL AHB 1.0 §1.2) — this is a BNetzA compliance violation"
856                                );
857                            }
858                        }
859                        // APERAK AHB 1.0 §2.4.1: Strom UTILMD/ORDERS APERAK must be
860                        // delivered within 45 minutes on weekdays, or by Sunday 12:00
861                        // if received on Saturday.  Log a compliance warning if the
862                        // delivery window was missed so operators can investigate.
863                        if msg.message_type.as_ref() == "APERAK" {
864                            let elapsed = time::OffsetDateTime::now_utc() - msg.created_at;
865                            if elapsed
866                                > time::Duration::minutes(
867                                    mako_fristen::APERAK_STROM_WEEKDAY_MINUTES,
868                                )
869                            {
870                                tracing::warn!(
871                                    message_id   = %msg.message_id,
872                                    elapsed_mins = elapsed.whole_minutes(),
873                                    "outbox worker: APERAK delivered after the 45-minute Strom \
874                                     sending window (APERAK AHB 1.0 §2.4.1) — \
875                                     check OutboxWorker and AS4 transport health"
876                                );
877                            }
878                        }
879                        // The message is out, so any delivery window that was
880                        // watching for it has been answered — retire it.
881                        //
882                        // Nothing else cancels these. A monitoring deadline that
883                        // outlives the obligation it monitors fires for every
884                        // process, including every one that answered on time,
885                        // and the scheduler cannot tell those apart because a
886                        // deadline reaching `due_now` is late by construction.
887                        // Leaving them registered turns the miss counters into
888                        // counts of *processes started*.
889                        self.discharge_delivery_window(&msg).await;
890                    }
891                    // Permanent error: dead-letter immediately without retrying.
892                    // PartnerUnknown requires operator intervention (add --as4-partner);
893                    // Serialization errors will never succeed on retry; a missing
894                    // wire-format renderer cannot appear between attempts — its own
895                    // documentation promises immediate dead-lettering, and until this
896                    // arm matched it, that promise was broken and the message burned
897                    // the whole retry budget first.
898                    Err(ref e)
899                        if e.is_partner_unknown()
900                            || e.is_renderer_not_implemented()
901                            || matches!(e, EngineError::Serialization(_)) =>
902                    {
903                        tracing::error!(
904                            message_id   = %msg.message_id,
905                            message_type = %msg.message_type,
906                            recipient    = %msg.recipient,
907                            error        = %e,
908                            "outbox worker: permanent send failure; dead-lettering without retry",
909                        );
910                        self.dead_letter_sink.reject(
911                            &crate::dead_letter::DeadLetterReason::OutboxExhausted {
912                                message_id: msg.message_id,
913                                message_type: msg.message_type.to_string(),
914                                recipient: msg.recipient.to_string(),
915                                last_error: e.to_string(),
916                                attempts: msg.attempt_count,
917                            },
918                        );
919                        if let Err(re) = self.store.acknowledge(msg.message_id).await {
920                            tracing::error!(
921                                message_id = %msg.message_id,
922                                error = %re,
923                                "outbox worker: acknowledge after permanent failure failed",
924                            );
925                        }
926                    }
927                    Err(e) => {
928                        // Stable jitter entropy derived from the UUID bytes of
929                        // `message_id`.  Using the last 8 bytes as a `u64` gives
930                        // uniform entropy across message IDs (UUIDs are random in
931                        // all 128 bits for v4) and is stable across Rust versions —
932                        // unlike `DefaultHasher`, whose algorithm is explicitly
933                        // documented as unstable.
934                        let entropy = {
935                            let uuid = msg.message_id.as_uuid();
936                            let bytes = uuid.as_bytes();
937                            u64::from_le_bytes(bytes[8..16].try_into().unwrap())
938                        };
939                        let delay = backoff_delay(msg.attempt_count, entropy);
940                        let retry_at = time::OffsetDateTime::now_utc()
941                            + time::Duration::try_from(delay).unwrap_or(time::Duration::minutes(5));
942                        tracing::warn!(
943                            message_id   = %msg.message_id,
944                            attempt      = msg.attempt_count,
945                            max_attempts = self.max_attempts,
946                            retry_in     = ?delay,
947                            error        = %e,
948                            "outbox worker: send failed; rescheduling with backoff",
949                        );
950                        if let Err(re) = self.store.reschedule(msg.message_id, retry_at).await {
951                            tracing::error!(
952                                message_id = %msg.message_id,
953                                error      = %re,
954                                "outbox worker: reschedule failed; message may be stuck",
955                            );
956                        }
957                    }
958                }
959            }
960        }
961    }
962}
963
964impl<ES, SS, OS, DS, PR> EngineContext<ES, SS, OS, DS, PR>
965where
966    ES: EventStore,
967    OS: OutboxStore + Clone,
968{
969    /// Construct an [`OutboxWorker`] that drains the outbox via `sender`.
970    ///
971    /// `batch_size` — messages fetched per poll cycle.
972    /// `poll_interval` — sleep duration when the batch is empty.
973    ///
974    /// `max_attempts` — attempt belt against runaway loops; the real budget is
975    /// `max_retry_window`, the message age after which delivery is abandoned.
976    /// The BDEW AS4 retry duty is stated in *hours* (72 h for unacknowledged
977    /// messages — `mako_as4::constants::MAX_RETRY_DURATION_SECS`), and the
978    /// full-jitter backoff makes an attempt count no proxy for a duration, so
979    /// both are taken and either exhausts the message.
980    ///
981    /// ```rust,ignore
982    /// use std::time::Duration;
983    ///
984    /// let worker = ctx.run_outbox_worker(
985    ///     my_sender, 50, Duration::from_secs(1),
986    ///     10_000, Duration::from_secs(72 * 3600),
987    /// );
988    /// tokio::spawn(async move { worker.run().await });
989    /// ```
990    #[must_use]
991    pub fn run_outbox_worker<S: As4Sender>(
992        &self,
993        sender: S,
994        batch_size: usize,
995        poll_interval: std::time::Duration,
996        max_attempts: u32,
997        max_retry_window: std::time::Duration,
998    ) -> OutboxWorker<OS, S, DS>
999    where
1000        DS: DeadlineStore + Clone,
1001    {
1002        OutboxWorker {
1003            store: self.outbox_store.clone(),
1004            sender,
1005            deadline_store: self.deadline_store.clone(),
1006            batch_size,
1007            poll_interval,
1008            max_attempts,
1009            max_retry_window,
1010            dead_letter_sink: self.dead_letter_sink.clone(),
1011            heartbeat: None,
1012            shutdown: None,
1013        }
1014    }
1015}
1016
1017impl<OS: OutboxStore, S: As4Sender, DS: DeadlineStore> OutboxWorker<OS, S, DS> {
1018    /// Attach a liveness heartbeat to this worker.
1019    ///
1020    /// The worker will store the current UTC Unix timestamp (seconds) into
1021    /// `heartbeat` at the end of every poll cycle.  Pass the same
1022    /// `Arc<AtomicI64>` to the health endpoint so it can detect stale workers.
1023    #[must_use]
1024    pub fn with_heartbeat(
1025        mut self,
1026        heartbeat: std::sync::Arc<std::sync::atomic::AtomicI64>,
1027    ) -> Self {
1028        self.heartbeat = Some(heartbeat);
1029        self
1030    }
1031
1032    /// Attach a graceful-shutdown token.
1033    ///
1034    /// Cancelling it makes [`OutboxWorker::run`] return at the next message
1035    /// boundary or immediately out of its idle sleep. Await the worker's
1036    /// `JoinHandle` afterwards: the point of the token is that the caller can
1037    /// close the event store *after* the worker has stopped writing to it.
1038    #[must_use]
1039    pub fn with_shutdown(mut self, shutdown: tokio_util::sync::CancellationToken) -> Self {
1040        self.shutdown = Some(shutdown);
1041        self
1042    }
1043
1044    /// Retire the delivery window `msg` was being watched by, if one is open.
1045    ///
1046    /// A delivery-window deadline exists to answer one question: *did this
1047    /// message go out in time?* Once it has gone out the question is settled,
1048    /// and leaving the deadline registered only guarantees a false alarm later.
1049    /// [`fristen::discharges_delivery_window`] decides which labels a given
1050    /// message type answers for; deadlines that merely share the stream (a
1051    /// process-response window, say) are left alone.
1052    ///
1053    /// Best-effort: a failure here costs a spurious alert at the window's close,
1054    /// never a lost or duplicated message, so it is logged rather than
1055    /// propagated — the delivery itself has already been acknowledged.
1056    ///
1057    /// [`fristen::discharges_delivery_window`]: mako_fristen::discharges_delivery_window
1058    async fn discharge_delivery_window(&self, msg: &crate::outbox::OutboxMessage) {
1059        let open = match self.deadline_store.for_stream(&msg.stream_id).await {
1060            Ok(deadlines) => deadlines,
1061            Err(e) => {
1062                tracing::warn!(
1063                    message_id   = %msg.message_id,
1064                    message_type = %msg.message_type,
1065                    error        = %e,
1066                    "outbox worker: could not read deadlines to discharge the delivery \
1067                     window; it may fire a spurious regulatory alert",
1068                );
1069                return;
1070            }
1071        };
1072
1073        for deadline in open
1074            .iter()
1075            .filter(|d| mako_fristen::discharges_delivery_window(&msg.message_type, d.label()))
1076        {
1077            if let Err(e) = self.deadline_store.cancel(deadline.deadline_id()).await {
1078                tracing::warn!(
1079                    message_id  = %msg.message_id,
1080                    deadline_id = %deadline.deadline_id(),
1081                    label       = %deadline.label(),
1082                    error       = %e,
1083                    "outbox worker: could not discharge the delivery window; \
1084                     it may fire a spurious regulatory alert",
1085                );
1086            } else {
1087                tracing::debug!(
1088                    message_id   = %msg.message_id,
1089                    message_type = %msg.message_type,
1090                    deadline_id  = %deadline.deadline_id(),
1091                    label        = %deadline.label(),
1092                    "outbox worker: message delivered — delivery window discharged",
1093                );
1094            }
1095        }
1096    }
1097}
1098
1099impl<ES, SS, OS, DS, PR> std::fmt::Debug for EngineContext<ES, SS, OS, DS, PR>
1100where
1101    ES: std::fmt::Debug,
1102    SS: std::fmt::Debug,
1103    OS: std::fmt::Debug,
1104    DS: std::fmt::Debug,
1105    PR: std::fmt::Debug,
1106{
1107    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1108        f.debug_struct("EngineContext")
1109            .field("registered_modules", &self.registered_modules)
1110            .field("registered_workflows", &self.registered_workflows)
1111            .field("pid_router_len", &self.pid_router.len())
1112            .finish_non_exhaustive()
1113    }
1114}
1115
1116// ── NoopAs4Sender / LogAs4Sender ──────────────────────────────────────────────
1117
1118/// An [`As4Sender`] that succeeds immediately without sending anything.
1119///
1120/// Use in tests and environments where outbound AS4 delivery is not yet
1121/// wired. All outbox messages are acknowledged (removed from the queue)
1122/// without being transmitted.
1123///
1124/// # ⚠️ Data loss warning
1125///
1126/// Every outbox message is **silently discarded** — no EDIFACT message is
1127/// sent to any counterparty. Do not use in production.
1128#[derive(Debug, Clone, Copy, Default)]
1129#[must_use = "NoopAs4Sender discards all outbound messages silently — use a real AS4 gateway in production"]
1130#[cfg_attr(
1131    not(any(test, feature = "testing")),
1132    deprecated = "NoopAs4Sender must not be wired in production builds; every \
1133                  outbound EDIFACT message would be silently discarded. Use \
1134                  a real As4Sender implementation instead."
1135)]
1136pub struct NoopAs4Sender;
1137
1138// The trait impl is test/testing-only: a release build without the `testing`
1139// feature cannot wire NoopAs4Sender into an outbox worker at all.
1140#[cfg(any(test, feature = "testing"))]
1141impl As4Sender for NoopAs4Sender {
1142    async fn send(&self, _msg: &OutboxMessage) -> Result<(), EngineError> {
1143        Ok(())
1144    }
1145}
1146
1147/// An [`As4Sender`] that logs every outbound message at `warn` level and
1148/// succeeds without transmitting.
1149///
1150/// Useful for development and integration-testing environments where the
1151/// full AS4 stack is not yet available but message visibility is desired.
1152/// All outbox messages are acknowledged (removed from the queue) after logging.
1153///
1154/// # ⚠️ Data loss warning
1155///
1156/// No EDIFACT message is sent to any counterparty. Do not use in production.
1157#[derive(Debug, Clone, Copy, Default)]
1158#[must_use = "LogAs4Sender discards all outbound messages — use a real AS4 gateway in production"]
1159pub struct LogAs4Sender;
1160
1161impl As4Sender for LogAs4Sender {
1162    async fn send(&self, msg: &OutboxMessage) -> Result<(), EngineError> {
1163        tracing::warn!(
1164            message_id   = %msg.message_id,
1165            message_type = %msg.message_type,
1166            recipient    = %msg.recipient,
1167            "LogAs4Sender: outbox message dropped — configure a real AS4 gateway for production",
1168        );
1169        Ok(())
1170    }
1171}
1172
1173// ── DeadlineScheduler ─────────────────────────────────────────────────────────
1174
1175/// A background task that polls [`DeadlineStore::due_now`] and dispatches
1176/// deadline commands to the owning processes via a caller-supplied function.
1177///
1178/// Obtain via [`EngineContext::run_deadline_scheduler`] and drive by spawning
1179/// [`DeadlineScheduler::run`] in a Tokio task.
1180///
1181/// # Dispatch function
1182///
1183/// The `dispatch` function receives a fired [`Deadline`] and returns a future
1184/// that dispatches the appropriate timeout command to the process. The function
1185/// is responsible for resuming the correct workflow and calling `execute`.
1186/// After the future completes, the scheduler cancels the deadline from the
1187/// store regardless of the dispatch outcome (to prevent re-firing).
1188///
1189/// ```rust,ignore
1190/// use std::time::Duration;
1191///
1192/// let scheduler = ctx.run_deadline_scheduler(
1193///     |deadline| async move {
1194///         tracing::warn!(
1195///             deadline_id = %deadline.deadline_id(),
1196///             label = %deadline.label(),
1197///             "deadline fired",
1198///         );
1199///         Ok(())
1200///     },
1201///     100,
1202///     Duration::from_secs(30),
1203/// );
1204/// tokio::spawn(async move { scheduler.run().await });
1205/// ```
1206pub struct DeadlineScheduler<DS: DeadlineStore> {
1207    store: DS,
1208    dispatch: Box<
1209        dyn Fn(
1210                Deadline,
1211            ) -> std::pin::Pin<
1212                Box<dyn std::future::Future<Output = Result<(), EngineError>> + Send>,
1213            > + Send
1214            + Sync,
1215    >,
1216    batch_size: usize,
1217    poll_interval: std::time::Duration,
1218    /// Optional liveness heartbeat — stores the current UTC Unix timestamp
1219    /// (seconds) after each poll cycle.
1220    heartbeat: Option<std::sync::Arc<std::sync::atomic::AtomicI64>>,
1221    /// Graceful-shutdown signal — see [`DeadlineScheduler::with_shutdown`].
1222    shutdown: Option<tokio_util::sync::CancellationToken>,
1223}
1224
1225impl<DS: DeadlineStore> DeadlineScheduler<DS> {
1226    /// Run the deadline poll loop until the shutdown token is cancelled.
1227    ///
1228    /// Cancellation is observed between deadlines and during the idle sleep, so
1229    /// a deadline already being dispatched runs to completion. A deadline left
1230    /// undispatched stays registered and fires on the next start — it is due, so
1231    /// the next `due_now` returns it again.
1232    pub async fn run(self) {
1233        loop {
1234            if self
1235                .shutdown
1236                .as_ref()
1237                .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
1238            {
1239                tracing::info!("deadline scheduler: shutdown signalled; stopping");
1240                return;
1241            }
1242            // Tick liveness at the *start* of every poll cycle, ahead of the
1243            // early-`continue` paths below.  An idle scheduler (no due
1244            // deadlines) is alive and must keep ticking; only one genuinely
1245            // hung inside an `.await` stops.
1246            if let Some(ref hb) = self.heartbeat {
1247                hb.store(
1248                    time::OffsetDateTime::now_utc().unix_timestamp(),
1249                    std::sync::atomic::Ordering::Relaxed,
1250                );
1251            }
1252
1253            let result = match self.store.due_now(self.batch_size).await {
1254                Ok(r) => r,
1255                Err(e) => {
1256                    tracing::warn!(
1257                        error = %e,
1258                        "deadline scheduler: store error polling due deadlines (will retry)",
1259                    );
1260                    if !sleep_or_cancel(self.poll_interval, self.shutdown.as_ref()).await {
1261                        return;
1262                    }
1263                    continue;
1264                }
1265            };
1266
1267            if result.deadlines.is_empty() {
1268                if !sleep_or_cancel(self.poll_interval, self.shutdown.as_ref()).await {
1269                    return;
1270                }
1271                continue;
1272            }
1273
1274            for deadline in result.deadlines {
1275                // Between deadlines, not inside one: a dispatch already running
1276                // must finish so its events and outbox entries commit together.
1277                if self
1278                    .shutdown
1279                    .as_ref()
1280                    .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
1281                {
1282                    tracing::info!(
1283                        "deadline scheduler: shutdown signalled mid-batch; \
1284                         undispatched deadlines remain due and fire on the next start"
1285                    );
1286                    return;
1287                }
1288                let id = deadline.deadline_id();
1289                let label = deadline.label().to_owned();
1290
1291                // An APERAK delivery window that reaches this point is a
1292                // regulatory violation under APERAK AHB 1.0 §2.4.1 (Strom
1293                // 45 min) / §2.3.1 (Gas 1 Werktag): the outbox worker discharges
1294                // the window the moment the APERAK goes out, so one that is
1295                // still registered when it comes due was never answered.
1296                //
1297                // Do NOT re-test `now > due_at` here. `due_now` selects on
1298                // `due_at <= now`, so that comparison is true by construction and
1299                // says nothing about compliance — it was the reason this counter
1300                // once tracked "Strom processes started" rather than "APERAKs
1301                // missed". The discharge is what carries the meaning.
1302                if label.starts_with(mako_fristen::APERAK_WINDOW_LABEL_PREFIX) {
1303                    let now = time::OffsetDateTime::now_utc();
1304                    crate::metrics::EngineMetrics::global().aperak_missed(&label);
1305                    tracing::error!(
1306                        deadline_id = %id,
1307                        label       = %label,
1308                        due_at      = %deadline.due_at(),
1309                        fired_at    = %now,
1310                        overdue_secs = (now - deadline.due_at()).whole_seconds(),
1311                        "APERAK delivery window closed with no delivery — regulatory \
1312                         violation (APERAK AHB 1.0 §2.4.1 Strom / §2.3.1 Gas). \
1313                         Counter: makod_aperak_missed_total",
1314                    );
1315                }
1316
1317                let should_cancel = match (self.dispatch)(deadline).await {
1318                    Ok(()) => true,
1319                    Err(ref e) if e.is_version_conflict() => {
1320                        // The process was modified concurrently; the timeout
1321                        // command will be retried on the next poll cycle.
1322                        // Do NOT cancel — let the deadline remain due so it
1323                        // fires again until a non-conflict dispatch succeeds.
1324                        tracing::warn!(
1325                            deadline_id = %id,
1326                            label       = %label,
1327                            "deadline scheduler: VersionConflict; will retry on next poll",
1328                        );
1329                        false
1330                    }
1331                    Err(e) => {
1332                        tracing::warn!(
1333                            deadline_id = %id,
1334                            label       = %label,
1335                            error       = %e,
1336                            "deadline scheduler: dispatch failed (permanent); cancelling",
1337                        );
1338                        true
1339                    }
1340                };
1341                if should_cancel && let Err(e) = self.store.cancel(id).await {
1342                    tracing::error!(
1343                        deadline_id = %id,
1344                        error       = %e,
1345                        "deadline scheduler: cancel failed; deadline may fire again",
1346                    );
1347                }
1348            }
1349
1350            // If has_more, loop immediately to drain the batch.
1351        }
1352    }
1353}
1354
1355impl<DS: DeadlineStore> DeadlineScheduler<DS> {
1356    /// Attach a liveness heartbeat to this scheduler.
1357    ///
1358    /// The scheduler will store the current UTC Unix timestamp (seconds) into
1359    /// `heartbeat` at the end of every poll cycle.
1360    #[must_use]
1361    pub fn with_heartbeat(
1362        mut self,
1363        heartbeat: std::sync::Arc<std::sync::atomic::AtomicI64>,
1364    ) -> Self {
1365        self.heartbeat = Some(heartbeat);
1366        self
1367    }
1368
1369    /// Attach a graceful-shutdown token.
1370    ///
1371    /// Cancelling it makes [`DeadlineScheduler::run`] return at the next
1372    /// deadline boundary or immediately out of its idle sleep, so the caller can
1373    /// close the event store once the scheduler has stopped writing to it.
1374    #[must_use]
1375    pub fn with_shutdown(mut self, shutdown: tokio_util::sync::CancellationToken) -> Self {
1376        self.shutdown = Some(shutdown);
1377        self
1378    }
1379}
1380
1381impl<ES, SS, OS, DS, PR> EngineContext<ES, SS, OS, DS, PR>
1382where
1383    ES: EventStore,
1384    DS: DeadlineStore + Clone,
1385{
1386    /// Construct a [`DeadlineScheduler`] that polls the deadline store and
1387    /// dispatches fired deadlines via `dispatch`.
1388    ///
1389    /// The `dispatch` function is called for every fired deadline. It should
1390    /// resume the owning process and execute the appropriate timeout command.
1391    ///
1392    /// `batch_size` — deadlines fetched per poll cycle.
1393    /// `poll_interval` — sleep duration when no deadlines are due.
1394    ///
1395    /// ```rust,ignore
1396    /// use std::time::Duration;
1397    ///
1398    /// let scheduler = ctx.run_deadline_scheduler(
1399    ///     |d| async move {
1400    ///         tracing::info!(label = %d.label(), "firing deadline");
1401    ///         Ok(())
1402    ///     },
1403    ///     100,
1404    ///     Duration::from_secs(30),
1405    /// );
1406    /// tokio::spawn(async move { scheduler.run().await });
1407    /// ```
1408    #[must_use]
1409    pub fn run_deadline_scheduler<F, Fut>(
1410        &self,
1411        dispatch: F,
1412        batch_size: usize,
1413        poll_interval: std::time::Duration,
1414    ) -> DeadlineScheduler<DS>
1415    where
1416        F: Fn(Deadline) -> Fut + Send + Sync + 'static,
1417        Fut: std::future::Future<Output = Result<(), EngineError>> + Send + 'static,
1418    {
1419        DeadlineScheduler {
1420            store: self.deadline_store.clone(),
1421            dispatch: Box::new(move |d| Box::pin(dispatch(d))),
1422            batch_size,
1423            poll_interval,
1424            heartbeat: None,
1425            shutdown: None,
1426        }
1427    }
1428}
1429
1430// ── EngineBuilder ─────────────────────────────────────────────────────────────
1431
1432/// Assembles engine infrastructure and produces an [`EngineContext`].
1433///
1434/// Uses type-state to enforce that an event store is provided before
1435/// [`build`] can be called. All other stores default to `Noop`
1436/// implementations.
1437///
1438/// ## Quick start
1439///
1440/// ```rust,ignore
1441/// // Minimal — event store only, all others are Noop:
1442/// let ctx = EngineBuilder::new()
1443///     .with_event_store(InMemoryEventStore::new())
1444///     .build();
1445///
1446/// // Full infrastructure:
1447/// let ctx = EngineBuilder::new()
1448///     .with_event_store(InMemoryEventStore::new())
1449///     .with_snapshot_store(InMemorySnapshotStore::new())
1450///     .with_outbox_store(InMemoryOutboxStore::new())
1451///     .with_deadline_store(InMemoryDeadlineStore::new())
1452///     .with_registry(InMemoryProcessRegistry::new())
1453///     .register(Box::new(GpkeModule))
1454///     .build();
1455/// ```
1456///
1457/// [`build`]: EngineBuilder::build
1458pub struct EngineBuilder<
1459    ES = (),
1460    SS = NoopSnapshotStore,
1461    OS = NoopOutboxStore,
1462    DS = NoopDeadlineStore,
1463    PR = NoopProcessRegistry,
1464> {
1465    event_store: ES,
1466    snapshot_store: SS,
1467    outbox_store: OS,
1468    deadline_store: DS,
1469    registry: PR,
1470    dead_letter_sink: Arc<dyn DeadLetterSink>,
1471    modules: Vec<Box<dyn EngineModule>>,
1472    /// Active [`DeploymentRoles`] for this engine instance.
1473    ///
1474    /// Controls role-conditional PID registration via
1475    /// [`EngineModule::register_pids_with_roles`]. Defaults to
1476    /// [`DeploymentRoles::all()`] for backward compatibility.
1477    deployment_roles: DeploymentRoles,
1478    /// Optional profile validator injected by `makod` or callers that have
1479    /// access to `edi-energy`.  When `Some`, called for each
1480    /// [`ProfileRequirement`] declared by registered modules.  When `None`,
1481    /// profile requirements are not validated (safe in unit tests).
1482    ///
1483    /// Signature: `fn(message_type: &str) -> bool`
1484    ///
1485    /// [`ProfileRequirement`]: crate::profile::ProfileRequirement
1486    profile_validator: Option<Box<dyn Fn(&str) -> bool + Send + Sync>>,
1487}
1488#[cfg(any(test, feature = "testing"))]
1489impl Default
1490    for EngineBuilder<
1491        (),
1492        NoopSnapshotStore,
1493        NoopOutboxStore,
1494        NoopDeadlineStore,
1495        NoopProcessRegistry,
1496    >
1497{
1498    fn default() -> Self {
1499        Self {
1500            event_store: (),
1501            snapshot_store: NoopSnapshotStore,
1502            outbox_store: NoopOutboxStore,
1503            deadline_store: NoopDeadlineStore,
1504            registry: NoopProcessRegistry,
1505            dead_letter_sink: Arc::new(LogDeadLetterSink),
1506            modules: Vec::new(),
1507            deployment_roles: DeploymentRoles::all(),
1508            profile_validator: None,
1509        }
1510    }
1511}
1512
1513#[cfg(any(test, feature = "testing"))]
1514impl EngineBuilder {
1515    /// Create a new builder with all `Noop` defaults.
1516    ///
1517    /// Only available in `#[cfg(test)]` or with the `testing` feature enabled,
1518    /// because the Noop defaults silently discard outbox messages, deadlines,
1519    /// and process registry entries. Production binaries must wire real stores
1520    /// via the `with_*` builder methods.
1521    ///
1522    /// Call [`with_event_store`] before [`build`] — the event store is
1523    /// **required**.
1524    ///
1525    /// [`with_event_store`]: EngineBuilder::with_event_store
1526    /// [`build`]: EngineBuilder::build
1527    #[must_use]
1528    pub fn new() -> Self {
1529        Self::default()
1530    }
1531}
1532
1533impl<OS, DS, PR> EngineBuilder<(), NoopSnapshotStore, OS, DS, PR>
1534where
1535    OS: OutboxStore,
1536    DS: DeadlineStore,
1537    PR: ProcessRegistry,
1538{
1539    /// Create a production-ready builder with explicit stores for outbox,
1540    /// deadline, and process registry.
1541    ///
1542    /// This constructor is available in all build configurations including
1543    /// production binaries. It enforces that the three stores that can cause
1544    /// silent data loss (`OutboxStore`, `DeadlineStore`, `ProcessRegistry`)
1545    /// are provided explicitly — there is no Noop fallback.
1546    ///
1547    /// `NoopSnapshotStore` is used as the snapshot default because it is safe
1548    /// for production: skipping snapshots means full replay, but no data loss.
1549    /// Override with [`with_snapshot_store`] to enable snapshot-accelerated
1550    /// replay.
1551    ///
1552    /// Call [`with_event_store`] before [`build`] — the event store is
1553    /// **required**.
1554    ///
1555    /// ```rust,ignore
1556    /// let ctx = EngineBuilder::with_stores(outbox, deadline, registry)
1557    ///     .with_event_store(store.clone())
1558    ///     .with_snapshot_store(InMemorySnapshotStore::new())
1559    ///     .build();
1560    /// ```
1561    ///
1562    /// [`with_snapshot_store`]: EngineBuilder::with_snapshot_store
1563    /// [`with_event_store`]: EngineBuilder::with_event_store
1564    /// [`build`]: EngineBuilder::build
1565    #[must_use]
1566    pub fn with_stores(outbox_store: OS, deadline_store: DS, registry: PR) -> Self {
1567        Self {
1568            event_store: (),
1569            snapshot_store: NoopSnapshotStore,
1570            outbox_store,
1571            deadline_store,
1572            registry,
1573            dead_letter_sink: Arc::new(LogDeadLetterSink),
1574            modules: Vec::new(),
1575            deployment_roles: DeploymentRoles::all(),
1576            profile_validator: None,
1577        }
1578    }
1579}
1580
1581impl<ES, SS, OS, DS, PR> EngineBuilder<ES, SS, OS, DS, PR> {
1582    /// Set the event store. **Required** — `build()` is only available once
1583    /// this has been called with a type that implements [`EventStore`].
1584    ///
1585    /// Replaces any previously set event store (type-state transition).
1586    #[must_use]
1587    pub fn with_event_store<ES2: EventStore>(
1588        self,
1589        store: ES2,
1590    ) -> EngineBuilder<ES2, SS, OS, DS, PR> {
1591        EngineBuilder {
1592            event_store: store,
1593            snapshot_store: self.snapshot_store,
1594            outbox_store: self.outbox_store,
1595            deadline_store: self.deadline_store,
1596            registry: self.registry,
1597            dead_letter_sink: self.dead_letter_sink,
1598            modules: self.modules,
1599            deployment_roles: self.deployment_roles,
1600            profile_validator: self.profile_validator,
1601        }
1602    }
1603
1604    /// Set the snapshot store (default: [`NoopSnapshotStore`]).
1605    ///
1606    /// ## Default: `NoopSnapshotStore`
1607    ///
1608    /// Without calling this method the builder uses [`NoopSnapshotStore`],
1609    /// which silently discards all snapshot writes and returns `None` for
1610    /// every snapshot read.  The engine still functions correctly — every
1611    /// command handling call replays the full event log from the beginning
1612    /// instead of starting from a stored snapshot.  For low-volume processes
1613    /// this is fine; for long-lived processes with many events the replay cost
1614    /// can become significant.
1615    ///
1616    /// Enable snapshotting in production by providing a real [`SnapshotStore`]
1617    /// implementation (e.g. the SlateDB-backed store in `makod`).  In tests,
1618    /// `InMemorySnapshotStore` is available behind the `testing` feature flag.
1619    ///
1620    /// Note: [`Process::state_with_snapshot`][crate::process::Process::state_with_snapshot]
1621    /// is a compile-time no-op when the snapshot store is `NoopSnapshotStore`
1622    /// — it never calls the store and always returns `None`, so no snapshot is
1623    /// ever saved or loaded.
1624    #[must_use]
1625    pub fn with_snapshot_store<SS2: SnapshotStore>(
1626        self,
1627        store: SS2,
1628    ) -> EngineBuilder<ES, SS2, OS, DS, PR> {
1629        EngineBuilder {
1630            event_store: self.event_store,
1631            snapshot_store: store,
1632            outbox_store: self.outbox_store,
1633            deadline_store: self.deadline_store,
1634            registry: self.registry,
1635            dead_letter_sink: self.dead_letter_sink,
1636            modules: self.modules,
1637            deployment_roles: self.deployment_roles,
1638            profile_validator: self.profile_validator,
1639        }
1640    }
1641
1642    /// Set the outbox store (default: [`NoopOutboxStore`]).
1643    #[must_use]
1644    pub fn with_outbox_store<OS2: OutboxStore>(
1645        self,
1646        store: OS2,
1647    ) -> EngineBuilder<ES, SS, OS2, DS, PR> {
1648        EngineBuilder {
1649            event_store: self.event_store,
1650            snapshot_store: self.snapshot_store,
1651            outbox_store: store,
1652            deadline_store: self.deadline_store,
1653            registry: self.registry,
1654            dead_letter_sink: self.dead_letter_sink,
1655            modules: self.modules,
1656            deployment_roles: self.deployment_roles,
1657            profile_validator: self.profile_validator,
1658        }
1659    }
1660
1661    /// Set the deadline store (default: [`NoopDeadlineStore`]).
1662    #[must_use]
1663    pub fn with_deadline_store<DS2: DeadlineStore>(
1664        self,
1665        store: DS2,
1666    ) -> EngineBuilder<ES, SS, OS, DS2, PR> {
1667        EngineBuilder {
1668            event_store: self.event_store,
1669            snapshot_store: self.snapshot_store,
1670            outbox_store: self.outbox_store,
1671            deadline_store: store,
1672            registry: self.registry,
1673            dead_letter_sink: self.dead_letter_sink,
1674            modules: self.modules,
1675            deployment_roles: self.deployment_roles,
1676            profile_validator: self.profile_validator,
1677        }
1678    }
1679
1680    /// Set the process registry (default: [`NoopProcessRegistry`]).
1681    #[must_use]
1682    pub fn with_registry<PR2: ProcessRegistry>(
1683        self,
1684        registry: PR2,
1685    ) -> EngineBuilder<ES, SS, OS, DS, PR2> {
1686        EngineBuilder {
1687            event_store: self.event_store,
1688            snapshot_store: self.snapshot_store,
1689            outbox_store: self.outbox_store,
1690            deadline_store: self.deadline_store,
1691            registry,
1692            dead_letter_sink: self.dead_letter_sink,
1693            modules: self.modules,
1694            deployment_roles: self.deployment_roles,
1695            profile_validator: self.profile_validator,
1696        }
1697    }
1698
1699    /// Set the dead-letter sink (default: [`LogDeadLetterSink`]).
1700    ///
1701    /// The dead-letter sink receives every message that cannot be routed to a
1702    /// workflow. The default [`LogDeadLetterSink`] emits `tracing::warn!`
1703    /// events, making rejections visible in log output without configuration.
1704    ///
1705    /// Override with a persistent DLQ implementation in production:
1706    ///
1707    /// ```rust,ignore
1708    /// use mako_engine::dead_letter::LogDeadLetterSink;
1709    ///
1710    /// let ctx = EngineBuilder::new()
1711    ///     .with_event_store(my_store)
1712    ///     .with_dead_letter_sink(MyPersistentDlq::new())
1713    ///     .build();
1714    /// ```
1715    ///
1716    /// [`LogDeadLetterSink`]: crate::dead_letter::LogDeadLetterSink
1717    #[must_use]
1718    pub fn with_dead_letter_sink(mut self, sink: impl DeadLetterSink) -> Self {
1719        self.dead_letter_sink = Arc::new(sink);
1720        self
1721    }
1722
1723    /// Register an `edi-energy` profile validator for startup profile checks.
1724    ///
1725    /// The closure receives a message-type string (e.g. `"UTILMD"`) and must
1726    /// return `true` if at least one active profile for that message type is
1727    /// registered for today's date.
1728    ///
1729    /// Wire this in `makod` using the `edi-energy` global registry:
1730    ///
1731    /// ```rust,ignore
1732    /// use edi_energy::registry::ReleaseRegistry;
1733    ///
1734    /// let today = time::OffsetDateTime::now_utc().date();
1735    /// builder.with_profile_validator(move |msg_type| {
1736    ///     ReleaseRegistry::global()
1737    ///         .profiles_for_str(msg_type)
1738    ///         .any(|p| match (p.valid_from(), p.valid_until()) {
1739    ///             (Some(f), Some(u)) => f <= today && today <= u,
1740    ///             (Some(f), None)    => f <= today,
1741    ///             (None, _)          => true,
1742    ///         })
1743    /// })
1744    /// ```
1745    ///
1746    /// Domain crates do **not** need to call this — they only declare
1747    /// [`profile_requirements`].
1748    ///
1749    /// [`profile_requirements`]: EngineModule::profile_requirements
1750    #[must_use]
1751    pub fn with_profile_validator(
1752        mut self,
1753        validator: impl Fn(&str) -> bool + Send + Sync + 'static,
1754    ) -> Self {
1755        self.profile_validator = Some(Box::new(validator));
1756        self
1757    }
1758
1759    /// Register a domain module.
1760    ///
1761    /// The module name becomes visible in
1762    /// [`EngineContext::registered_modules`] after [`build`] is called.
1763    ///
1764    /// [`build`]: EngineBuilder::build
1765    #[must_use]
1766    pub fn register(mut self, module: Box<dyn EngineModule>) -> Self {
1767        self.modules.push(module);
1768        self
1769    }
1770
1771    /// Register multiple [`EngineModule`]s at once from a pre-built `Vec`.
1772    ///
1773    /// Equivalent to calling [`register`] in a loop. Useful when the set of
1774    /// modules is assembled conditionally (e.g. via `#[cfg]`-gated pushes to a
1775    /// `Vec<Box<dyn EngineModule>>`) before the builder chain starts.
1776    ///
1777    /// [`register`]: EngineBuilder::register
1778    #[must_use]
1779    pub fn register_many(mut self, modules: Vec<Box<dyn EngineModule>>) -> Self {
1780        self.modules.extend(modules);
1781        self
1782    }
1783
1784    /// Set the active [`DeploymentRoles`] for this engine instance.
1785    ///
1786    /// Controls role-conditional PID registration in [`EngineModule::register_pids_with_roles`].
1787    ///
1788    /// The default is [`DeploymentRoles::all()`], which registers every PID unconditionally
1789    /// — identical to the pre-role-aware behavior. Providing an explicit role set
1790    /// restricts role-conditional blocks to only the declared roles:
1791    ///
1792    /// - **NB-only** (`DeploymentRoles::nb()`): 19001/19002 route to `gpke-konfiguration`;
1793    ///   WiM nMSB blocks are skipped.
1794    /// - **nMSB-only** (`DeploymentRoles::nmsb()`): 19001/19002 route to `wim-geraeteubernahme`;
1795    ///   GPKE NB blocks are skipped.
1796    /// - **NB + gMSB** (`DeploymentRoles::nb_msb()`): most common Stadtwerke combination.
1797    ///
1798    /// # Conflict guard
1799    ///
1800    /// When two modules would register the same PID to **different** workflows, the
1801    /// engine panics during [`build`]. Set explicit roles to prevent both modules from
1802    /// activating the same PID simultaneously:
1803    ///
1804    /// ```rust,ignore
1805    /// use mako_engine::marktrolle::DeploymentRoles;
1806    ///
1807    /// let ctx = EngineBuilder::with_stores(outbox, deadline, registry)
1808    ///     .with_event_store(store)
1809    ///     .with_deployment_roles(DeploymentRoles::nb())  // only NB: GPKE gets 19001/19002
1810    ///     .register(Box::new(GpkeModule))
1811    ///     .register(Box::new(WimModule))  // nMSB block skipped — no conflict
1812    ///     .build();
1813    /// ```
1814    ///
1815    /// [`build`]: EngineBuilder::build
1816    #[must_use]
1817    pub fn with_deployment_roles(mut self, roles: DeploymentRoles) -> Self {
1818        self.deployment_roles = roles;
1819        self
1820    }
1821}
1822
1823impl<ES, SS, OS, DS, PR> EngineBuilder<ES, SS, OS, DS, PR>
1824where
1825    ES: EventStore,
1826    SS: SnapshotStore,
1827    OS: OutboxStore,
1828    DS: DeadlineStore,
1829    PR: ProcessRegistry,
1830{
1831    /// Build the [`EngineContext`].
1832    ///
1833    /// Consumes the builder. All registered modules and configured stores are
1834    /// moved into the returned [`EngineContext`].
1835    ///
1836    /// This method is only available when `ES` implements [`EventStore`].
1837    /// If you have not called [`with_event_store`], this will not compile.
1838    ///
1839    /// # Panics
1840    ///
1841    /// Panics when any registered module returns `Err` from
1842    /// [`EngineModule::configure`]. The panic message includes the module
1843    /// name and the error string so the deployment failure is actionable.
1844    ///
1845    /// [`with_event_store`]: EngineBuilder::with_event_store
1846    #[must_use]
1847    #[allow(clippy::too_many_lines)]
1848    pub fn build(self) -> EngineContext<ES, SS, OS, DS, PR> {
1849        // ── Noop store safety checks ──────────────────────────────────────────
1850        //
1851        // Noop stores lose data silently: NoopDeadlineStore drops every APERAK
1852        // deadline (BNetzA violation), NoopOutboxStore discards all outbound
1853        // messages, NoopProcessRegistry loses conversation routing on restart.
1854        //
1855        // In production builds (no `testing` feature, not running under
1856        // `#[test]`), the Noop constructors are cfg-gated out so this branch
1857        // is dead code and compiles away. In test/testing/tracing builds we
1858        // emit warnings so test harnesses see the configuration in log output.
1859        //
1860        // IMPORTANT: if you are reading this because a panic fired in production,
1861        // it means the `testing` feature was accidentally enabled in the binary.
1862        // Remove it from the production Cargo.toml feature list immediately.
1863        {
1864            let os_name = std::any::type_name::<OS>();
1865            let ds_name = std::any::type_name::<DS>();
1866            let pr_name = std::any::type_name::<PR>();
1867
1868            // Regulatory-critical stores: panic in any build context if these
1869            // are noop. OutboxStore and DeadlineStore must be durable in
1870            // production; ProcessRegistry must survive restarts.
1871            #[cfg(not(any(test, feature = "testing")))]
1872            {
1873                assert!(
1874                    !ds_name.contains("NoopDeadlineStore"),
1875                    "EngineBuilder::build: NoopDeadlineStore is active in a \
1876                     non-testing build. This silently discards all APERAK deadlines, \
1877                     which is an immediately reportable BNetzA violation \
1878                     (BK6-22-024 §5, BK7-24-01-009). \
1879                     Call .with_deadline_store(SlateDbStore::as_deadline_store()) \
1880                     in your production engine assembly. \
1881                     If this is a test, enable the 'testing' feature."
1882                );
1883                assert!(
1884                    !os_name.contains("NoopOutboxStore"),
1885                    "EngineBuilder::build: NoopOutboxStore is active in a \
1886                     non-testing build. This silently discards all outbound \
1887                     APERAK, CONTRL, and UTILMD messages. \
1888                     Call .with_outbox_store(SlateDbStore::as_outbox_store()) \
1889                     in your production engine assembly. \
1890                     If this is a test, enable the 'testing' feature."
1891                );
1892                assert!(
1893                    !pr_name.contains("NoopProcessRegistry"),
1894                    "EngineBuilder::build: NoopProcessRegistry is active in a \
1895                     non-testing build. This means conversation routing \
1896                     (PID → stream_id lookup) is lost on every restart, \
1897                     breaking all WiM, GeLi Gas, and GPKE in-flight processes. \
1898                     Call .with_registry(SlateDbStore::as_process_registry()) \
1899                     in your production engine assembly. \
1900                     If this is a test, enable the 'testing' feature."
1901                );
1902            }
1903
1904            // In test/testing/tracing builds: emit warnings instead of panicking.
1905            #[cfg(any(test, feature = "testing", feature = "tracing"))]
1906            {
1907                let ss_name = std::any::type_name::<SS>();
1908                if ss_name.contains("NoopSnapshotStore") {
1909                    tracing::warn!(
1910                        store = ss_name,
1911                        "EngineBuilder: NoopSnapshotStore is active — snapshots will not be \
1912                         persisted. Use SlateDbStore::as_snapshot_store() in production."
1913                    );
1914                }
1915                if os_name.contains("NoopOutboxStore") {
1916                    tracing::warn!(
1917                        store = os_name,
1918                        "EngineBuilder: NoopOutboxStore is active — outbound messages will be \
1919                         silently discarded. Use SlateDbStore::as_outbox_store() in production."
1920                    );
1921                }
1922                if ds_name.contains("NoopDeadlineStore") {
1923                    tracing::warn!(
1924                        store = ds_name,
1925                        "EngineBuilder: NoopDeadlineStore is active — scheduled deadlines will \
1926                         not fire after restart. Use SlateDbStore::as_deadline_store() in production."
1927                    );
1928                }
1929                if pr_name.contains("NoopProcessRegistry") {
1930                    tracing::warn!(
1931                        store = pr_name,
1932                        "EngineBuilder: NoopProcessRegistry is active — process routing will be \
1933                         lost on restart. Use SlateDbStore::as_process_registry() in production."
1934                    );
1935                }
1936            }
1937        }
1938        // Validate every module before assembling the context.
1939        // A missing adapter or misconfigured module fails at startup (not at
1940        // first inbound message), making deployment failures observable immediately.
1941        for module in &self.modules {
1942            if let Err(msg) = module.configure() {
1943                panic!(
1944                    "EngineBuilder::build: module '{}' failed configuration validation: {}",
1945                    module.name(),
1946                    msg
1947                );
1948            }
1949            // Validate profile requirements via the injected validator.
1950            // Domain crates declare requirements; only the binary crate (makod)
1951            // injects the edi-energy registry — domain crates need no edi-energy
1952            // import for this check.
1953            if let Some(ref validator) = self.profile_validator {
1954                for req in module.profile_requirements() {
1955                    assert!(
1956                        validator(req.message_type),
1957                        "EngineBuilder::build: module '{}' requires an active edi-energy \
1958                             profile for '{}' ({}) but none is registered for today's date. \
1959                             Run `cargo xtask codegen` to add the missing profile.",
1960                        module.name(),
1961                        req.message_type,
1962                        req.label,
1963                    );
1964                }
1965            }
1966        }
1967        // Build the PID router from all registered modules.
1968        // Also assert that no two modules claim the same PID — a PID overlap
1969        // is always a configuration error: one module's messages would be
1970        // silently swallowed by another's workflow, producing missing-process
1971        // errors or incorrect audit trails.
1972        let mut pid_router = PidRouter::new();
1973        let mut pid_owners: std::collections::HashMap<u32, &str> = std::collections::HashMap::new();
1974        // Keep each module's scratch router so we can build `pid_router` from
1975        // them in a second pass with the resolved ownership table.
1976        let mut module_scratches: Vec<PidRouter> = Vec::with_capacity(self.modules.len());
1977
1978        // Pass 1 — detect conflicts, determine PID ownership (first-wins for
1979        // explicit roles, last-wins for DeploymentRoles::all()).
1980        for module in &self.modules {
1981            // Temporarily build a scratch router to read this module's PIDs
1982            // for cross-module overlap detection (module-ownership level).
1983            let mut scratch = PidRouter::new();
1984            module.register_pids_with_roles(&mut scratch, &self.deployment_roles);
1985            for pid in scratch.registered_pids() {
1986                if let Some(prev) = pid_owners.insert(pid, module.name()) {
1987                    if self.deployment_roles.is_all() {
1988                        // With DeploymentRoles::all() (the default), role-conditional PIDs
1989                        // are registered by all modules that claim them, producing last-wins
1990                        // semantics. This is acceptable for single-role and dev/test deployments.
1991                        //
1992                        // In production multi-role deployments where both an NB and nMSB role
1993                        // are served by the same instance, set explicit roles via
1994                        // `EngineBuilder::with_deployment_roles` to prevent silent misrouting.
1995                        //
1996                        // We emit a debug-level log here (not warn) because the vast majority
1997                        // of deployments are single-role and this overlap is expected/harmless.
1998                        #[cfg(feature = "tracing")]
1999                        tracing::debug!(
2000                            pid,
2001                            previous_module = prev,
2002                            current_module = module.name(),
2003                            "PID registered by multiple modules with DeploymentRoles::all(); \
2004                             last module wins (use with_deployment_roles for strict routing)",
2005                        );
2006                        let _ = prev; // suppress unused-variable warning when tracing is off
2007                    } else {
2008                        // Explicit roles: the FIRST module to register a PID retains ownership.
2009                        // Restore the previous (first) owner and emit a warning so the operator
2010                        // can investigate.  A panic would be too strict: some shared PIDs
2011                        // (e.g. REMADV 33001/33002) are legitimately claimed by both GPKE and
2012                        // WiM billing; conversation-ID routing is the long-term solution, but
2013                        // first-wins gives correct behaviour for all current deployments.
2014                        pid_owners.insert(pid, prev); // restore first owner
2015                        #[cfg(feature = "tracing")]
2016                        tracing::warn!(
2017                            pid,
2018                            first_module = prev,
2019                            second_module = module.name(),
2020                            "PID {pid} claimed by both '{prev}' and '{}' with explicit \
2021                             DeploymentRoles; first module ('{prev}') retains ownership. \
2022                             Verify PID registration is correct for this deployment.",
2023                            module.name(),
2024                        );
2025                        #[cfg(not(feature = "tracing"))]
2026                        let _ = prev; // suppress unused-variable warning when tracing is off
2027                    }
2028                }
2029            }
2030            module_scratches.push(scratch);
2031        }
2032
2033        // Pass 2 — build the real `pid_router` from the scratch pads, respecting
2034        // the ownership table built in pass 1.
2035        for (module, scratch) in self.modules.iter().zip(module_scratches.iter()) {
2036            // Unambiguous (Sparte-agnostic) entries: only register if this module
2037            // owns the PID in the resolved ownership table.
2038            for pid in scratch.registered_pids() {
2039                if pid_owners.get(&pid).copied() == Some(module.name())
2040                    && let Some(wf) = scratch.route(pid)
2041                {
2042                    pid_router.register(pid, wf);
2043                }
2044            }
2045            // Commodity (Sparte-qualified) entries use distinct (pid, Sparte) keys
2046            // and never conflict across modules; register them all unconditionally.
2047            for (pid, sparte, wf) in scratch.registered_commodity_entries() {
2048                pid_router.register_with_sparte(pid, sparte, wf);
2049            }
2050        }
2051        let registered_modules = self.modules.iter().map(|m| m.name()).collect();
2052        let registered_workflows = self
2053            .modules
2054            .iter()
2055            .flat_map(|m| m.workflow_names().iter().copied())
2056            .collect();
2057        EngineContext {
2058            event_store: Arc::new(self.event_store),
2059            snapshot_store: self.snapshot_store,
2060            outbox_store: self.outbox_store,
2061            deadline_store: self.deadline_store,
2062            registry: self.registry,
2063            dead_letter_sink: self.dead_letter_sink,
2064            pid_router,
2065            registered_modules,
2066            registered_workflows,
2067        }
2068    }
2069}
2070
2071#[cfg(test)]
2072mod tests {
2073    use super::*;
2074    use crate::{
2075        deadline::InMemoryDeadlineStore,
2076        error::WorkflowError,
2077        event_store::InMemoryEventStore,
2078        ids::TenantId,
2079        outbox::InMemoryOutboxStore,
2080        pid_router::PidRouter,
2081        registry::InMemoryProcessRegistry,
2082        snapshot::InMemorySnapshotStore,
2083        version::WorkflowId,
2084        workflow::{CommandPayload, EventPayload, Workflow},
2085    };
2086
2087    // ── Minimal workflow for spawn/resume tests ───────────────────────────────
2088
2089    #[derive(serde::Serialize, serde::Deserialize)]
2090    struct PingEvent;
2091
2092    impl EventPayload for PingEvent {
2093        fn event_type(&self) -> &'static str {
2094            "Ping"
2095        }
2096    }
2097
2098    struct PingCommand;
2099
2100    impl CommandPayload for PingCommand {}
2101
2102    #[derive(Default, Clone)]
2103    struct PingState;
2104
2105    struct PingWorkflow;
2106
2107    impl Workflow for PingWorkflow {
2108        type State = PingState;
2109        type Event = PingEvent;
2110        type Command = PingCommand;
2111
2112        fn apply(state: PingState, _: &PingEvent) -> PingState {
2113            state
2114        }
2115
2116        fn handle(
2117            _: &PingState,
2118            _: PingCommand,
2119        ) -> Result<crate::workflow::WorkflowOutput<PingEvent>, WorkflowError> {
2120            Ok(vec![PingEvent].into())
2121        }
2122    }
2123
2124    struct TestModule;
2125
2126    impl EngineModule for TestModule {
2127        fn name(&self) -> &'static str {
2128            "test-module"
2129        }
2130    }
2131
2132    // ── Tests ─────────────────────────────────────────────────────────────────
2133
2134    #[test]
2135    fn build_with_event_store_only() {
2136        let ctx = EngineBuilder::new()
2137            .with_event_store(InMemoryEventStore::new())
2138            .build();
2139        assert!(ctx.registered_modules().is_empty());
2140    }
2141
2142    #[test]
2143    fn build_with_all_stores_and_module() {
2144        let ctx = EngineBuilder::new()
2145            .with_event_store(InMemoryEventStore::new())
2146            .with_snapshot_store(InMemorySnapshotStore::new())
2147            .with_outbox_store(InMemoryOutboxStore::new())
2148            .with_deadline_store(InMemoryDeadlineStore::new())
2149            .with_registry(InMemoryProcessRegistry::new())
2150            .register(Box::new(TestModule))
2151            .build();
2152        assert_eq!(ctx.registered_modules(), &["test-module"]);
2153    }
2154
2155    #[test]
2156    fn multiple_modules_ordered() {
2157        struct ModA;
2158        impl EngineModule for ModA {
2159            fn name(&self) -> &'static str {
2160                "mod-a"
2161            }
2162        }
2163        struct ModB;
2164        impl EngineModule for ModB {
2165            fn name(&self) -> &'static str {
2166                "mod-b"
2167            }
2168        }
2169
2170        let ctx = EngineBuilder::new()
2171            .with_event_store(InMemoryEventStore::new())
2172            .register(Box::new(ModA))
2173            .register(Box::new(ModB))
2174            .build();
2175        assert_eq!(ctx.registered_modules(), &["mod-a", "mod-b"]);
2176    }
2177
2178    #[tokio::test]
2179    async fn spawn_creates_independent_processes() {
2180        let ctx = EngineBuilder::new()
2181            .with_event_store(InMemoryEventStore::new())
2182            .build();
2183        let wf_id = WorkflowId::new("ping", "FV2024-10-01");
2184
2185        let p1 = ctx.spawn::<PingWorkflow>(TenantId::new(), wf_id.clone());
2186        let p2 = ctx.spawn::<PingWorkflow>(TenantId::new(), wf_id);
2187
2188        assert_ne!(p1.process_id(), p2.process_id());
2189    }
2190
2191    #[tokio::test]
2192    async fn resume_sees_previously_appended_events() {
2193        let store = InMemoryEventStore::new();
2194        let ctx = EngineBuilder::new().with_event_store(store).build();
2195
2196        let p = ctx.spawn::<PingWorkflow>(TenantId::new(), WorkflowId::new("ping", "FV2024-10-01"));
2197        p.execute(PingCommand).await.unwrap();
2198
2199        let identity = p.identity();
2200        let resumed = ctx.resume::<PingWorkflow>(identity);
2201        assert_eq!(resumed.event_count().await.unwrap(), 1);
2202    }
2203
2204    #[tokio::test]
2205    async fn registry_routes_process_via_conversation_key() {
2206        use crate::registry::RegistryKey;
2207        let ctx = EngineBuilder::new()
2208            .with_event_store(InMemoryEventStore::new())
2209            .with_registry(InMemoryProcessRegistry::new())
2210            .build();
2211
2212        let p = ctx.spawn::<PingWorkflow>(TenantId::new(), WorkflowId::new("ping", "FV2024-10-01"));
2213        let tenant = p.tenant_id();
2214        let conv_key = RegistryKey::parse("conv:test-conversation-123").expect("valid key");
2215        ctx.registry()
2216            .register(tenant, &conv_key, p.identity())
2217            .await
2218            .unwrap();
2219
2220        let found = ctx
2221            .registry()
2222            .lookup(tenant, &conv_key)
2223            .await
2224            .unwrap()
2225            .expect("must be registered");
2226        let resumed = ctx.resume::<PingWorkflow>(found);
2227        assert_eq!(resumed.process_id(), p.process_id());
2228    }
2229
2230    #[test]
2231    fn pid_router_populated_by_module_register_pids() {
2232        struct PidModule;
2233        impl EngineModule for PidModule {
2234            fn name(&self) -> &'static str {
2235                "pid-module"
2236            }
2237            fn register_pids(&self, router: &mut PidRouter) {
2238                router.register(55001, "gpke-supplier-change");
2239                router.register(55002, "gpke-supplier-change");
2240            }
2241        }
2242
2243        let ctx = EngineBuilder::new()
2244            .with_event_store(InMemoryEventStore::new())
2245            .register(Box::new(PidModule))
2246            .build();
2247
2248        assert_eq!(ctx.pid_router().route(55001), Some("gpke-supplier-change"));
2249        assert_eq!(ctx.pid_router().route(55002), Some("gpke-supplier-change"));
2250        assert!(ctx.pid_router().route(99999).is_none());
2251        assert_eq!(ctx.pid_router().len(), 2);
2252    }
2253
2254    /// Verify that `register_pids_with_roles` gates PIDs behind role checks.
2255    ///
2256    /// Scenario: two modules share PID 19001.
2257    /// - ModuleA registers 19001 → "workflow-a" when role `Nb` is present.
2258    /// - ModuleB registers 19001 → "workflow-b" when role `Nmsb` is explicitly set
2259    ///   (not on `all()`).
2260    ///
2261    /// - `all()`: ModuleA fires (Nb ∈ all), ModuleB does NOT (is_all → skip).
2262    ///   → 19001 routes to "workflow-a".
2263    /// - `from_roles([Nb])`: ModuleA fires, ModuleB skips.
2264    ///   → 19001 routes to "workflow-a".
2265    /// - `from_roles([Nmsb])`: ModuleA skips, ModuleB fires.
2266    ///   → 19001 routes to "workflow-b".
2267    #[test]
2268    fn register_pids_with_roles_gates_pids_correctly() {
2269        use crate::marktrolle::{DeploymentRoles, Marktrolle};
2270
2271        struct ModuleA;
2272        impl EngineModule for ModuleA {
2273            fn name(&self) -> &'static str {
2274                "module-a"
2275            }
2276            fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
2277                if roles.contains(Marktrolle::Nb) {
2278                    router.register(19_001, "workflow-a");
2279                }
2280            }
2281        }
2282
2283        struct ModuleB;
2284        impl EngineModule for ModuleB {
2285            fn name(&self) -> &'static str {
2286                "module-b"
2287            }
2288            fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
2289                // Only fires on explicit Nmsb, not on all() (backward-compat sentinel).
2290                if !roles.is_all() && roles.contains(Marktrolle::Nmsb) {
2291                    router.register(19_001, "workflow-b");
2292                    router.register(19_015, "workflow-b");
2293                }
2294            }
2295        }
2296
2297        let build = |roles: DeploymentRoles| {
2298            EngineBuilder::new()
2299                .with_event_store(InMemoryEventStore::new())
2300                .with_deployment_roles(roles)
2301                .register(Box::new(ModuleA))
2302                .register(Box::new(ModuleB))
2303                .build()
2304        };
2305
2306        // all() → backward compat: ModuleA registers 19001 (Nb ∈ all), ModuleB skips.
2307        let ctx = build(DeploymentRoles::all());
2308        assert_eq!(ctx.pid_router().route(19_001), Some("workflow-a"));
2309        assert!(ctx.pid_router().route(19_015).is_none());
2310
2311        // Explicit Nb → same result: ModuleA registers, ModuleB (nMSB) skips.
2312        let ctx = build(DeploymentRoles::nb());
2313        assert_eq!(ctx.pid_router().route(19_001), Some("workflow-a"));
2314        assert!(ctx.pid_router().route(19_015).is_none());
2315
2316        // Explicit Nmsb → ModuleA skips (Nb ∉ roles), ModuleB registers.
2317        let ctx = build(DeploymentRoles::nmsb());
2318        assert_eq!(ctx.pid_router().route(19_001), Some("workflow-b"));
2319        assert_eq!(ctx.pid_router().route(19_015), Some("workflow-b"));
2320    }
2321
2322    /// Verify that explicit roles with two conflicting modules use first-wins semantics
2323    /// (the first module to register a PID retains ownership; the second is silently skipped).
2324    #[test]
2325    fn register_pids_with_roles_conflict_uses_first_wins_with_explicit_roles() {
2326        use crate::marktrolle::{DeploymentRoles, Marktrolle};
2327
2328        struct ConflictA;
2329        impl EngineModule for ConflictA {
2330            fn name(&self) -> &'static str {
2331                "conflict-a"
2332            }
2333            fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
2334                if roles.contains(Marktrolle::Nb) {
2335                    router.register(19_001, "workflow-a");
2336                }
2337            }
2338        }
2339
2340        struct ConflictB;
2341        impl EngineModule for ConflictB {
2342            fn name(&self) -> &'static str {
2343                "conflict-b"
2344            }
2345            fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
2346                if !roles.is_all() && roles.contains(Marktrolle::Nmsb) {
2347                    router.register(19_001, "workflow-b"); // same PID, different workflow
2348                }
2349            }
2350        }
2351
2352        // from_roles([Nb, Nmsb]): both modules fire for PID 19_001.
2353        // First-wins: ConflictA (registered first) retains ownership → "workflow-a".
2354        let ctx = EngineBuilder::new()
2355            .with_event_store(InMemoryEventStore::new())
2356            .with_deployment_roles(DeploymentRoles::from_roles([
2357                Marktrolle::Nb,
2358                Marktrolle::Nmsb,
2359            ]))
2360            .register(Box::new(ConflictA))
2361            .register(Box::new(ConflictB))
2362            .build();
2363        assert_eq!(
2364            ctx.pid_router().route(19_001),
2365            Some("workflow-a"),
2366            "first module should win on PID conflict with explicit roles"
2367        );
2368    }
2369
2370    // ── Graceful shutdown ─────────────────────────────────────────────────────
2371
2372    /// Cancelling the token must make `run` return.
2373    ///
2374    /// The workers used to loop until the process exited: the shutdown path
2375    /// cancelled a token nobody read, dropped their `JoinHandle`s — which does
2376    /// not abort a Tokio task — and then closed the event store underneath
2377    /// them. An outbox `acknowledge` losing that race leaves the counterparty
2378    /// holding a message the outbox still shows as pending, and the next start
2379    /// delivers it again.
2380    #[tokio::test]
2381    async fn a_cancelled_outbox_worker_returns() {
2382        let worker = OutboxWorker {
2383            store: InMemoryOutboxStore::new(),
2384            sender: AlwaysDelivers,
2385            deadline_store: InMemoryDeadlineStore::new(),
2386            batch_size: 10,
2387            // Far longer than the timeout below: the point is that cancellation
2388            // interrupts the idle sleep rather than being noticed after it.
2389            poll_interval: std::time::Duration::from_secs(300),
2390            max_attempts: 48,
2391            max_retry_window: std::time::Duration::from_secs(72 * 3600),
2392            dead_letter_sink: std::sync::Arc::new(crate::dead_letter::LogDeadLetterSink),
2393            heartbeat: None,
2394            shutdown: None,
2395        };
2396        let token = tokio_util::sync::CancellationToken::new();
2397        let worker = worker.with_shutdown(token.clone());
2398
2399        let handle = tokio::spawn(worker.run());
2400        // Let it reach the sleep, then signal.
2401        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2402        token.cancel();
2403
2404        tokio::time::timeout(std::time::Duration::from_secs(5), handle)
2405            .await
2406            .expect("outbox worker must return promptly after cancellation")
2407            .expect("outbox worker must not panic");
2408    }
2409
2410    #[tokio::test]
2411    async fn a_cancelled_deadline_scheduler_returns() {
2412        let scheduler = DeadlineScheduler {
2413            store: InMemoryDeadlineStore::new(),
2414            dispatch: Box::new(|_| Box::pin(async { Ok(()) })),
2415            batch_size: 100,
2416            poll_interval: std::time::Duration::from_secs(300),
2417            heartbeat: None,
2418            shutdown: None,
2419        };
2420        let token = tokio_util::sync::CancellationToken::new();
2421        let scheduler = scheduler.with_shutdown(token.clone());
2422
2423        let handle = tokio::spawn(scheduler.run());
2424        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2425        token.cancel();
2426
2427        tokio::time::timeout(std::time::Duration::from_secs(5), handle)
2428            .await
2429            .expect("deadline scheduler must return promptly after cancellation")
2430            .expect("deadline scheduler must not panic");
2431    }
2432
2433    /// A token cancelled before the first poll must stop the worker without it
2434    /// touching the store at all — the case where shutdown arrives during boot.
2435    #[tokio::test]
2436    async fn a_worker_cancelled_before_it_starts_does_no_work() {
2437        let outbox = InMemoryOutboxStore::new();
2438        let stream_id = crate::ids::StreamId::new("gpke/shutdown-test");
2439        let msg = outbox_message(&stream_id, "UTILMD");
2440        outbox.enqueue(std::slice::from_ref(&msg)).await.unwrap();
2441
2442        let token = tokio_util::sync::CancellationToken::new();
2443        token.cancel();
2444
2445        let worker = OutboxWorker {
2446            store: outbox.clone(),
2447            sender: AlwaysDelivers,
2448            deadline_store: InMemoryDeadlineStore::new(),
2449            batch_size: 10,
2450            poll_interval: std::time::Duration::from_millis(5),
2451            max_attempts: 48,
2452            max_retry_window: std::time::Duration::from_secs(72 * 3600),
2453            dead_letter_sink: std::sync::Arc::new(crate::dead_letter::LogDeadLetterSink),
2454            heartbeat: None,
2455            shutdown: None,
2456        }
2457        .with_shutdown(token);
2458
2459        tokio::time::timeout(std::time::Duration::from_secs(5), worker.run())
2460            .await
2461            .expect("an already-cancelled worker must return immediately");
2462
2463        assert_eq!(
2464            outbox.pending_now(10).await.unwrap().len(),
2465            1,
2466            "the message must stay queued for the next start, not be delivered \
2467             by a worker that was told to stop",
2468        );
2469    }
2470
2471    // ── APERAK delivery-window discharge ──────────────────────────────────────
2472
2473    /// A sender that always succeeds, so the worker takes the delivery path.
2474    struct AlwaysDelivers;
2475    impl As4Sender for AlwaysDelivers {
2476        async fn send(&self, _msg: &crate::outbox::OutboxMessage) -> Result<(), EngineError> {
2477            Ok(())
2478        }
2479    }
2480
2481    fn outbox_message(
2482        stream_id: &crate::ids::StreamId,
2483        message_type: &str,
2484    ) -> crate::outbox::OutboxMessage {
2485        crate::outbox::OutboxMessage::new(
2486            stream_id.clone(),
2487            crate::ids::ProcessId::new(),
2488            TenantId::new(),
2489            crate::ids::CorrelationId::new(),
2490            crate::ids::ConversationId::new(),
2491            crate::ids::EventId::new(),
2492            message_type,
2493            "9900357000004",
2494            serde_json::json!({}),
2495        )
2496    }
2497
2498    fn deadline_on(
2499        stream_id: &crate::ids::StreamId,
2500        msg: &crate::outbox::OutboxMessage,
2501        label: &str,
2502    ) -> Deadline {
2503        Deadline::new(
2504            stream_id.clone(),
2505            msg.process_id,
2506            msg.tenant_id,
2507            WorkflowId::new("gpke-supplier-change", "FV2025-10-01"),
2508            label,
2509            time::OffsetDateTime::now_utc() + time::Duration::hours(6),
2510        )
2511    }
2512
2513    /// Deliver `msg` through the worker's real loop and return the labels that
2514    /// survive on its stream.
2515    ///
2516    /// Drives `run` rather than calling the discharge directly — the wiring is
2517    /// the thing under test, and calling the method straight passes even when
2518    /// `run` never invokes it.
2519    async fn labels_surviving_delivery(
2520        msg: &crate::outbox::OutboxMessage,
2521        stream_id: &crate::ids::StreamId,
2522        registered: &[&str],
2523    ) -> Vec<String> {
2524        let deadlines = InMemoryDeadlineStore::new();
2525        for label in registered {
2526            deadlines
2527                .register(&deadline_on(stream_id, msg, label))
2528                .await
2529                .unwrap();
2530        }
2531        let outbox = InMemoryOutboxStore::new();
2532        outbox.enqueue(std::slice::from_ref(msg)).await.unwrap();
2533
2534        let worker = OutboxWorker {
2535            store: outbox.clone(),
2536            sender: AlwaysDelivers,
2537            deadline_store: deadlines.clone(),
2538            batch_size: 10,
2539            poll_interval: std::time::Duration::from_millis(5),
2540            max_attempts: 48,
2541            max_retry_window: std::time::Duration::from_secs(72 * 3600),
2542            dead_letter_sink: std::sync::Arc::new(crate::dead_letter::LogDeadLetterSink),
2543            heartbeat: None,
2544            shutdown: None,
2545        };
2546        // `run` never returns; give it enough cycles to drain the one message.
2547        let _ = tokio::time::timeout(std::time::Duration::from_millis(300), worker.run()).await;
2548        assert!(
2549            outbox.pending_now(10).await.unwrap().is_empty(),
2550            "the message must have been delivered and acknowledged",
2551        );
2552
2553        let mut left: Vec<String> = deadlines
2554            .for_stream(stream_id)
2555            .await
2556            .unwrap()
2557            .iter()
2558            .map(|d| d.label().to_owned())
2559            .collect();
2560        left.sort();
2561        left
2562    }
2563
2564    /// Delivering a message must retire the window that was watching for it.
2565    ///
2566    /// These windows are registered when the message is enqueued and nothing
2567    /// else ever cancels them, so without the discharge they fire for **every**
2568    /// process — including every one that answered on time. The scheduler cannot
2569    /// tell those apart (a deadline reaching `due_now` is late by construction),
2570    /// so the miss counters would track processes started, not obligations
2571    /// missed.
2572    #[tokio::test]
2573    async fn delivering_a_message_discharges_its_delivery_window() {
2574        // (message type, the window it answers for)
2575        for (message_type, window) in [
2576            ("APERAK", mako_fristen::APERAK_STROM_WINDOW_LABEL),
2577            ("APERAK", mako_fristen::APERAK_GAS_FOLGEPROZESS_LABEL),
2578            ("APERAK", mako_fristen::APERAK_GAS_INITIALPROZESS_LABEL),
2579            ("CONTRL", mako_fristen::CONTRL_FRIST_LABEL),
2580        ] {
2581            let stream_id = crate::ids::StreamId::new("gpke-supplier-change-1");
2582            let msg = outbox_message(&stream_id, message_type);
2583            // A process-response deadline shares the stream and must survive:
2584            // it is waiting on the counterparty, not on our delivery.
2585            let left =
2586                labels_surviving_delivery(&msg, &stream_id, &[window, "gpke-response-window"])
2587                    .await;
2588            assert_eq!(
2589                left,
2590                vec!["gpke-response-window"],
2591                "delivering {message_type} must discharge `{window}` and leave \
2592                 every other deadline alone",
2593            );
2594        }
2595    }
2596
2597    /// A delivery must not discharge a *different* message's window.
2598    ///
2599    /// The CONTRL and APERAK obligations run concurrently on the same
2600    /// interchange. Acknowledging syntax (CONTRL) says nothing about whether the
2601    /// application-level APERAK went out, so discharging both on one delivery
2602    /// would silence a real violation.
2603    #[tokio::test]
2604    async fn a_delivery_does_not_discharge_another_messages_window() {
2605        let stream_id = crate::ids::StreamId::new("gpke-supplier-change-1");
2606        let contrl = outbox_message(&stream_id, "CONTRL");
2607
2608        let left = labels_surviving_delivery(
2609            &contrl,
2610            &stream_id,
2611            &[
2612                mako_fristen::CONTRL_FRIST_LABEL,
2613                mako_fristen::APERAK_STROM_WINDOW_LABEL,
2614            ],
2615        )
2616        .await;
2617
2618        assert_eq!(
2619            left,
2620            vec![mako_fristen::APERAK_STROM_WINDOW_LABEL.to_owned()],
2621            "a delivered CONTRL discharges only the CONTRL window; the APERAK \
2622             obligation is still outstanding",
2623        );
2624    }
2625
2626    /// Every delivery-window label must be discharged by the message it watches.
2627    ///
2628    /// This is the invariant the miss counters rest on. A window label that
2629    /// `discharges_delivery_window` does not recognise is never retired, so it
2630    /// fires on every process and is counted as a regulatory violation each
2631    /// time — which is precisely how `makod_aperak_missed_total` once came to
2632    /// count Strom processes rather than missed APERAKs.
2633    ///
2634    /// Adding a delivery window means adding a row here.
2635    #[test]
2636    fn every_delivery_window_label_is_discharged_by_its_message() {
2637        for (message_type, label) in [
2638            ("APERAK", mako_fristen::APERAK_STROM_WINDOW_LABEL),
2639            ("APERAK", mako_fristen::APERAK_GAS_FOLGEPROZESS_LABEL),
2640            ("APERAK", mako_fristen::APERAK_GAS_INITIALPROZESS_LABEL),
2641            ("CONTRL", mako_fristen::CONTRL_FRIST_LABEL),
2642        ] {
2643            assert!(
2644                mako_fristen::discharges_delivery_window(message_type, label),
2645                "delivering {message_type} must discharge `{label}`, or the window \
2646                 outlives its obligation and alerts on every process",
2647            );
2648        }
2649    }
2650
2651    // ── Retry-budget classification ───────────────────────────────────────────
2652
2653    /// Sink double that records every rejection's attempt count.
2654    #[derive(Default)]
2655    struct RecordingSink(std::sync::Mutex<Vec<u32>>);
2656    impl crate::dead_letter::DeadLetterSink for std::sync::Arc<RecordingSink> {
2657        fn reject(&self, reason: &crate::dead_letter::DeadLetterReason) {
2658            if let crate::dead_letter::DeadLetterReason::OutboxExhausted { attempts, .. } = reason {
2659                self.0
2660                    .lock()
2661                    .unwrap_or_else(std::sync::PoisonError::into_inner)
2662                    .push(*attempts);
2663            }
2664        }
2665    }
2666
2667    struct NoRenderer;
2668    impl As4Sender for NoRenderer {
2669        async fn send(&self, msg: &crate::outbox::OutboxMessage) -> Result<(), EngineError> {
2670            Err(EngineError::RendererNotImplemented {
2671                message_type: msg.message_type.as_ref().into(),
2672                message_id: msg.message_id.to_string().into(),
2673            })
2674        }
2675    }
2676
2677    async fn drive_worker<S: As4Sender>(
2678        sender: S,
2679        msg: crate::outbox::OutboxMessage,
2680    ) -> (InMemoryOutboxStore, std::sync::Arc<RecordingSink>) {
2681        let outbox = InMemoryOutboxStore::new();
2682        outbox.enqueue(std::slice::from_ref(&msg)).await.unwrap();
2683        let sink = std::sync::Arc::new(RecordingSink::default());
2684        let worker = OutboxWorker {
2685            store: outbox.clone(),
2686            sender,
2687            deadline_store: InMemoryDeadlineStore::new(),
2688            batch_size: 10,
2689            poll_interval: std::time::Duration::from_millis(5),
2690            max_attempts: 48,
2691            max_retry_window: std::time::Duration::from_secs(72 * 3600),
2692            dead_letter_sink: std::sync::Arc::new(std::sync::Arc::clone(&sink)),
2693            heartbeat: None,
2694            shutdown: None,
2695        };
2696        let _ = tokio::time::timeout(std::time::Duration::from_millis(300), worker.run()).await;
2697        (outbox, sink)
2698    }
2699
2700    /// `RendererNotImplemented` is documented as permanent — the worker must
2701    /// dead-letter it on the *first* attempt, not burn the retry budget on a
2702    /// failure that cannot heal between attempts. Until the permanent arm
2703    /// matched it, this promise was broken.
2704    #[tokio::test(start_paused = true)]
2705    async fn a_missing_renderer_dead_letters_without_retrying() {
2706        let stream_id = crate::ids::StreamId::new("test-renderer-missing");
2707        let msg = outbox_message(&stream_id, "MSCONS");
2708        let (outbox, sink) = drive_worker(NoRenderer, msg).await;
2709
2710        assert!(
2711            outbox.pending_now(10).await.unwrap().is_empty(),
2712            "the message must be acknowledged, not left for another attempt",
2713        );
2714        let rejections = sink
2715            .0
2716            .lock()
2717            .unwrap_or_else(std::sync::PoisonError::into_inner)
2718            .clone();
2719        assert_eq!(
2720            rejections,
2721            vec![0],
2722            "exactly one dead-letter, on the first attempt (attempt_count 0)",
2723        );
2724    }
2725
2726    /// The retry budget is a *window*, not a count: a message whose age has
2727    /// exceeded it after at least one attempt is dead-lettered even though the
2728    /// attempt belt is nowhere near exhausted — full-jitter backoff makes a
2729    /// count no proxy for the 72 h duty.
2730    #[tokio::test(start_paused = true)]
2731    async fn an_aged_message_with_a_prior_attempt_is_dead_lettered() {
2732        let stream_id = crate::ids::StreamId::new("test-window-exhausted");
2733        let mut msg = outbox_message(&stream_id, "UTILMD");
2734        msg.created_at = time::OffsetDateTime::now_utc() - time::Duration::hours(73);
2735        msg.attempt_count = 1;
2736        let (outbox, sink) = drive_worker(AlwaysDelivers, msg).await;
2737
2738        assert!(
2739            outbox.pending_now(10).await.unwrap().is_empty(),
2740            "the exhausted message must leave the outbox",
2741        );
2742        let rejections = sink
2743            .0
2744            .lock()
2745            .unwrap_or_else(std::sync::PoisonError::into_inner)
2746            .clone();
2747        assert_eq!(
2748            rejections,
2749            vec![1],
2750            "the window, not the attempt belt, must have dead-lettered it",
2751        );
2752    }
2753
2754    /// A message that aged past the window while the worker was down still
2755    /// gets its first try — the window is only consulted after an attempt, so
2756    /// downtime never buries a message unsent.
2757    #[tokio::test(start_paused = true)]
2758    async fn an_aged_message_with_no_attempts_is_still_tried_once() {
2759        let stream_id = crate::ids::StreamId::new("test-aged-first-try");
2760        let mut msg = outbox_message(&stream_id, "UTILMD");
2761        msg.created_at = time::OffsetDateTime::now_utc() - time::Duration::hours(200);
2762        let (outbox, sink) = drive_worker(AlwaysDelivers, msg).await;
2763
2764        assert!(
2765            outbox.pending_now(10).await.unwrap().is_empty(),
2766            "the message must have been delivered and acknowledged",
2767        );
2768        assert!(
2769            sink.0
2770                .lock()
2771                .unwrap_or_else(std::sync::PoisonError::into_inner)
2772                .is_empty(),
2773            "delivery, not dead-lettering: age alone must never bury a message",
2774        );
2775    }
2776}