Skip to main content

mako_engine/
builder.rs

1//! [`EngineModule`] trait, [`EngineBuilder`], and [`EngineContext`].
2//!
3// Allow using deprecated Noop stores as *type-level defaults* in EngineBuilder / EngineContext
4// generic parameters.  The types are deprecated to prevent instantiation in production code,
5// but using them as default type parameters in struct definitions (not instantiating them) is
6// the intended pattern for the type-state builder API.
7#![allow(deprecated)]
8//!
9//! # Summary
10//!
11//! `EngineBuilder` assembles all engine infrastructure into a single
12//! [`EngineContext`] value. Domain modules (GPKE, WiM, GeLi Gas, …) register
13//! themselves at startup via the [`EngineModule`] trait, making their names
14//! visible in diagnostics and health checks.
15//!
16//! # Type-state guarantee
17//!
18//! [`EngineBuilder::build`] is only available when the event store type
19//! parameter `ES` implements [`EventStore`]. Forgetting to call
20//! [`with_event_store`] is a **compile-time error**, not a runtime panic.
21//!
22//! All other stores default to their respective `Noop` implementations:
23//!
24//! | Store | Default |
25//! |-------|---------|
26//! | Snapshot store | [`NoopSnapshotStore`] |
27//! | Outbox store | [`NoopOutboxStore`] |
28//! | Deadline store | [`NoopDeadlineStore`] |
29//! | Process registry | [`NoopProcessRegistry`] |
30//!
31//! # Assembly example
32//!
33//! ```rust,ignore
34//! use mako_engine::builder::{EngineBuilder, EngineModule};
35//! use mako_engine::event_store::InMemoryEventStore;
36//! use mako_engine::outbox::InMemoryOutboxStore;
37//! use mako_engine::deadline::InMemoryDeadlineStore;
38//! use mako_engine::registry::InMemoryProcessRegistry;
39//! use mako_engine::snapshot::InMemorySnapshotStore;
40//!
41//! struct GpkeModule;
42//! impl EngineModule for GpkeModule { fn name(&self) -> &'static str { "gpke" } }
43//!
44//! let ctx = EngineBuilder::new()
45//!     .with_event_store(InMemoryEventStore::new())
46//!     .with_snapshot_store(InMemorySnapshotStore::new())
47//!     .with_outbox_store(InMemoryOutboxStore::new())
48//!     .with_deadline_store(InMemoryDeadlineStore::new())
49//!     .with_registry(InMemoryProcessRegistry::new())
50//!     .register(Box::new(GpkeModule))
51//!     .build();
52//!
53//! // Spawn a fresh process:
54//! let p = ctx.spawn::<SupplierChangeWorkflow>(tenant_id, workflow_id);
55//! p.execute(ReceiveUtilmd { .. }).await?;
56//!
57//! // Resume an existing process from a persisted identity:
58//! let identity = ctx.registry.lookup(&conv_id.to_string()).await?.unwrap();
59//! let p = ctx.resume::<SupplierChangeWorkflow>(identity);
60//!
61//! // Access stores for delivery workers / schedulers:
62//! let pending = ctx.outbox_store.pending_now(50).await?;
63//! let overdue = ctx.deadline_store.due_now(50).await?;
64//! ```
65//!
66//! [`with_event_store`]: EngineBuilder::with_event_store
67
68// Type-state generics can produce long signatures that trip up the
69// `type_complexity` lint; suppress it for this module only.
70#![allow(clippy::type_complexity)]
71
72// The Noop* types are marked #[deprecated] to guard against accidental
73// production use.  The builder is the only place they're instantiated as
74// defaults; suppress the lint here explicitly.
75#[allow(deprecated)]
76use crate::{
77    dead_letter::{DeadLetterSink, LogDeadLetterSink},
78    deadline::{Deadline, DeadlineStore, NoopDeadlineStore},
79    error::EngineError,
80    event_store::EventStore,
81    ids::{ProcessIdentity, TenantId},
82    marktrolle::DeploymentRoles,
83    outbox::{NoopOutboxStore, OutboxMessage, OutboxStore},
84    pid_router::PidRouter,
85    process::Process,
86    registry::{NoopProcessRegistry, ProcessRegistry},
87    snapshot::{NoopSnapshotStore, SnapshotStore},
88    version::WorkflowId,
89    workflow::Workflow,
90};
91
92use std::sync::Arc;
93
94// ── EngineModule ──────────────────────────────────────────────────────────────
95
96/// A self-contained domain module that registers with the engine at startup.
97///
98/// Domain crates implement this trait to declare their presence in the engine.
99/// The module name is surfaced in [`EngineContext::registered_modules`] for
100/// diagnostics, health checks, and log output.
101///
102/// ## Startup validation
103///
104/// Override [`configure`] to perform adapter coverage checks at engine startup
105/// time. The engine calls [`configure`] for every registered module during
106/// [`EngineBuilder::build`] and panics with an actionable message if any
107/// module returns `Err`. This surfaces missing adapter registrations as a
108/// startup failure rather than a silent runtime error.
109///
110/// ## Example
111///
112/// ```rust,ignore
113/// pub struct GpkeModule;
114///
115/// impl EngineModule for GpkeModule {
116///     fn name(&self) -> &'static str { "gpke" }
117///
118///     fn configure(&self) -> Result<(), String> {
119///         // Validate that every known BDEW format version has an adapter:
120///         GPKE_ADAPTER_REGISTRY
121///             .validate_policy(&GpkeWorkflow::version_policy(), &KNOWN_FVS)
122///             .map_err(|uncovered| format!(
123///                 "gpke: missing adapters for format versions: {:?}",
124///                 uncovered
125///             ))
126///     }
127/// }
128///
129/// let ctx = EngineBuilder::new()
130///     .with_event_store(my_store)
131///     .register(Box::new(GpkeModule))
132///     .build(); // panics if GpkeModule::configure returns Err
133///
134/// assert_eq!(ctx.registered_modules(), &["gpke"]);
135/// ```
136///
137/// [`configure`]: EngineModule::configure
138pub trait EngineModule: Send + 'static {
139    /// Stable, unique name for this domain module.
140    ///
141    /// Used in diagnostics, health checks, and structured log output.
142    /// Choose a short lowercase identifier (e.g. `"gpke"`, `"wim"`,
143    /// `"geli"`).
144    fn name(&self) -> &'static str;
145
146    /// Register all PIDs this module handles into the shared [`PidRouter`].
147    ///
148    /// # Mutability contract
149    ///
150    /// This method is called **exactly once** by [`EngineBuilder::build`],
151    /// before the resulting [`EngineContext`] is handed to the caller. The
152    /// `&mut PidRouter` reference is only available here, at build time.
153    /// After `build` returns the router is **sealed** — the engine provides
154    /// only a shared `&PidRouter` reference, with no mutation path at runtime.
155    ///
156    /// Consequence: **all PIDs a module will ever need must be registered
157    /// here**. Do not attempt to register PIDs lazily from async handlers or
158    /// after the engine has started — there is no API for that by design.
159    ///
160    /// Duplicate registrations (same PID from two modules) silently overwrite
161    /// the previous mapping; the last module to register wins. Use
162    /// `cargo xtask validate-pruefids` to catch accidental PID conflicts
163    /// between modules before they reach production.
164    ///
165    /// For role-conditional registration (PIDs that should only be active for
166    /// specific BDEW Marktrollen), override [`register_pids_with_roles`] instead.
167    ///
168    /// # Example
169    ///
170    /// ```rust,ignore
171    /// fn register_pids(&self, router: &mut PidRouter) {
172    ///     // GPKE Lieferantenwechsel / Lieferbeginn (BK6-22-024, PIDs 55001, 55002, 55017)
173    ///     for &pid in &[55001_u32, 55002, 55017] {
174    ///         router.register(pid, "gpke-supplier-change");
175    ///     }
176    /// }
177    /// ```
178    ///
179    /// [`register_pids_with_roles`]: EngineModule::register_pids_with_roles
180    fn register_pids(&self, _router: &mut PidRouter) {}
181
182    /// Register PIDs with role-context awareness.
183    ///
184    /// This is the **preferred override** for modules that have role-conditional
185    /// PID registrations — PIDs that should only be active when this `makod`
186    /// instance holds a specific [`Marktrolle`].
187    ///
188    /// The default implementation calls [`register_pids`] (role-agnostic) so
189    /// existing modules that override `register_pids` continue to work without
190    /// changes.
191    ///
192    /// Override this method instead of `register_pids` when any PID registration
193    /// should be conditional on the deployment role:
194    ///
195    /// ```rust,ignore
196    /// use mako_engine::marktrolle::Marktrolle;
197    ///
198    /// fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
199    ///     // Always register: 55001, 55002 (not role-specific)
200    ///     for pid in [55001_u32, 55002] { router.register_with_module(pid, "gpke-supplier-change", self.name()); }
201    ///
202    ///     // Only when NB role: 19001/19002 inbound ORDRSP from MSB
203    ///     if roles.contains(Marktrolle::Nb) {
204    ///         for pid in [19001_u32, 19002] { router.register_with_module(pid, "gpke-konfiguration", self.name()); }
205    ///     }
206    /// }
207    /// ```
208    ///
209    /// # Conflict guard
210    ///
211    /// Use [`PidRouter::register_with_module`] (not `register`) inside this
212    /// method. The conflict guard panics at build time if two modules register
213    /// the same PID to different workflows — this makes role misconfigurations
214    /// visible at startup rather than silently misrouting messages.
215    ///
216    /// [`Marktrolle`]: crate::marktrolle::Marktrolle
217    /// [`register_pids`]: EngineModule::register_pids
218    fn register_pids_with_roles(&self, router: &mut PidRouter, _roles: &DeploymentRoles) {
219        self.register_pids(router);
220    }
221
222    /// Workflow names this module handles for deadline dispatch.
223    ///
224    /// Return the same name strings that [`register_pids`] maps PIDs to.
225    /// These names are stored in [`EngineContext::registered_workflows`] and
226    /// used to validate that every workflow that has deadlines scheduled is
227    /// covered by the deadline scheduler dispatch function at runtime.
228    ///
229    /// The default implementation returns an empty slice. Override it to
230    /// declare all workflow names that may fire deadlines:
231    ///
232    /// ```rust,ignore
233    /// fn workflow_names(&self) -> &'static [&'static str] {
234    ///     &["gpke-supplier-change", "gpke-abrechnung"]
235    /// }
236    /// ```
237    ///
238    /// [`register_pids`]: EngineModule::register_pids
239    /// [`EngineContext::registered_workflows`]: crate::builder::EngineContext::registered_workflows
240    fn workflow_names(&self) -> &'static [&'static str] {
241        &[]
242    }
243
244    /// Declare the EDIFACT profile types this module requires at runtime.
245    ///
246    /// Returning a non-empty slice causes [`EngineBuilder::build`] to call the
247    /// registered profile validator for each requirement.  If no active profile
248    /// exists for a required message type, `build` panics with an actionable
249    /// error so deployment fails fast rather than silently.
250    ///
251    /// Domain crates declare their format requirements here rather than
252    /// calling `edi_energy::registry::ReleaseRegistry::global()` inside
253    /// `configure()`, so `edi-energy` stays out of their production
254    /// `[dependencies]`.
255    ///
256    /// ```rust,ignore
257    /// fn profile_requirements(&self) -> &'static [ProfileRequirement] {
258    ///     &[
259    ///         ProfileRequirement { message_type: "UTILMD", label: "UTILMD Strom (GPKE)" },
260    ///         ProfileRequirement { message_type: "INVOIC", label: "INVOIC Abrechnung (GPKE)" },
261    ///     ]
262    /// }
263    /// ```
264    ///
265    /// [`ProfileRequirement`]: crate::profile::ProfileRequirement
266    fn profile_requirements(&self) -> &'static [crate::profile::ProfileRequirement] {
267        &[]
268    }
269
270    /// Validate adapter coverage and configuration at engine startup.
271    ///
272    /// Called by [`EngineBuilder::build`] after all modules are registered.
273    /// Return `Ok(())` when the module is fully configured. Return `Err(msg)`
274    /// with an actionable description when an adapter or configuration is
275    /// missing — the engine will panic with that message so the deployment
276    /// fails early rather than silently.
277    ///
278    /// The default implementation is a no-op (always returns `Ok(())`).
279    /// Override it in domain crates to call
280    /// [`AdapterRegistry::validate_policy`] and emit structured errors.
281    ///
282    /// Note: if your validation needs access to the edi-energy profile
283    /// registry, use [`profile_requirements`] instead — it does not require
284    /// importing `edi-energy` in domain crates.
285    ///
286    /// [`AdapterRegistry::validate_policy`]: crate::message_adapter::AdapterRegistry::validate_policy
287    /// [`profile_requirements`]: EngineModule::profile_requirements
288    ///
289    /// # Errors
290    ///
291    /// Returns a descriptive error string when the module's configuration is invalid.
292    fn configure(&self) -> Result<(), String> {
293        Ok(())
294    }
295}
296
297// ── EngineContext ─────────────────────────────────────────────────────────────
298
299/// Assembled engine infrastructure returned by [`EngineBuilder::build`].
300///
301/// `EngineContext` bundles all stores and the process registry into a single
302/// value. It is the root dependency for:
303///
304/// - Spawning new processes ([`spawn`])
305/// - Resuming existing processes ([`resume`])
306/// - Running outbox delivery workers (`outbox_store.pending_now(…)`)
307/// - Driving the deadline scheduler (`deadline_store.due_now(…)`)
308///
309/// ## Generic parameters
310///
311/// | Param | Role | Default |
312/// |-------|------|---------|
313/// | `ES`  | [`EventStore`] backend | — (required) |
314/// | `SS`  | [`SnapshotStore`] backend | [`NoopSnapshotStore`] |
315/// | `OS`  | [`OutboxStore`] backend  | [`NoopOutboxStore`]   |
316/// | `DS`  | [`DeadlineStore`] backend | [`NoopDeadlineStore`] |
317/// | `PR`  | [`ProcessRegistry`] backend | [`NoopProcessRegistry`] |
318///
319/// In most codebases all type parameters are inferred from the builder calls.
320///
321/// [`spawn`]: EngineContext::spawn
322/// [`resume`]: EngineContext::resume
323pub struct EngineContext<
324    ES,
325    SS = NoopSnapshotStore,
326    OS = NoopOutboxStore,
327    DS = NoopDeadlineStore,
328    PR = NoopProcessRegistry,
329> {
330    event_store: Arc<ES>,
331    snapshot_store: SS,
332    outbox_store: OS,
333    deadline_store: DS,
334    registry: PR,
335    /// Dead-letter sink for unroutable or unprocessable inbound messages.
336    ///
337    /// Stored as `Arc<dyn DeadLetterSink>` so callers can share it across
338    /// tasks without an extra type parameter on `EngineContext`.
339    pub dead_letter_sink: Arc<dyn DeadLetterSink>,
340    /// PID-to-workflow routing table, populated from all registered modules.
341    pid_router: PidRouter,
342    registered_modules: Vec<&'static str>,
343    /// Workflow names declared by all registered modules via
344    /// [`EngineModule::workflow_names`]. Used to validate deadline scheduler
345    /// coverage at runtime (see [`EngineContext::registered_workflows`]).
346    registered_workflows: Vec<&'static str>,
347}
348
349// ── Type aliases ──────────────────────────────────────────────────────────────
350
351/// An [`EngineContext`] with all optional subsystems disabled.
352///
353/// Uses `NoopSnapshotStore` and, in `testing`-enabled builds, Noop
354/// implementations for outbox, deadline, and process registry. Suitable for
355/// tests and minimal deployments where only a durable event store is required.
356///
357/// All five type parameters are inferred from context when used with
358/// [`EngineBuilder`]:
359///
360/// ```rust,ignore
361/// // Only available in test / testing-feature builds:
362/// use mako_engine::builder::{EngineBuilder, MinimalEngine};
363/// use mako_engine::event_store::InMemoryEventStore;
364///
365/// let ctx: MinimalEngine<InMemoryEventStore> = EngineBuilder::new()
366///     .with_event_store(InMemoryEventStore::new())
367///     .build();
368/// ```
369pub type MinimalEngine<ES> = EngineContext<ES>;
370
371impl<ES, SS, OS, DS, PR> EngineContext<ES, SS, OS, DS, PR>
372where
373    ES: EventStore,
374{
375    /// Spawn a new process and return a typed `Process<W, Arc<ES>>` handle.
376    ///
377    /// No `ES: Clone` bound is required — the engine stores the event store
378    /// behind an `Arc` so spawning is always a cheap pointer clone.
379    ///
380    /// ```rust,ignore
381    /// let p = ctx.spawn::<SupplierChangeWorkflow>(tenant_id, workflow_id);
382    /// p.execute(ReceiveUtilmd { .. }).await?;
383    /// ```
384    #[must_use]
385    pub fn spawn<W: Workflow>(
386        &self,
387        tenant_id: TenantId,
388        workflow_id: WorkflowId,
389    ) -> Process<W, Arc<ES>> {
390        Process::new(Arc::clone(&self.event_store), tenant_id, workflow_id)
391    }
392
393    /// Resume an existing process from a [`ProcessIdentity`].
394    ///
395    /// ```rust,ignore
396    /// let identity = ctx.registry()
397    ///     .lookup(tenant_id, &conv_id.to_string())
398    ///     .await?
399    ///     .ok_or(EngineError::Registry("unknown conversation".into()))?;
400    /// let p = ctx.resume::<SupplierChangeWorkflow>(identity);
401    /// p.execute(HandleAperak { .. }).await?;
402    /// ```
403    #[must_use]
404    pub fn resume<W: Workflow>(&self, identity: ProcessIdentity) -> Process<W, Arc<ES>> {
405        Process::from_identity(Arc::clone(&self.event_store), identity)
406    }
407
408    /// Names of all domain modules registered with the builder, in
409    /// registration order.
410    #[must_use]
411    pub fn registered_modules(&self) -> &[&'static str] {
412        &self.registered_modules
413    }
414
415    /// Workflow names declared by all registered modules, in registration order.
416    ///
417    /// Use this in the deadline scheduler dispatch function to detect unknown
418    /// workflow names at startup. If a deadline fires for a workflow name that
419    /// is not in this list, the scheduler's dispatch function should emit an
420    /// error rather than silently dropping the deadline:
421    ///
422    /// ```rust,ignore
423    /// let known = ctx.registered_workflows().iter().copied().collect::<HashSet<_>>();
424    /// let scheduler = ctx.run_deadline_scheduler(
425    ///     move |deadline| {
426    ///         let wf = deadline.workflow_id().name.as_ref();
427    ///         if !known.contains(wf) {
428    ///             tracing::error!(workflow = %wf, "deadline fired for unregistered workflow");
429    ///             return Box::pin(async { Ok(()) });
430    ///         }
431    ///         // dispatch by workflow name …
432    ///         Box::pin(async { Ok(()) })
433    ///     },
434    ///     100,
435    ///     Duration::from_secs(30),
436    /// );
437    /// ```
438    #[must_use]
439    pub fn registered_workflows(&self) -> &[&'static str] {
440        &self.registered_workflows
441    }
442
443    /// The event store backend (behind an `Arc`).
444    #[must_use]
445    pub fn event_store(&self) -> &Arc<ES> {
446        &self.event_store
447    }
448
449    /// The snapshot store backend.
450    #[must_use]
451    pub fn snapshot_store(&self) -> &SS {
452        &self.snapshot_store
453    }
454
455    /// The outbox store backend.
456    ///
457    /// Poll `outbox_store().pending_now(limit)` in a background task to drain
458    /// the delivery queue.
459    #[must_use]
460    pub fn outbox_store(&self) -> &OS {
461        &self.outbox_store
462    }
463
464    /// The deadline store backend.
465    ///
466    /// Poll `deadline_store().due_now(limit)` in a background scheduler to
467    /// fire overdue process timers.
468    #[must_use]
469    pub fn deadline_store(&self) -> &DS {
470        &self.deadline_store
471    }
472
473    /// The process routing registry.
474    ///
475    /// Register a [`ProcessIdentity`] under a `(tenant_id, key)` pair at
476    /// process creation, then `lookup` it when routing inbound messages.
477    #[must_use]
478    pub fn registry(&self) -> &PR {
479        &self.registry
480    }
481
482    /// The dead-letter sink for unroutable or unprocessable messages.
483    ///
484    /// Call [`DeadLetterSink::reject`] when an inbound message cannot be
485    /// dispatched to any workflow. The default sink emits `tracing::warn!`
486    /// so rejections are always visible in the log output.
487    #[must_use]
488    pub fn dead_letter_sink(&self) -> &Arc<dyn DeadLetterSink> {
489        &self.dead_letter_sink
490    }
491
492    /// Assert that no Noop store is active — call this during production startup.
493    ///
494    /// Checks the type names of `OS`, `DS`, and `PR` against the string `"Noop"`.
495    /// Panics with a human-readable message if any match, directing the operator
496    /// to configure a persistent backend.
497    ///
498    /// # When to call
499    ///
500    /// Call this early in `makod`'s startup path (and `--check` mode) to catch
501    /// deployments where a Noop store was accidentally wired — e.g. the
502    /// `[outbox]`, `[deadline]`, or `[registry]` configuration section was
503    /// omitted from `makod.toml`.  The check is defence-in-depth: in release
504    /// builds without the `testing` feature, Noop stores cannot implement the
505    /// required traits at all and the compiler would have already rejected them.
506    ///
507    /// # Panics
508    ///
509    /// Panics when any of `OS`, `DS`, or `PR` is a Noop implementation.
510    pub fn assert_production_stores(&self) {
511        let checks: &[(&str, &str)] = &[
512            ("OutboxStore", std::any::type_name::<OS>()),
513            ("DeadlineStore", std::any::type_name::<DS>()),
514            ("ProcessRegistry", std::any::type_name::<PR>()),
515        ];
516        for (trait_name, type_name) in checks {
517            assert!(
518                !type_name.contains("Noop"),
519                "makod: Noop{trait_name} is active — \
520                 configure a persistent {trait_name} backend in makod.toml. \
521                 Type resolved to: {type_name}"
522            );
523        }
524    }
525
526    /// The PID-to-workflow routing table.
527    ///
528    /// Populated **once** during [`EngineBuilder::build`] by calling
529    /// [`EngineModule::register_pids`] on every registered module in
530    /// registration order. After `build` returns the table is **sealed** —
531    /// it is read-only for the lifetime of the `EngineContext` and may be
532    /// freely shared across async tasks without synchronisation.
533    ///
534    /// # Mutability contract
535    ///
536    /// There is intentionally no `pid_router_mut()` accessor. Adding PIDs
537    /// after the engine is built would create a TOCTOU race between the
538    /// dispatch path (which calls `route(pid)`) and any hypothetical
539    /// concurrent mutator. Instead, register all PIDs during the build phase
540    /// via `EngineModule::register_pids`.
541    ///
542    /// If a new process family needs to be added without restarting the
543    /// binary, rebuild and restart `makod` — hot-swap of PID routing is not
544    /// supported.
545    ///
546    /// # Example — dispatch at the AS4 reception boundary
547    ///
548    /// ```rust,ignore
549    /// let workflow_name = ctx.pid_router().route(pid)
550    ///     .ok_or_else(|| EngineError::Workflow(WorkflowError::InvalidCommand(
551    ///         format!("no workflow registered for PID {pid}").into()
552    ///     )))?;
553    ///
554    /// match workflow_name {
555    ///     "gpke-supplier-change" => dispatch::<GpkeSupplierChangeWorkflow>(&ctx, pid, payload).await,
556    ///     "wim-device-change"    => dispatch::<WimDeviceChangeWorkflow>(&ctx, pid, payload).await,
557    ///     other => Err(EngineError::Workflow(WorkflowError::InvalidCommand(
558    ///         format!("unhandled workflow name: {other}").into()
559    ///     ))),
560    /// }
561    /// ```
562    #[must_use]
563    pub fn pid_router(&self) -> &PidRouter {
564        &self.pid_router
565    }
566}
567
568// ── As4Sender ─────────────────────────────────────────────────────────────────
569
570/// Sends a single AS4 / EDIINT-over-HTTP outbound message.
571///
572/// Implement this trait for your AS4 gateway client and pass it to
573/// [`EngineContext::run_outbox_worker`].
574///
575/// # Contract
576///
577/// Return `Ok(())` only after the message has been **durably accepted** by the
578/// receiving MSH.  Return `Err(…)` on transient or permanent failure — the
579/// outbox worker calls [`OutboxStore::reschedule`] so the message is retried.
580pub trait As4Sender: Send + Sync + 'static {
581    /// Transmit `msg` and return when the remote MSH has accepted it.
582    fn send(
583        &self,
584        msg: &OutboxMessage,
585    ) -> impl std::future::Future<Output = Result<(), EngineError>> + Send;
586}
587
588// ── OutboxWorker ──────────────────────────────────────────────────────────────
589
590/// A background worker that drains the outbox by polling pending
591/// [`OutboxMessage`]s and dispatching them via an [`As4Sender`].
592///
593/// Obtain via [`EngineContext::run_outbox_worker`] and drive by spawning
594/// [`OutboxWorker::run`] in a Tokio task.
595///
596/// # Polling behaviour
597///
598/// When the poll returns an empty batch the worker sleeps for `poll_interval`
599/// before polling again.  Non-empty batches are processed immediately.
600///
601/// # Error handling
602///
603/// Successful sends are acknowledged via [`OutboxStore::acknowledge`].
604/// Failed sends are rescheduled via [`OutboxStore::reschedule`] using
605/// **full-jitter exponential backoff**: `delay = rand(0, min(MAX, BASE * 2^n))`
606/// where `n = attempt_count`. This avoids thundering-herd when multiple
607/// `makod` instances restart simultaneously after a receiver outage.
608///
609/// When `attempt_count >= max_attempts`, the message is **acknowledged** (removed
610/// from the outbox) and a [`DeadLetterReason::OutboxExhausted`] record is written
611/// to the dead-letter sink. This prevents permanently-undeliverable messages
612/// from clogging the outbox forever.
613///
614/// All errors are emitted as structured `tracing` events at `warn` / `error`
615/// level rather than `eprintln!`, so they appear in the application's log
616/// pipeline with full context (message_id, error).
617///
618/// # Example
619///
620/// ```rust,ignore
621/// use std::time::Duration;
622///
623/// let worker = ctx.run_outbox_worker(my_sender, 50, Duration::from_secs(1));
624/// tokio::spawn(async move { worker.run().await });
625/// ```
626///
627/// [`DeadLetterReason::OutboxExhausted`]: crate::dead_letter::DeadLetterReason::OutboxExhausted
628pub struct OutboxWorker<OS: OutboxStore, S: As4Sender, DS: DeadlineStore> {
629    store: OS,
630    sender: S,
631    /// Used to discharge a delivery-window deadline once the message it was
632    /// watching has actually been sent — see [`OutboxWorker::run`].
633    deadline_store: DS,
634    batch_size: usize,
635    poll_interval: std::time::Duration,
636    /// Maximum total delivery attempts before a message is dead-lettered — a
637    /// runaway belt, not the budget. The budget is [`Self::max_retry_window`]:
638    /// the backoff is full-jitter, so an attempt *count* cannot promise a
639    /// retry *duration*, and the BDEW retry duty is stated in hours.
640    max_attempts: u32,
641    /// Maximum age (from `created_at`) a message is retried for before it is
642    /// dead-lettered. This is what honours a time-stated retry duty (BDEW AS4
643    /// Kommunikationshandbuch: 72 h for unacknowledged messages) — see
644    /// `mako_as4::constants::MAX_RETRY_DURATION_SECS`.
645    ///
646    /// Checked only after at least one attempt: a message that aged in a
647    /// stopped worker still gets its first try rather than being buried
648    /// unsent.
649    max_retry_window: std::time::Duration,
650    /// Sink for messages that exceed `max_attempts` or `max_retry_window`.
651    dead_letter_sink: std::sync::Arc<dyn crate::dead_letter::DeadLetterSink>,
652    /// Optional liveness heartbeat — stores the current UTC Unix timestamp
653    /// (seconds) after each poll cycle so health probes can detect stale workers.
654    heartbeat: Option<std::sync::Arc<std::sync::atomic::AtomicI64>>,
655    /// Graceful-shutdown signal. When cancelled the worker finishes the message
656    /// it is delivering, then returns from [`OutboxWorker::run`] — see
657    /// [`OutboxWorker::with_shutdown`].
658    shutdown: Option<tokio_util::sync::CancellationToken>,
659}
660
661/// Sleep for `dur`, returning early if `token` is cancelled.
662///
663/// Returns `true` when the sleep completed and the caller should keep looping,
664/// `false` when the token was cancelled and the caller must return.
665///
666/// A worker that sleeps on a bare `tokio::time::sleep` cannot observe a
667/// shutdown until its poll interval elapses. For the deadline scheduler that is
668/// 30 seconds by default — longer than a typical container termination grace
669/// period, which turns a graceful drain into a SIGKILL.
670///
671/// Public so that binaries running their own poll-loop workers alongside the
672/// engine's (projection catch-up, webhook delivery, retention purges) can honour
673/// the same token and stop before the store is closed.
674pub async fn sleep_or_cancel(
675    dur: std::time::Duration,
676    token: Option<&tokio_util::sync::CancellationToken>,
677) -> bool {
678    let Some(t) = token else {
679        tokio::time::sleep(dur).await;
680        return true;
681    };
682    tokio::select! {
683        () = tokio::time::sleep(dur) => true,
684        () = t.cancelled() => false,
685    }
686}
687
688/// Compute a full-jitter exponential backoff delay.
689///
690/// `attempt` is the number of prior attempts (0 = first retry).
691/// `entropy` provides randomness; derive from a stable message identifier
692/// (e.g. hash of `message_id`) rather than the current timestamp — a
693/// timestamp-derived value is deterministic within a single batch, which
694/// defeats jitter when multiple messages fail simultaneously.
695///
696/// | attempt | window (s) | expected delay (s) |
697/// |---------|------------|-------------------|
698/// | 0       | 5          | 2.5               |
699/// | 1       | 10         | 5                 |
700/// | 2       | 20         | 10                |
701/// | 3       | 40         | 20                |
702/// | 4       | 80         | 40                |
703/// | 5+      | 300 (cap)  | 150               |
704fn backoff_delay(attempt: u32, entropy: u64) -> std::time::Duration {
705    const BASE_SECS: u64 = 5;
706    const MAX_SECS: u64 = 300;
707    // Exponential window: BASE * 2^attempt, capped at MAX.
708    let window = BASE_SECS
709        .saturating_mul(1u64.wrapping_shl(attempt.min(5)))
710        .min(MAX_SECS);
711    // Full jitter: uniform random in [0, window).
712    let jitter_secs = if window == 0 { 0 } else { entropy % window };
713    std::time::Duration::from_secs(jitter_secs)
714}
715
716impl<OS: OutboxStore, S: As4Sender, DS: DeadlineStore> OutboxWorker<OS, S, DS> {
717    /// Run the outbox drain loop until the shutdown token is cancelled.
718    ///
719    /// Without a token (see [`OutboxWorker::with_shutdown`]) the loop runs until
720    /// the task is aborted or the process exits. With one, cancellation is
721    /// observed between messages and during the idle sleep, so an in-flight
722    /// delivery is always finished and acknowledged before the worker returns —
723    /// dropping it mid-`send` would risk a duplicate AS4 delivery on restart.
724    ///
725    /// # Panics
726    ///
727    /// Panics if `time::Duration::try_from(delay)` overflows (unreachable for
728    /// the delay values produced by `backoff_delay`).
729    #[allow(clippy::too_many_lines)]
730    pub async fn run(self) {
731        loop {
732            if self
733                .shutdown
734                .as_ref()
735                .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
736            {
737                tracing::info!("outbox worker: shutdown signalled; stopping");
738                return;
739            }
740            // Tick liveness at the *start* of every poll cycle, ahead of the
741            // early-`continue` paths below.  An idle worker (empty outbox) and
742            // one retrying after a store error are both alive and must keep
743            // ticking; only a worker genuinely hung inside an `.await` stops.
744            if let Some(ref hb) = self.heartbeat {
745                hb.store(
746                    time::OffsetDateTime::now_utc().unix_timestamp(),
747                    std::sync::atomic::Ordering::Relaxed,
748                );
749            }
750
751            let batch = match self.store.pending_now(self.batch_size).await {
752                Ok(b) => b,
753                Err(e) => {
754                    tracing::warn!(error = %e, "outbox worker: store error polling pending messages (will retry)");
755                    if !sleep_or_cancel(self.poll_interval, self.shutdown.as_ref()).await {
756                        return;
757                    }
758                    continue;
759                }
760            };
761
762            if batch.is_empty() {
763                if !sleep_or_cancel(self.poll_interval, self.shutdown.as_ref()).await {
764                    return;
765                }
766                continue;
767            }
768
769            for msg in batch {
770                // Between messages, not inside one: a `send` that is already in
771                // flight must run to its `acknowledge`, or the counterparty
772                // receives a message the outbox still believes is pending and
773                // redelivers it after the restart.
774                if self
775                    .shutdown
776                    .as_ref()
777                    .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
778                {
779                    tracing::info!(
780                        "outbox worker: shutdown signalled mid-batch; \
781                         remaining messages stay queued for the next start"
782                    );
783                    return;
784                }
785                // ── Retry budget ──────────────────────────────────────
786                // `attempt_count` starts at 0 and is incremented on each
787                // `reschedule` call. The message is permanently undeliverable
788                // when the retry *window* has elapsed (the BDEW duty is stated
789                // in hours, and full-jitter backoff makes a count no proxy for
790                // a duration) or the attempt belt is exhausted: acknowledge it
791                // (remove from outbox) and dead-letter it so the regulatory
792                // audit trail is preserved. The window is only consulted after
793                // a first attempt, so a message that aged while the worker was
794                // down still gets tried once.
795                let age = time::OffsetDateTime::now_utc() - msg.created_at;
796                let window_elapsed = msg.attempt_count > 0
797                    && age
798                        >= time::Duration::try_from(self.max_retry_window)
799                            .unwrap_or(time::Duration::hours(72));
800                if msg.attempt_count >= self.max_attempts || window_elapsed {
801                    tracing::error!(
802                        message_id   = %msg.message_id,
803                        message_type = %msg.message_type,
804                        recipient    = %msg.recipient,
805                        attempts     = msg.attempt_count,
806                        max_attempts = self.max_attempts,
807                        age_secs     = age.whole_seconds(),
808                        window_secs  = self.max_retry_window.as_secs(),
809                        "outbox worker: retry budget exhausted; dead-lettering message",
810                    );
811                    self.dead_letter_sink.reject(
812                        &crate::dead_letter::DeadLetterReason::OutboxExhausted {
813                            message_id: msg.message_id,
814                            message_type: msg.message_type.to_string(),
815                            recipient: msg.recipient.to_string(),
816                            last_error: format!(
817                                "delivery exhausted after {} attempts",
818                                msg.attempt_count
819                            ),
820                            attempts: msg.attempt_count,
821                        },
822                    );
823                    if let Err(e) = self.store.acknowledge(msg.message_id).await {
824                        tracing::error!(
825                            message_id = %msg.message_id,
826                            error = %e,
827                            "outbox worker: acknowledge after exhaust failed; message may reappear",
828                        );
829                    }
830                    continue;
831                }
832
833                match self.sender.send(&msg).await {
834                    Ok(()) => {
835                        if let Err(e) = self.store.acknowledge(msg.message_id).await {
836                            tracing::warn!(
837                                message_id = %msg.message_id,
838                                error = %e,
839                                "outbox worker: acknowledge failed",
840                            );
841                        }
842                        // CONTRL AHB 1.0 §1.2: the CONTRL must be delivered
843                        // within 6 wall-clock hours of interchange receipt.
844                        // `msg.created_at` is when the PendingOutbox was
845                        // materialised (which should equal the ingest timestamp
846                        // for transport-layer CONTRL obligations).
847                        if msg.message_type.as_ref() == "CONTRL" {
848                            let elapsed = time::OffsetDateTime::now_utc() - msg.created_at;
849                            if elapsed > time::Duration::hours(mako_fristen::CONTRL_FRIST_HOURS) {
850                                tracing::warn!(
851                                    message_id   = %msg.message_id,
852                                    elapsed_secs = elapsed.whole_seconds(),
853                                    max_secs     = mako_fristen::CONTRL_FRIST_HOURS * 3600,
854                                    "outbox worker: CONTRL delivered OUTSIDE the 6h Übertragungsfrist \
855                                     (CONTRL AHB 1.0 §1.2) — this is a BNetzA compliance violation"
856                                );
857                            }
858                        }
859                        // APERAK AHB 1.0 §2.4.1: Strom UTILMD/ORDERS APERAK must be
860                        // delivered within 45 minutes on weekdays, or by Sunday 12:00
861                        // if received on Saturday.  Log a compliance warning if the
862                        // delivery window was missed so operators can investigate.
863                        if msg.message_type.as_ref() == "APERAK" {
864                            let elapsed = time::OffsetDateTime::now_utc() - msg.created_at;
865                            if elapsed
866                                > time::Duration::minutes(
867                                    mako_fristen::APERAK_STROM_WEEKDAY_MINUTES,
868                                )
869                            {
870                                tracing::warn!(
871                                    message_id   = %msg.message_id,
872                                    elapsed_mins = elapsed.whole_minutes(),
873                                    "outbox worker: APERAK delivered after the 45-minute Strom \
874                                     sending window (APERAK AHB 1.0 §2.4.1) — \
875                                     check OutboxWorker and AS4 transport health"
876                                );
877                            }
878                        }
879                        // The message is out, so any delivery window that was
880                        // watching for it has been answered — retire it.
881                        //
882                        // Nothing else cancels these. A monitoring deadline that
883                        // outlives the obligation it monitors fires for every
884                        // process, including every one that answered on time,
885                        // and the scheduler cannot tell those apart because a
886                        // deadline reaching `due_now` is late by construction.
887                        // Leaving them registered turns the miss counters into
888                        // counts of *processes started*.
889                        self.discharge_delivery_window(&msg).await;
890                    }
891                    // Permanent error: dead-letter immediately without retrying.
892                    // PartnerUnknown requires operator intervention (add --as4-partner);
893                    // Serialization errors will never succeed on retry; a missing
894                    // wire-format renderer cannot appear between attempts — its own
895                    // documentation promises immediate dead-lettering, and until this
896                    // arm matched it, that promise was broken and the message burned
897                    // the whole retry budget first.
898                    Err(ref e)
899                        if e.is_partner_unknown()
900                            || e.is_renderer_not_implemented()
901                            || matches!(e, EngineError::Serialization(_)) =>
902                    {
903                        tracing::error!(
904                            message_id   = %msg.message_id,
905                            message_type = %msg.message_type,
906                            recipient    = %msg.recipient,
907                            error        = %e,
908                            "outbox worker: permanent send failure; dead-lettering without retry",
909                        );
910                        self.dead_letter_sink.reject(
911                            &crate::dead_letter::DeadLetterReason::OutboxExhausted {
912                                message_id: msg.message_id,
913                                message_type: msg.message_type.to_string(),
914                                recipient: msg.recipient.to_string(),
915                                last_error: e.to_string(),
916                                attempts: msg.attempt_count,
917                            },
918                        );
919                        if let Err(re) = self.store.acknowledge(msg.message_id).await {
920                            tracing::error!(
921                                message_id = %msg.message_id,
922                                error = %re,
923                                "outbox worker: acknowledge after permanent failure failed",
924                            );
925                        }
926                    }
927                    Err(e) => {
928                        // Stable jitter entropy derived from the UUID bytes of
929                        // `message_id`.  Using the last 8 bytes as a `u64` gives
930                        // uniform entropy across message IDs (UUIDs are random in
931                        // all 128 bits for v4) and is stable across Rust versions —
932                        // unlike `DefaultHasher`, whose algorithm is explicitly
933                        // documented as unstable.
934                        let entropy = {
935                            let uuid = msg.message_id.as_uuid();
936                            let bytes = uuid.as_bytes();
937                            u64::from_le_bytes(bytes[8..16].try_into().unwrap())
938                        };
939                        let delay = backoff_delay(msg.attempt_count, entropy);
940                        let retry_at = time::OffsetDateTime::now_utc()
941                            + time::Duration::try_from(delay).unwrap_or(time::Duration::minutes(5));
942                        tracing::warn!(
943                            message_id   = %msg.message_id,
944                            attempt      = msg.attempt_count,
945                            max_attempts = self.max_attempts,
946                            retry_in     = ?delay,
947                            error        = %e,
948                            "outbox worker: send failed; rescheduling with backoff",
949                        );
950                        if let Err(re) = self.store.reschedule(msg.message_id, retry_at).await {
951                            tracing::error!(
952                                message_id = %msg.message_id,
953                                error      = %re,
954                                "outbox worker: reschedule failed; message may be stuck",
955                            );
956                        }
957                    }
958                }
959            }
960        }
961    }
962}
963
964impl<ES, SS, OS, DS, PR> EngineContext<ES, SS, OS, DS, PR>
965where
966    ES: EventStore,
967    OS: OutboxStore + Clone,
968{
969    /// Construct an [`OutboxWorker`] that drains the outbox via `sender`.
970    ///
971    /// `batch_size` — messages fetched per poll cycle.
972    /// `poll_interval` — sleep duration when the batch is empty.
973    ///
974    /// `max_attempts` — attempt belt against runaway loops; the real budget is
975    /// `max_retry_window`, the message age after which delivery is abandoned.
976    /// The BDEW AS4 retry duty is stated in *hours* (72 h for unacknowledged
977    /// messages — `mako_as4::constants::MAX_RETRY_DURATION_SECS`), and the
978    /// full-jitter backoff makes an attempt count no proxy for a duration, so
979    /// both are taken and either exhausts the message.
980    ///
981    /// ```rust,ignore
982    /// use std::time::Duration;
983    ///
984    /// let worker = ctx.run_outbox_worker(
985    ///     my_sender, 50, Duration::from_secs(1),
986    ///     10_000, Duration::from_secs(72 * 3600),
987    /// );
988    /// tokio::spawn(async move { worker.run().await });
989    /// ```
990    #[must_use]
991    pub fn run_outbox_worker<S: As4Sender>(
992        &self,
993        sender: S,
994        batch_size: usize,
995        poll_interval: std::time::Duration,
996        max_attempts: u32,
997        max_retry_window: std::time::Duration,
998    ) -> OutboxWorker<OS, S, DS>
999    where
1000        DS: DeadlineStore + Clone,
1001    {
1002        OutboxWorker {
1003            store: self.outbox_store.clone(),
1004            sender,
1005            deadline_store: self.deadline_store.clone(),
1006            batch_size,
1007            poll_interval,
1008            max_attempts,
1009            max_retry_window,
1010            dead_letter_sink: self.dead_letter_sink.clone(),
1011            heartbeat: None,
1012            shutdown: None,
1013        }
1014    }
1015}
1016
1017impl<OS: OutboxStore, S: As4Sender, DS: DeadlineStore> OutboxWorker<OS, S, DS> {
1018    /// Attach a liveness heartbeat to this worker.
1019    ///
1020    /// The worker will store the current UTC Unix timestamp (seconds) into
1021    /// `heartbeat` at the end of every poll cycle.  Pass the same
1022    /// `Arc<AtomicI64>` to the health endpoint so it can detect stale workers.
1023    #[must_use]
1024    pub fn with_heartbeat(
1025        mut self,
1026        heartbeat: std::sync::Arc<std::sync::atomic::AtomicI64>,
1027    ) -> Self {
1028        self.heartbeat = Some(heartbeat);
1029        self
1030    }
1031
1032    /// Attach a graceful-shutdown token.
1033    ///
1034    /// Cancelling it makes [`OutboxWorker::run`] return at the next message
1035    /// boundary or immediately out of its idle sleep. Await the worker's
1036    /// `JoinHandle` afterwards: the point of the token is that the caller can
1037    /// close the event store *after* the worker has stopped writing to it.
1038    #[must_use]
1039    pub fn with_shutdown(mut self, shutdown: tokio_util::sync::CancellationToken) -> Self {
1040        self.shutdown = Some(shutdown);
1041        self
1042    }
1043
1044    /// Retire the delivery window `msg` was being watched by, if one is open.
1045    ///
1046    /// A delivery-window deadline exists to answer one question: *did this
1047    /// message go out in time?* Once it has gone out the question is settled,
1048    /// and leaving the deadline registered only guarantees a false alarm later.
1049    /// [`fristen::discharges_delivery_window`] decides which labels a given
1050    /// message type answers for; deadlines that merely share the stream (a
1051    /// process-response window, say) are left alone.
1052    ///
1053    /// Best-effort: a failure here costs a spurious alert at the window's close,
1054    /// never a lost or duplicated message, so it is logged rather than
1055    /// propagated — the delivery itself has already been acknowledged.
1056    ///
1057    /// [`fristen::discharges_delivery_window`]: mako_fristen::discharges_delivery_window
1058    async fn discharge_delivery_window(&self, msg: &crate::outbox::OutboxMessage) {
1059        let open = match self.deadline_store.for_stream(&msg.stream_id).await {
1060            Ok(deadlines) => deadlines,
1061            Err(e) => {
1062                tracing::warn!(
1063                    message_id   = %msg.message_id,
1064                    message_type = %msg.message_type,
1065                    error        = %e,
1066                    "outbox worker: could not read deadlines to discharge the delivery \
1067                     window; it may fire a spurious regulatory alert",
1068                );
1069                return;
1070            }
1071        };
1072
1073        for deadline in open
1074            .iter()
1075            .filter(|d| mako_fristen::discharges_delivery_window(&msg.message_type, d.label()))
1076        {
1077            if let Err(e) = self.deadline_store.cancel(deadline.deadline_id()).await {
1078                tracing::warn!(
1079                    message_id  = %msg.message_id,
1080                    deadline_id = %deadline.deadline_id(),
1081                    label       = %deadline.label(),
1082                    error       = %e,
1083                    "outbox worker: could not discharge the delivery window; \
1084                     it may fire a spurious regulatory alert",
1085                );
1086            } else {
1087                tracing::debug!(
1088                    message_id   = %msg.message_id,
1089                    message_type = %msg.message_type,
1090                    deadline_id  = %deadline.deadline_id(),
1091                    label        = %deadline.label(),
1092                    "outbox worker: message delivered — delivery window discharged",
1093                );
1094            }
1095        }
1096    }
1097}
1098
1099impl<ES, SS, OS, DS, PR> std::fmt::Debug for EngineContext<ES, SS, OS, DS, PR>
1100where
1101    ES: std::fmt::Debug,
1102    SS: std::fmt::Debug,
1103    OS: std::fmt::Debug,
1104    DS: std::fmt::Debug,
1105    PR: std::fmt::Debug,
1106{
1107    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1108        f.debug_struct("EngineContext")
1109            .field("registered_modules", &self.registered_modules)
1110            .field("registered_workflows", &self.registered_workflows)
1111            .field("pid_router_len", &self.pid_router.len())
1112            .finish_non_exhaustive()
1113    }
1114}
1115
1116// ── NoopAs4Sender / LogAs4Sender ──────────────────────────────────────────────
1117
1118/// An [`As4Sender`] that succeeds immediately without sending anything.
1119///
1120/// Use in tests and environments where outbound AS4 delivery is not yet
1121/// wired. All outbox messages are acknowledged (removed from the queue)
1122/// without being transmitted.
1123///
1124/// # ⚠️ Data loss warning
1125///
1126/// Every outbox message is **silently discarded** — no EDIFACT message is
1127/// sent to any counterparty. Do not use in production.
1128#[derive(Debug, Clone, Copy, Default)]
1129#[must_use = "NoopAs4Sender discards all outbound messages silently — use a real AS4 gateway in production"]
1130#[cfg_attr(
1131    not(any(test, feature = "testing")),
1132    deprecated = "NoopAs4Sender must not be wired in production builds; every \
1133                  outbound EDIFACT message would be silently discarded. Use \
1134                  a real As4Sender implementation instead."
1135)]
1136pub struct NoopAs4Sender;
1137
1138// The trait impl is test/testing-only: a release build without the `testing`
1139// feature cannot wire NoopAs4Sender into an outbox worker at all.
1140#[cfg(any(test, feature = "testing"))]
1141impl As4Sender for NoopAs4Sender {
1142    async fn send(&self, _msg: &OutboxMessage) -> Result<(), EngineError> {
1143        Ok(())
1144    }
1145}
1146
1147/// An [`As4Sender`] that logs every outbound message at `warn` level and
1148/// succeeds without transmitting.
1149///
1150/// Useful for development and integration-testing environments where the
1151/// full AS4 stack is not yet available but message visibility is desired.
1152/// All outbox messages are acknowledged (removed from the queue) after logging.
1153///
1154/// # ⚠️ Data loss warning
1155///
1156/// No EDIFACT message is sent to any counterparty. Do not use in production.
1157#[derive(Debug, Clone, Copy, Default)]
1158#[must_use = "LogAs4Sender discards all outbound messages — use a real AS4 gateway in production"]
1159pub struct LogAs4Sender;
1160
1161impl As4Sender for LogAs4Sender {
1162    async fn send(&self, msg: &OutboxMessage) -> Result<(), EngineError> {
1163        tracing::warn!(
1164            message_id   = %msg.message_id,
1165            message_type = %msg.message_type,
1166            recipient    = %msg.recipient,
1167            "LogAs4Sender: outbox message dropped — configure a real AS4 gateway for production",
1168        );
1169        Ok(())
1170    }
1171}
1172
1173// ── DeadlineScheduler ─────────────────────────────────────────────────────────
1174
1175/// A background task that polls [`DeadlineStore::due_now`] and dispatches
1176/// deadline commands to the owning processes via a caller-supplied function.
1177///
1178/// Obtain via [`EngineContext::run_deadline_scheduler`] and drive by spawning
1179/// [`DeadlineScheduler::run`] in a Tokio task.
1180///
1181/// # Dispatch function
1182///
1183/// The `dispatch` function receives a fired [`Deadline`] and returns a future
1184/// that dispatches the appropriate timeout command to the process. The function
1185/// is responsible for resuming the correct workflow and calling `execute`.
1186/// After the future completes, the scheduler cancels the deadline from the
1187/// store regardless of the dispatch outcome (to prevent re-firing).
1188///
1189/// ```rust,ignore
1190/// use std::time::Duration;
1191///
1192/// let scheduler = ctx.run_deadline_scheduler(
1193///     |deadline| async move {
1194///         tracing::warn!(
1195///             deadline_id = %deadline.deadline_id(),
1196///             label = %deadline.label(),
1197///             "deadline fired",
1198///         );
1199///         Ok(())
1200///     },
1201///     100,
1202///     Duration::from_secs(30),
1203/// );
1204/// tokio::spawn(async move { scheduler.run().await });
1205/// ```
1206pub struct DeadlineScheduler<DS: DeadlineStore> {
1207    store: DS,
1208    dispatch: Box<
1209        dyn Fn(
1210                Deadline,
1211            ) -> std::pin::Pin<
1212                Box<dyn std::future::Future<Output = Result<(), EngineError>> + Send>,
1213            > + Send
1214            + Sync,
1215    >,
1216    batch_size: usize,
1217    poll_interval: std::time::Duration,
1218    /// Optional liveness heartbeat — stores the current UTC Unix timestamp
1219    /// (seconds) after each poll cycle.
1220    heartbeat: Option<std::sync::Arc<std::sync::atomic::AtomicI64>>,
1221    /// Graceful-shutdown signal — see [`DeadlineScheduler::with_shutdown`].
1222    shutdown: Option<tokio_util::sync::CancellationToken>,
1223}
1224
1225impl<DS: DeadlineStore> DeadlineScheduler<DS> {
1226    /// Run the deadline poll loop until the shutdown token is cancelled.
1227    ///
1228    /// Cancellation is observed between deadlines and during the idle sleep, so
1229    /// a deadline already being dispatched runs to completion. A deadline left
1230    /// undispatched stays registered and fires on the next start — it is due, so
1231    /// the next `due_now` returns it again.
1232    pub async fn run(self) {
1233        loop {
1234            if self
1235                .shutdown
1236                .as_ref()
1237                .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
1238            {
1239                tracing::info!("deadline scheduler: shutdown signalled; stopping");
1240                return;
1241            }
1242            // Tick liveness at the *start* of every poll cycle, ahead of the
1243            // early-`continue` paths below.  An idle scheduler (no due
1244            // deadlines) is alive and must keep ticking; only one genuinely
1245            // hung inside an `.await` stops.
1246            if let Some(ref hb) = self.heartbeat {
1247                hb.store(
1248                    time::OffsetDateTime::now_utc().unix_timestamp(),
1249                    std::sync::atomic::Ordering::Relaxed,
1250                );
1251            }
1252
1253            let result = match self.store.due_now(self.batch_size).await {
1254                Ok(r) => r,
1255                Err(e) => {
1256                    tracing::warn!(
1257                        error = %e,
1258                        "deadline scheduler: store error polling due deadlines (will retry)",
1259                    );
1260                    if !sleep_or_cancel(self.poll_interval, self.shutdown.as_ref()).await {
1261                        return;
1262                    }
1263                    continue;
1264                }
1265            };
1266
1267            if result.deadlines.is_empty() {
1268                if !sleep_or_cancel(self.poll_interval, self.shutdown.as_ref()).await {
1269                    return;
1270                }
1271                continue;
1272            }
1273
1274            for deadline in result.deadlines {
1275                // Between deadlines, not inside one: a dispatch already running
1276                // must finish so its events and outbox entries commit together.
1277                if self
1278                    .shutdown
1279                    .as_ref()
1280                    .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
1281                {
1282                    tracing::info!(
1283                        "deadline scheduler: shutdown signalled mid-batch; \
1284                         undispatched deadlines remain due and fire on the next start"
1285                    );
1286                    return;
1287                }
1288                let id = deadline.deadline_id();
1289                let label = deadline.label().to_owned();
1290
1291                // An APERAK delivery window that reaches this point is a
1292                // regulatory violation under APERAK AHB 1.0 §2.4.1 (Strom
1293                // 45 min) / §2.3.1 (Gas 1 Werktag): the outbox worker discharges
1294                // the window the moment the APERAK goes out, so one that is
1295                // still registered when it comes due was never answered.
1296                //
1297                // Do NOT re-test `now > due_at` here. `due_now` selects on
1298                // `due_at <= now`, so that comparison is true by construction and
1299                // says nothing about compliance — it was the reason this counter
1300                // once tracked "Strom processes started" rather than "APERAKs
1301                // missed". The discharge is what carries the meaning.
1302                if label.starts_with(mako_fristen::APERAK_WINDOW_LABEL_PREFIX) {
1303                    let now = time::OffsetDateTime::now_utc();
1304                    crate::metrics::EngineMetrics::global().aperak_missed(&label);
1305                    tracing::error!(
1306                        deadline_id = %id,
1307                        label       = %label,
1308                        due_at      = %deadline.due_at(),
1309                        fired_at    = %now,
1310                        overdue_secs = (now - deadline.due_at()).whole_seconds(),
1311                        "APERAK delivery window closed with no delivery — regulatory \
1312                         violation (APERAK AHB 1.0 §2.4.1 Strom / §2.3.1 Gas). \
1313                         Counter: makod_aperak_missed_total",
1314                    );
1315                }
1316
1317                let should_cancel = match (self.dispatch)(deadline).await {
1318                    Ok(()) => true,
1319                    Err(ref e) if e.is_version_conflict() => {
1320                        // The process was modified concurrently; the timeout
1321                        // command will be retried on the next poll cycle.
1322                        // Do NOT cancel — let the deadline remain due so it
1323                        // fires again until a non-conflict dispatch succeeds.
1324                        tracing::warn!(
1325                            deadline_id = %id,
1326                            label       = %label,
1327                            "deadline scheduler: VersionConflict; will retry on next poll",
1328                        );
1329                        false
1330                    }
1331                    Err(e) => {
1332                        tracing::warn!(
1333                            deadline_id = %id,
1334                            label       = %label,
1335                            error       = %e,
1336                            "deadline scheduler: dispatch failed (permanent); cancelling",
1337                        );
1338                        true
1339                    }
1340                };
1341                if should_cancel && let Err(e) = self.store.cancel(id).await {
1342                    tracing::error!(
1343                        deadline_id = %id,
1344                        error       = %e,
1345                        "deadline scheduler: cancel failed; deadline may fire again",
1346                    );
1347                }
1348            }
1349
1350            // If has_more, loop immediately to drain the batch.
1351        }
1352    }
1353}
1354
1355impl<DS: DeadlineStore> DeadlineScheduler<DS> {
1356    /// Attach a liveness heartbeat to this scheduler.
1357    ///
1358    /// The scheduler will store the current UTC Unix timestamp (seconds) into
1359    /// `heartbeat` at the end of every poll cycle.
1360    #[must_use]
1361    pub fn with_heartbeat(
1362        mut self,
1363        heartbeat: std::sync::Arc<std::sync::atomic::AtomicI64>,
1364    ) -> Self {
1365        self.heartbeat = Some(heartbeat);
1366        self
1367    }
1368
1369    /// Attach a graceful-shutdown token.
1370    ///
1371    /// Cancelling it makes [`DeadlineScheduler::run`] return at the next
1372    /// deadline boundary or immediately out of its idle sleep, so the caller can
1373    /// close the event store once the scheduler has stopped writing to it.
1374    #[must_use]
1375    pub fn with_shutdown(mut self, shutdown: tokio_util::sync::CancellationToken) -> Self {
1376        self.shutdown = Some(shutdown);
1377        self
1378    }
1379}
1380
1381impl<ES, SS, OS, DS, PR> EngineContext<ES, SS, OS, DS, PR>
1382where
1383    ES: EventStore,
1384    DS: DeadlineStore + Clone,
1385{
1386    /// Construct a [`DeadlineScheduler`] that polls the deadline store and
1387    /// dispatches fired deadlines via `dispatch`.
1388    ///
1389    /// The `dispatch` function is called for every fired deadline. It should
1390    /// resume the owning process and execute the appropriate timeout command.
1391    ///
1392    /// `batch_size` — deadlines fetched per poll cycle.
1393    /// `poll_interval` — sleep duration when no deadlines are due.
1394    ///
1395    /// ```rust,ignore
1396    /// use std::time::Duration;
1397    ///
1398    /// let scheduler = ctx.run_deadline_scheduler(
1399    ///     |d| async move {
1400    ///         tracing::info!(label = %d.label(), "firing deadline");
1401    ///         Ok(())
1402    ///     },
1403    ///     100,
1404    ///     Duration::from_secs(30),
1405    /// );
1406    /// tokio::spawn(async move { scheduler.run().await });
1407    /// ```
1408    #[must_use]
1409    pub fn run_deadline_scheduler<F, Fut>(
1410        &self,
1411        dispatch: F,
1412        batch_size: usize,
1413        poll_interval: std::time::Duration,
1414    ) -> DeadlineScheduler<DS>
1415    where
1416        F: Fn(Deadline) -> Fut + Send + Sync + 'static,
1417        Fut: std::future::Future<Output = Result<(), EngineError>> + Send + 'static,
1418    {
1419        DeadlineScheduler {
1420            store: self.deadline_store.clone(),
1421            dispatch: Box::new(move |d| Box::pin(dispatch(d))),
1422            batch_size,
1423            poll_interval,
1424            heartbeat: None,
1425            shutdown: None,
1426        }
1427    }
1428}
1429
1430// ── EngineBuilder ─────────────────────────────────────────────────────────────
1431
1432/// Assembles engine infrastructure and produces an [`EngineContext`].
1433///
1434/// Uses type-state to enforce that an event store is provided before
1435/// [`build`] can be called. All other stores default to `Noop`
1436/// implementations.
1437///
1438/// ## Quick start
1439///
1440/// ```rust,ignore
1441/// // Minimal — event store only, all others are Noop:
1442/// let ctx = EngineBuilder::new()
1443///     .with_event_store(InMemoryEventStore::new())
1444///     .build();
1445///
1446/// // Full infrastructure:
1447/// let ctx = EngineBuilder::new()
1448///     .with_event_store(InMemoryEventStore::new())
1449///     .with_snapshot_store(InMemorySnapshotStore::new())
1450///     .with_outbox_store(InMemoryOutboxStore::new())
1451///     .with_deadline_store(InMemoryDeadlineStore::new())
1452///     .with_registry(InMemoryProcessRegistry::new())
1453///     .register(Box::new(GpkeModule))
1454///     .build();
1455/// ```
1456///
1457/// [`build`]: EngineBuilder::build
1458pub struct EngineBuilder<
1459    ES = (),
1460    SS = NoopSnapshotStore,
1461    OS = NoopOutboxStore,
1462    DS = NoopDeadlineStore,
1463    PR = NoopProcessRegistry,
1464> {
1465    event_store: ES,
1466    snapshot_store: SS,
1467    outbox_store: OS,
1468    deadline_store: DS,
1469    registry: PR,
1470    dead_letter_sink: Arc<dyn DeadLetterSink>,
1471    modules: Vec<Box<dyn EngineModule>>,
1472    /// Active [`DeploymentRoles`] for this engine instance.
1473    ///
1474    /// Controls role-conditional PID registration via
1475    /// [`EngineModule::register_pids_with_roles`]. Defaults to
1476    /// [`DeploymentRoles::all()`]: an engine that names no roles registers
1477    /// every PID its modules declare, which is what a test harness and a
1478    /// combined-role deployment both want.
1479    deployment_roles: DeploymentRoles,
1480    /// Optional profile validator injected by `makod` or callers that have
1481    /// access to `edi-energy`.  When `Some`, called for each
1482    /// [`ProfileRequirement`] declared by registered modules.  When `None`,
1483    /// profile requirements are not validated (safe in unit tests).
1484    ///
1485    /// Signature: `fn(message_type: &str) -> bool`
1486    ///
1487    /// [`ProfileRequirement`]: crate::profile::ProfileRequirement
1488    profile_validator: Option<Box<dyn Fn(&str) -> bool + Send + Sync>>,
1489}
1490#[cfg(any(test, feature = "testing"))]
1491impl Default
1492    for EngineBuilder<
1493        (),
1494        NoopSnapshotStore,
1495        NoopOutboxStore,
1496        NoopDeadlineStore,
1497        NoopProcessRegistry,
1498    >
1499{
1500    fn default() -> Self {
1501        Self {
1502            event_store: (),
1503            snapshot_store: NoopSnapshotStore,
1504            outbox_store: NoopOutboxStore,
1505            deadline_store: NoopDeadlineStore,
1506            registry: NoopProcessRegistry,
1507            dead_letter_sink: Arc::new(LogDeadLetterSink),
1508            modules: Vec::new(),
1509            deployment_roles: DeploymentRoles::all(),
1510            profile_validator: None,
1511        }
1512    }
1513}
1514
1515#[cfg(any(test, feature = "testing"))]
1516impl EngineBuilder {
1517    /// Create a new builder with all `Noop` defaults.
1518    ///
1519    /// Only available in `#[cfg(test)]` or with the `testing` feature enabled,
1520    /// because the Noop defaults silently discard outbox messages, deadlines,
1521    /// and process registry entries. Production binaries must wire real stores
1522    /// via the `with_*` builder methods.
1523    ///
1524    /// Call [`with_event_store`] before [`build`] — the event store is
1525    /// **required**.
1526    ///
1527    /// [`with_event_store`]: EngineBuilder::with_event_store
1528    /// [`build`]: EngineBuilder::build
1529    #[must_use]
1530    pub fn new() -> Self {
1531        Self::default()
1532    }
1533}
1534
1535impl<OS, DS, PR> EngineBuilder<(), NoopSnapshotStore, OS, DS, PR>
1536where
1537    OS: OutboxStore,
1538    DS: DeadlineStore,
1539    PR: ProcessRegistry,
1540{
1541    /// Create a production-ready builder with explicit stores for outbox,
1542    /// deadline, and process registry.
1543    ///
1544    /// This constructor is available in all build configurations including
1545    /// production binaries. It enforces that the three stores that can cause
1546    /// silent data loss (`OutboxStore`, `DeadlineStore`, `ProcessRegistry`)
1547    /// are provided explicitly — there is no Noop fallback.
1548    ///
1549    /// `NoopSnapshotStore` is used as the snapshot default because it is safe
1550    /// for production: skipping snapshots means full replay, but no data loss.
1551    /// Override with [`with_snapshot_store`] to enable snapshot-accelerated
1552    /// replay.
1553    ///
1554    /// Call [`with_event_store`] before [`build`] — the event store is
1555    /// **required**.
1556    ///
1557    /// ```rust,ignore
1558    /// let ctx = EngineBuilder::with_stores(outbox, deadline, registry)
1559    ///     .with_event_store(store.clone())
1560    ///     .with_snapshot_store(InMemorySnapshotStore::new())
1561    ///     .build();
1562    /// ```
1563    ///
1564    /// [`with_snapshot_store`]: EngineBuilder::with_snapshot_store
1565    /// [`with_event_store`]: EngineBuilder::with_event_store
1566    /// [`build`]: EngineBuilder::build
1567    #[must_use]
1568    pub fn with_stores(outbox_store: OS, deadline_store: DS, registry: PR) -> Self {
1569        Self {
1570            event_store: (),
1571            snapshot_store: NoopSnapshotStore,
1572            outbox_store,
1573            deadline_store,
1574            registry,
1575            dead_letter_sink: Arc::new(LogDeadLetterSink),
1576            modules: Vec::new(),
1577            deployment_roles: DeploymentRoles::all(),
1578            profile_validator: None,
1579        }
1580    }
1581}
1582
1583impl<ES, SS, OS, DS, PR> EngineBuilder<ES, SS, OS, DS, PR> {
1584    /// Set the event store. **Required** — `build()` is only available once
1585    /// this has been called with a type that implements [`EventStore`].
1586    ///
1587    /// Replaces any previously set event store (type-state transition).
1588    #[must_use]
1589    pub fn with_event_store<ES2: EventStore>(
1590        self,
1591        store: ES2,
1592    ) -> EngineBuilder<ES2, SS, OS, DS, PR> {
1593        EngineBuilder {
1594            event_store: store,
1595            snapshot_store: self.snapshot_store,
1596            outbox_store: self.outbox_store,
1597            deadline_store: self.deadline_store,
1598            registry: self.registry,
1599            dead_letter_sink: self.dead_letter_sink,
1600            modules: self.modules,
1601            deployment_roles: self.deployment_roles,
1602            profile_validator: self.profile_validator,
1603        }
1604    }
1605
1606    /// Set the snapshot store (default: [`NoopSnapshotStore`]).
1607    ///
1608    /// ## Default: `NoopSnapshotStore`
1609    ///
1610    /// Without calling this method the builder uses [`NoopSnapshotStore`],
1611    /// which silently discards all snapshot writes and returns `None` for
1612    /// every snapshot read.  The engine still functions correctly — every
1613    /// command handling call replays the full event log from the beginning
1614    /// instead of starting from a stored snapshot.  For low-volume processes
1615    /// this is fine; for long-lived processes with many events the replay cost
1616    /// can become significant.
1617    ///
1618    /// Enable snapshotting in production by providing a real [`SnapshotStore`]
1619    /// implementation (e.g. the SlateDB-backed store in `makod`).  In tests,
1620    /// `InMemorySnapshotStore` is available behind the `testing` feature flag.
1621    ///
1622    /// Note: [`Process::state_with_snapshot`][crate::process::Process::state_with_snapshot]
1623    /// is a compile-time no-op when the snapshot store is `NoopSnapshotStore`
1624    /// — it never calls the store and always returns `None`, so no snapshot is
1625    /// ever saved or loaded.
1626    #[must_use]
1627    pub fn with_snapshot_store<SS2: SnapshotStore>(
1628        self,
1629        store: SS2,
1630    ) -> EngineBuilder<ES, SS2, OS, DS, PR> {
1631        EngineBuilder {
1632            event_store: self.event_store,
1633            snapshot_store: store,
1634            outbox_store: self.outbox_store,
1635            deadline_store: self.deadline_store,
1636            registry: self.registry,
1637            dead_letter_sink: self.dead_letter_sink,
1638            modules: self.modules,
1639            deployment_roles: self.deployment_roles,
1640            profile_validator: self.profile_validator,
1641        }
1642    }
1643
1644    /// Set the outbox store (default: [`NoopOutboxStore`]).
1645    #[must_use]
1646    pub fn with_outbox_store<OS2: OutboxStore>(
1647        self,
1648        store: OS2,
1649    ) -> EngineBuilder<ES, SS, OS2, DS, PR> {
1650        EngineBuilder {
1651            event_store: self.event_store,
1652            snapshot_store: self.snapshot_store,
1653            outbox_store: store,
1654            deadline_store: self.deadline_store,
1655            registry: self.registry,
1656            dead_letter_sink: self.dead_letter_sink,
1657            modules: self.modules,
1658            deployment_roles: self.deployment_roles,
1659            profile_validator: self.profile_validator,
1660        }
1661    }
1662
1663    /// Set the deadline store (default: [`NoopDeadlineStore`]).
1664    #[must_use]
1665    pub fn with_deadline_store<DS2: DeadlineStore>(
1666        self,
1667        store: DS2,
1668    ) -> EngineBuilder<ES, SS, OS, DS2, PR> {
1669        EngineBuilder {
1670            event_store: self.event_store,
1671            snapshot_store: self.snapshot_store,
1672            outbox_store: self.outbox_store,
1673            deadline_store: store,
1674            registry: self.registry,
1675            dead_letter_sink: self.dead_letter_sink,
1676            modules: self.modules,
1677            deployment_roles: self.deployment_roles,
1678            profile_validator: self.profile_validator,
1679        }
1680    }
1681
1682    /// Set the process registry (default: [`NoopProcessRegistry`]).
1683    #[must_use]
1684    pub fn with_registry<PR2: ProcessRegistry>(
1685        self,
1686        registry: PR2,
1687    ) -> EngineBuilder<ES, SS, OS, DS, PR2> {
1688        EngineBuilder {
1689            event_store: self.event_store,
1690            snapshot_store: self.snapshot_store,
1691            outbox_store: self.outbox_store,
1692            deadline_store: self.deadline_store,
1693            registry,
1694            dead_letter_sink: self.dead_letter_sink,
1695            modules: self.modules,
1696            deployment_roles: self.deployment_roles,
1697            profile_validator: self.profile_validator,
1698        }
1699    }
1700
1701    /// Set the dead-letter sink (default: [`LogDeadLetterSink`]).
1702    ///
1703    /// The dead-letter sink receives every message that cannot be routed to a
1704    /// workflow. The default [`LogDeadLetterSink`] emits `tracing::warn!`
1705    /// events, making rejections visible in log output without configuration.
1706    ///
1707    /// Override with a persistent DLQ implementation in production:
1708    ///
1709    /// ```rust,ignore
1710    /// use mako_engine::dead_letter::LogDeadLetterSink;
1711    ///
1712    /// let ctx = EngineBuilder::new()
1713    ///     .with_event_store(my_store)
1714    ///     .with_dead_letter_sink(MyPersistentDlq::new())
1715    ///     .build();
1716    /// ```
1717    ///
1718    /// [`LogDeadLetterSink`]: crate::dead_letter::LogDeadLetterSink
1719    #[must_use]
1720    pub fn with_dead_letter_sink(mut self, sink: impl DeadLetterSink) -> Self {
1721        self.dead_letter_sink = Arc::new(sink);
1722        self
1723    }
1724
1725    /// Register an `edi-energy` profile validator for startup profile checks.
1726    ///
1727    /// The closure receives a message-type string (e.g. `"UTILMD"`) and must
1728    /// return `true` if at least one active profile for that message type is
1729    /// registered for today's date.
1730    ///
1731    /// Wire this in `makod` using the `edi-energy` global registry:
1732    ///
1733    /// ```rust,ignore
1734    /// use edi_energy::registry::ReleaseRegistry;
1735    ///
1736    /// let today = mako_fristen::heute();
1737    /// builder.with_profile_validator(move |msg_type| {
1738    ///     ReleaseRegistry::global()
1739    ///         .profiles_for_str(msg_type)
1740    ///         .any(|p| match (p.valid_from(), p.valid_until()) {
1741    ///             (Some(f), Some(u)) => f <= today && today <= u,
1742    ///             (Some(f), None)    => f <= today,
1743    ///             (None, _)          => true,
1744    ///         })
1745    /// })
1746    /// ```
1747    ///
1748    /// Domain crates do **not** need to call this — they only declare
1749    /// [`profile_requirements`].
1750    ///
1751    /// [`profile_requirements`]: EngineModule::profile_requirements
1752    #[must_use]
1753    pub fn with_profile_validator(
1754        mut self,
1755        validator: impl Fn(&str) -> bool + Send + Sync + 'static,
1756    ) -> Self {
1757        self.profile_validator = Some(Box::new(validator));
1758        self
1759    }
1760
1761    /// Register a domain module.
1762    ///
1763    /// The module name becomes visible in
1764    /// [`EngineContext::registered_modules`] after [`build`] is called.
1765    ///
1766    /// [`build`]: EngineBuilder::build
1767    #[must_use]
1768    pub fn register(mut self, module: Box<dyn EngineModule>) -> Self {
1769        self.modules.push(module);
1770        self
1771    }
1772
1773    /// Register multiple [`EngineModule`]s at once from a pre-built `Vec`.
1774    ///
1775    /// Equivalent to calling [`register`] in a loop. Useful when the set of
1776    /// modules is assembled conditionally (e.g. via `#[cfg]`-gated pushes to a
1777    /// `Vec<Box<dyn EngineModule>>`) before the builder chain starts.
1778    ///
1779    /// [`register`]: EngineBuilder::register
1780    #[must_use]
1781    pub fn register_many(mut self, modules: Vec<Box<dyn EngineModule>>) -> Self {
1782        self.modules.extend(modules);
1783        self
1784    }
1785
1786    /// Set the active [`DeploymentRoles`] for this engine instance.
1787    ///
1788    /// Controls role-conditional PID registration in [`EngineModule::register_pids_with_roles`].
1789    ///
1790    /// The default is [`DeploymentRoles::all()`], which registers every PID unconditionally
1791    /// — identical to the pre-role-aware behavior. Providing an explicit role set
1792    /// restricts role-conditional blocks to only the declared roles:
1793    ///
1794    /// - **NB-only** (`DeploymentRoles::nb()`): 19001/19002 route to `gpke-konfiguration`;
1795    ///   WiM nMSB blocks are skipped.
1796    /// - **nMSB-only** (`DeploymentRoles::nmsb()`): 19001/19002 route to `wim-geraeteubernahme`;
1797    ///   GPKE NB blocks are skipped.
1798    /// - **NB + gMSB** (`DeploymentRoles::nb_msb()`): most common Stadtwerke combination.
1799    ///
1800    /// # Conflict guard
1801    ///
1802    /// When two modules would register the same PID to **different** workflows, the
1803    /// engine panics during [`build`]. Set explicit roles to prevent both modules from
1804    /// activating the same PID simultaneously:
1805    ///
1806    /// ```rust,ignore
1807    /// use mako_engine::marktrolle::DeploymentRoles;
1808    ///
1809    /// let ctx = EngineBuilder::with_stores(outbox, deadline, registry)
1810    ///     .with_event_store(store)
1811    ///     .with_deployment_roles(DeploymentRoles::nb())  // only NB: GPKE gets 19001/19002
1812    ///     .register(Box::new(GpkeModule))
1813    ///     .register(Box::new(WimModule))  // nMSB block skipped — no conflict
1814    ///     .build();
1815    /// ```
1816    ///
1817    /// [`build`]: EngineBuilder::build
1818    #[must_use]
1819    pub fn with_deployment_roles(mut self, roles: DeploymentRoles) -> Self {
1820        self.deployment_roles = roles;
1821        self
1822    }
1823}
1824
1825impl<ES, SS, OS, DS, PR> EngineBuilder<ES, SS, OS, DS, PR>
1826where
1827    ES: EventStore,
1828    SS: SnapshotStore,
1829    OS: OutboxStore,
1830    DS: DeadlineStore,
1831    PR: ProcessRegistry,
1832{
1833    /// Build the [`EngineContext`].
1834    ///
1835    /// Consumes the builder. All registered modules and configured stores are
1836    /// moved into the returned [`EngineContext`].
1837    ///
1838    /// This method is only available when `ES` implements [`EventStore`].
1839    /// If you have not called [`with_event_store`], this will not compile.
1840    ///
1841    /// # Panics
1842    ///
1843    /// Panics when any registered module returns `Err` from
1844    /// [`EngineModule::configure`]. The panic message includes the module
1845    /// name and the error string so the deployment failure is actionable.
1846    ///
1847    /// [`with_event_store`]: EngineBuilder::with_event_store
1848    #[must_use]
1849    #[allow(clippy::too_many_lines)]
1850    pub fn build(self) -> EngineContext<ES, SS, OS, DS, PR> {
1851        // ── Noop store safety checks ──────────────────────────────────────────
1852        //
1853        // Noop stores lose data silently: NoopDeadlineStore drops every APERAK
1854        // deadline (BNetzA violation), NoopOutboxStore discards all outbound
1855        // messages, NoopProcessRegistry loses conversation routing on restart.
1856        //
1857        // In production builds (no `testing` feature, not running under
1858        // `#[test]`), the Noop constructors are cfg-gated out so this branch
1859        // is dead code and compiles away. In test/testing/tracing builds we
1860        // emit warnings so test harnesses see the configuration in log output.
1861        //
1862        // IMPORTANT: if you are reading this because a panic fired in production,
1863        // it means the `testing` feature was accidentally enabled in the binary.
1864        // Remove it from the production Cargo.toml feature list immediately.
1865        {
1866            let os_name = std::any::type_name::<OS>();
1867            let ds_name = std::any::type_name::<DS>();
1868            let pr_name = std::any::type_name::<PR>();
1869
1870            // Regulatory-critical stores: panic in any build context if these
1871            // are noop. OutboxStore and DeadlineStore must be durable in
1872            // production; ProcessRegistry must survive restarts.
1873            #[cfg(not(any(test, feature = "testing")))]
1874            {
1875                assert!(
1876                    !ds_name.contains("NoopDeadlineStore"),
1877                    "EngineBuilder::build: NoopDeadlineStore is active in a \
1878                     non-testing build. This silently discards all APERAK deadlines, \
1879                     which is an immediately reportable BNetzA violation \
1880                     (BK6-22-024 §5, BK7-24-01-009). \
1881                     Call .with_deadline_store(SlateDbStore::as_deadline_store()) \
1882                     in your production engine assembly. \
1883                     If this is a test, enable the 'testing' feature."
1884                );
1885                assert!(
1886                    !os_name.contains("NoopOutboxStore"),
1887                    "EngineBuilder::build: NoopOutboxStore is active in a \
1888                     non-testing build. This silently discards all outbound \
1889                     APERAK, CONTRL, and UTILMD messages. \
1890                     Call .with_outbox_store(SlateDbStore::as_outbox_store()) \
1891                     in your production engine assembly. \
1892                     If this is a test, enable the 'testing' feature."
1893                );
1894                assert!(
1895                    !pr_name.contains("NoopProcessRegistry"),
1896                    "EngineBuilder::build: NoopProcessRegistry is active in a \
1897                     non-testing build. This means conversation routing \
1898                     (PID → stream_id lookup) is lost on every restart, \
1899                     breaking all WiM, GeLi Gas, and GPKE in-flight processes. \
1900                     Call .with_registry(SlateDbStore::as_process_registry()) \
1901                     in your production engine assembly. \
1902                     If this is a test, enable the 'testing' feature."
1903                );
1904            }
1905
1906            // In test/testing/tracing builds: emit warnings instead of panicking.
1907            #[cfg(any(test, feature = "testing", feature = "tracing"))]
1908            {
1909                let ss_name = std::any::type_name::<SS>();
1910                if ss_name.contains("NoopSnapshotStore") {
1911                    tracing::warn!(
1912                        store = ss_name,
1913                        "EngineBuilder: NoopSnapshotStore is active — snapshots will not be \
1914                         persisted. Use SlateDbStore::as_snapshot_store() in production."
1915                    );
1916                }
1917                if os_name.contains("NoopOutboxStore") {
1918                    tracing::warn!(
1919                        store = os_name,
1920                        "EngineBuilder: NoopOutboxStore is active — outbound messages will be \
1921                         silently discarded. Use SlateDbStore::as_outbox_store() in production."
1922                    );
1923                }
1924                if ds_name.contains("NoopDeadlineStore") {
1925                    tracing::warn!(
1926                        store = ds_name,
1927                        "EngineBuilder: NoopDeadlineStore is active — scheduled deadlines will \
1928                         not fire after restart. Use SlateDbStore::as_deadline_store() in production."
1929                    );
1930                }
1931                if pr_name.contains("NoopProcessRegistry") {
1932                    tracing::warn!(
1933                        store = pr_name,
1934                        "EngineBuilder: NoopProcessRegistry is active — process routing will be \
1935                         lost on restart. Use SlateDbStore::as_process_registry() in production."
1936                    );
1937                }
1938            }
1939        }
1940        // Validate every module before assembling the context.
1941        // A missing adapter or misconfigured module fails at startup (not at
1942        // first inbound message), making deployment failures observable immediately.
1943        for module in &self.modules {
1944            if let Err(msg) = module.configure() {
1945                panic!(
1946                    "EngineBuilder::build: module '{}' failed configuration validation: {}",
1947                    module.name(),
1948                    msg
1949                );
1950            }
1951            // Validate profile requirements via the injected validator.
1952            // Domain crates declare requirements; only the binary crate (makod)
1953            // injects the edi-energy registry — domain crates need no edi-energy
1954            // import for this check.
1955            if let Some(ref validator) = self.profile_validator {
1956                for req in module.profile_requirements() {
1957                    assert!(
1958                        validator(req.message_type),
1959                        "EngineBuilder::build: module '{}' requires an active edi-energy \
1960                             profile for '{}' ({}) but none is registered for today's date. \
1961                             Run `cargo xtask codegen` to add the missing profile.",
1962                        module.name(),
1963                        req.message_type,
1964                        req.label,
1965                    );
1966                }
1967            }
1968        }
1969        // Build the PID router from all registered modules.
1970        // Also assert that no two modules claim the same PID — a PID overlap
1971        // is always a configuration error: one module's messages would be
1972        // silently swallowed by another's workflow, producing missing-process
1973        // errors or incorrect audit trails.
1974        let mut pid_router = PidRouter::new();
1975        let mut pid_owners: std::collections::HashMap<u32, &str> = std::collections::HashMap::new();
1976        // Keep each module's scratch router so we can build `pid_router` from
1977        // them in a second pass with the resolved ownership table.
1978        let mut module_scratches: Vec<PidRouter> = Vec::with_capacity(self.modules.len());
1979
1980        // Pass 1 — detect conflicts, determine PID ownership (first-wins for
1981        // explicit roles, last-wins for DeploymentRoles::all()).
1982        for module in &self.modules {
1983            // Temporarily build a scratch router to read this module's PIDs
1984            // for cross-module overlap detection (module-ownership level).
1985            let mut scratch = PidRouter::new();
1986            module.register_pids_with_roles(&mut scratch, &self.deployment_roles);
1987            for pid in scratch.registered_pids() {
1988                if let Some(prev) = pid_owners.insert(pid, module.name()) {
1989                    if self.deployment_roles.is_all() {
1990                        // With DeploymentRoles::all() (the default), role-conditional PIDs
1991                        // are registered by all modules that claim them, producing last-wins
1992                        // semantics. This is acceptable for single-role and dev/test deployments.
1993                        //
1994                        // In production multi-role deployments where both an NB and nMSB role
1995                        // are served by the same instance, set explicit roles via
1996                        // `EngineBuilder::with_deployment_roles` to prevent silent misrouting.
1997                        //
1998                        // We emit a debug-level log here (not warn) because the vast majority
1999                        // of deployments are single-role and this overlap is expected/harmless.
2000                        #[cfg(feature = "tracing")]
2001                        tracing::debug!(
2002                            pid,
2003                            previous_module = prev,
2004                            current_module = module.name(),
2005                            "PID registered by multiple modules with DeploymentRoles::all(); \
2006                             last module wins (use with_deployment_roles for strict routing)",
2007                        );
2008                        let _ = prev; // suppress unused-variable warning when tracing is off
2009                    } else {
2010                        // Explicit roles: the FIRST module to register a PID retains ownership.
2011                        // Restore the previous (first) owner and emit a warning so the operator
2012                        // can investigate.  A panic would be too strict: some shared PIDs
2013                        // (e.g. REMADV 33001/33002) are legitimately claimed by both GPKE and
2014                        // WiM billing; conversation-ID routing is the long-term solution, but
2015                        // first-wins gives correct behaviour for all current deployments.
2016                        pid_owners.insert(pid, prev); // restore first owner
2017                        #[cfg(feature = "tracing")]
2018                        tracing::warn!(
2019                            pid,
2020                            first_module = prev,
2021                            second_module = module.name(),
2022                            "PID {pid} claimed by both '{prev}' and '{}' with explicit \
2023                             DeploymentRoles; first module ('{prev}') retains ownership. \
2024                             Verify PID registration is correct for this deployment.",
2025                            module.name(),
2026                        );
2027                        #[cfg(not(feature = "tracing"))]
2028                        let _ = prev; // suppress unused-variable warning when tracing is off
2029                    }
2030                }
2031            }
2032            module_scratches.push(scratch);
2033        }
2034
2035        // Pass 2 — build the real `pid_router` from the scratch pads, respecting
2036        // the ownership table built in pass 1.
2037        for (module, scratch) in self.modules.iter().zip(module_scratches.iter()) {
2038            // Unambiguous (Sparte-agnostic) entries: only register if this module
2039            // owns the PID in the resolved ownership table.
2040            for pid in scratch.registered_pids() {
2041                if pid_owners.get(&pid).copied() == Some(module.name())
2042                    && let Some(wf) = scratch.route(pid)
2043                {
2044                    pid_router.register(pid, wf);
2045                }
2046            }
2047            // Commodity (Sparte-qualified) entries use distinct (pid, Sparte) keys
2048            // and never conflict across modules; register them all unconditionally.
2049            for (pid, sparte, wf) in scratch.registered_commodity_entries() {
2050                pid_router.register_with_sparte(pid, sparte, wf);
2051            }
2052        }
2053        let registered_modules = self.modules.iter().map(|m| m.name()).collect();
2054        let registered_workflows = self
2055            .modules
2056            .iter()
2057            .flat_map(|m| m.workflow_names().iter().copied())
2058            .collect();
2059        EngineContext {
2060            event_store: Arc::new(self.event_store),
2061            snapshot_store: self.snapshot_store,
2062            outbox_store: self.outbox_store,
2063            deadline_store: self.deadline_store,
2064            registry: self.registry,
2065            dead_letter_sink: self.dead_letter_sink,
2066            pid_router,
2067            registered_modules,
2068            registered_workflows,
2069        }
2070    }
2071}
2072
2073#[cfg(test)]
2074mod tests {
2075    use super::*;
2076    use crate::{
2077        deadline::InMemoryDeadlineStore,
2078        error::WorkflowError,
2079        event_store::InMemoryEventStore,
2080        ids::TenantId,
2081        outbox::InMemoryOutboxStore,
2082        pid_router::PidRouter,
2083        registry::InMemoryProcessRegistry,
2084        snapshot::InMemorySnapshotStore,
2085        version::WorkflowId,
2086        workflow::{CommandPayload, EventPayload, Workflow},
2087    };
2088
2089    // ── Minimal workflow for spawn/resume tests ───────────────────────────────
2090
2091    #[derive(serde::Serialize, serde::Deserialize)]
2092    struct PingEvent;
2093
2094    impl EventPayload for PingEvent {
2095        fn event_type(&self) -> &'static str {
2096            "Ping"
2097        }
2098    }
2099
2100    struct PingCommand;
2101
2102    impl CommandPayload for PingCommand {}
2103
2104    #[derive(Default, Clone)]
2105    struct PingState;
2106
2107    struct PingWorkflow;
2108
2109    impl Workflow for PingWorkflow {
2110        type State = PingState;
2111        type Event = PingEvent;
2112        type Command = PingCommand;
2113
2114        fn apply(state: PingState, _: &PingEvent) -> PingState {
2115            state
2116        }
2117
2118        fn handle(
2119            _: &PingState,
2120            _: PingCommand,
2121        ) -> Result<crate::workflow::WorkflowOutput<PingEvent>, WorkflowError> {
2122            Ok(vec![PingEvent].into())
2123        }
2124    }
2125
2126    struct TestModule;
2127
2128    impl EngineModule for TestModule {
2129        fn name(&self) -> &'static str {
2130            "test-module"
2131        }
2132    }
2133
2134    // ── Tests ─────────────────────────────────────────────────────────────────
2135
2136    #[test]
2137    fn build_with_event_store_only() {
2138        let ctx = EngineBuilder::new()
2139            .with_event_store(InMemoryEventStore::new())
2140            .build();
2141        assert!(ctx.registered_modules().is_empty());
2142    }
2143
2144    #[test]
2145    fn build_with_all_stores_and_module() {
2146        let ctx = EngineBuilder::new()
2147            .with_event_store(InMemoryEventStore::new())
2148            .with_snapshot_store(InMemorySnapshotStore::new())
2149            .with_outbox_store(InMemoryOutboxStore::new())
2150            .with_deadline_store(InMemoryDeadlineStore::new())
2151            .with_registry(InMemoryProcessRegistry::new())
2152            .register(Box::new(TestModule))
2153            .build();
2154        assert_eq!(ctx.registered_modules(), &["test-module"]);
2155    }
2156
2157    #[test]
2158    fn multiple_modules_ordered() {
2159        struct ModA;
2160        impl EngineModule for ModA {
2161            fn name(&self) -> &'static str {
2162                "mod-a"
2163            }
2164        }
2165        struct ModB;
2166        impl EngineModule for ModB {
2167            fn name(&self) -> &'static str {
2168                "mod-b"
2169            }
2170        }
2171
2172        let ctx = EngineBuilder::new()
2173            .with_event_store(InMemoryEventStore::new())
2174            .register(Box::new(ModA))
2175            .register(Box::new(ModB))
2176            .build();
2177        assert_eq!(ctx.registered_modules(), &["mod-a", "mod-b"]);
2178    }
2179
2180    #[tokio::test]
2181    async fn spawn_creates_independent_processes() {
2182        let ctx = EngineBuilder::new()
2183            .with_event_store(InMemoryEventStore::new())
2184            .build();
2185        let wf_id = WorkflowId::new("ping", "FV2024-10-01");
2186
2187        let p1 = ctx.spawn::<PingWorkflow>(TenantId::new(), wf_id.clone());
2188        let p2 = ctx.spawn::<PingWorkflow>(TenantId::new(), wf_id);
2189
2190        assert_ne!(p1.process_id(), p2.process_id());
2191    }
2192
2193    #[tokio::test]
2194    async fn resume_sees_previously_appended_events() {
2195        let store = InMemoryEventStore::new();
2196        let ctx = EngineBuilder::new().with_event_store(store).build();
2197
2198        let p = ctx.spawn::<PingWorkflow>(TenantId::new(), WorkflowId::new("ping", "FV2024-10-01"));
2199        p.execute(PingCommand).await.unwrap();
2200
2201        let identity = p.identity();
2202        let resumed = ctx.resume::<PingWorkflow>(identity);
2203        assert_eq!(resumed.event_count().await.unwrap(), 1);
2204    }
2205
2206    #[tokio::test]
2207    async fn registry_routes_process_via_conversation_key() {
2208        use crate::registry::RegistryKey;
2209        let ctx = EngineBuilder::new()
2210            .with_event_store(InMemoryEventStore::new())
2211            .with_registry(InMemoryProcessRegistry::new())
2212            .build();
2213
2214        let p = ctx.spawn::<PingWorkflow>(TenantId::new(), WorkflowId::new("ping", "FV2024-10-01"));
2215        let tenant = p.tenant_id();
2216        let conv_key = RegistryKey::parse("conv:test-conversation-123").expect("valid key");
2217        ctx.registry()
2218            .register(tenant, &conv_key, p.identity())
2219            .await
2220            .unwrap();
2221
2222        let found = ctx
2223            .registry()
2224            .lookup(tenant, &conv_key)
2225            .await
2226            .unwrap()
2227            .expect("must be registered");
2228        let resumed = ctx.resume::<PingWorkflow>(found);
2229        assert_eq!(resumed.process_id(), p.process_id());
2230    }
2231
2232    #[test]
2233    fn pid_router_populated_by_module_register_pids() {
2234        struct PidModule;
2235        impl EngineModule for PidModule {
2236            fn name(&self) -> &'static str {
2237                "pid-module"
2238            }
2239            fn register_pids(&self, router: &mut PidRouter) {
2240                router.register(55001, "gpke-supplier-change");
2241                router.register(55002, "gpke-supplier-change");
2242            }
2243        }
2244
2245        let ctx = EngineBuilder::new()
2246            .with_event_store(InMemoryEventStore::new())
2247            .register(Box::new(PidModule))
2248            .build();
2249
2250        assert_eq!(ctx.pid_router().route(55001), Some("gpke-supplier-change"));
2251        assert_eq!(ctx.pid_router().route(55002), Some("gpke-supplier-change"));
2252        assert!(ctx.pid_router().route(99999).is_none());
2253        assert_eq!(ctx.pid_router().len(), 2);
2254    }
2255
2256    /// Verify that `register_pids_with_roles` gates PIDs behind role checks.
2257    ///
2258    /// Scenario: two modules share PID 19001.
2259    /// - ModuleA registers 19001 → "workflow-a" when role `Nb` is present.
2260    /// - ModuleB registers 19001 → "workflow-b" when role `Nmsb` is explicitly set
2261    ///   (not on `all()`).
2262    ///
2263    /// - `all()`: ModuleA fires (Nb ∈ all), ModuleB does NOT (is_all → skip).
2264    ///   → 19001 routes to "workflow-a".
2265    /// - `from_roles([Nb])`: ModuleA fires, ModuleB skips.
2266    ///   → 19001 routes to "workflow-a".
2267    /// - `from_roles([Nmsb])`: ModuleA skips, ModuleB fires.
2268    ///   → 19001 routes to "workflow-b".
2269    #[test]
2270    fn register_pids_with_roles_gates_pids_correctly() {
2271        use crate::marktrolle::{DeploymentRoles, Marktrolle};
2272
2273        struct ModuleA;
2274        impl EngineModule for ModuleA {
2275            fn name(&self) -> &'static str {
2276                "module-a"
2277            }
2278            fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
2279                if roles.contains(Marktrolle::Nb) {
2280                    router.register(19_001, "workflow-a");
2281                }
2282            }
2283        }
2284
2285        struct ModuleB;
2286        impl EngineModule for ModuleB {
2287            fn name(&self) -> &'static str {
2288                "module-b"
2289            }
2290            fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
2291                // Only fires on explicit Nmsb, not on all() (backward-compat sentinel).
2292                if !roles.is_all() && roles.contains(Marktrolle::Nmsb) {
2293                    router.register(19_001, "workflow-b");
2294                    router.register(19_015, "workflow-b");
2295                }
2296            }
2297        }
2298
2299        let build = |roles: DeploymentRoles| {
2300            EngineBuilder::new()
2301                .with_event_store(InMemoryEventStore::new())
2302                .with_deployment_roles(roles)
2303                .register(Box::new(ModuleA))
2304                .register(Box::new(ModuleB))
2305                .build()
2306        };
2307
2308        // all() → backward compat: ModuleA registers 19001 (Nb ∈ all), ModuleB skips.
2309        let ctx = build(DeploymentRoles::all());
2310        assert_eq!(ctx.pid_router().route(19_001), Some("workflow-a"));
2311        assert!(ctx.pid_router().route(19_015).is_none());
2312
2313        // Explicit Nb → same result: ModuleA registers, ModuleB (nMSB) skips.
2314        let ctx = build(DeploymentRoles::nb());
2315        assert_eq!(ctx.pid_router().route(19_001), Some("workflow-a"));
2316        assert!(ctx.pid_router().route(19_015).is_none());
2317
2318        // Explicit Nmsb → ModuleA skips (Nb ∉ roles), ModuleB registers.
2319        let ctx = build(DeploymentRoles::nmsb());
2320        assert_eq!(ctx.pid_router().route(19_001), Some("workflow-b"));
2321        assert_eq!(ctx.pid_router().route(19_015), Some("workflow-b"));
2322    }
2323
2324    /// Verify that explicit roles with two conflicting modules use first-wins semantics
2325    /// (the first module to register a PID retains ownership; the second is silently skipped).
2326    #[test]
2327    fn register_pids_with_roles_conflict_uses_first_wins_with_explicit_roles() {
2328        use crate::marktrolle::{DeploymentRoles, Marktrolle};
2329
2330        struct ConflictA;
2331        impl EngineModule for ConflictA {
2332            fn name(&self) -> &'static str {
2333                "conflict-a"
2334            }
2335            fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
2336                if roles.contains(Marktrolle::Nb) {
2337                    router.register(19_001, "workflow-a");
2338                }
2339            }
2340        }
2341
2342        struct ConflictB;
2343        impl EngineModule for ConflictB {
2344            fn name(&self) -> &'static str {
2345                "conflict-b"
2346            }
2347            fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
2348                if !roles.is_all() && roles.contains(Marktrolle::Nmsb) {
2349                    router.register(19_001, "workflow-b"); // same PID, different workflow
2350                }
2351            }
2352        }
2353
2354        // from_roles([Nb, Nmsb]): both modules fire for PID 19_001.
2355        // First-wins: ConflictA (registered first) retains ownership → "workflow-a".
2356        let ctx = EngineBuilder::new()
2357            .with_event_store(InMemoryEventStore::new())
2358            .with_deployment_roles(DeploymentRoles::from_roles([
2359                Marktrolle::Nb,
2360                Marktrolle::Nmsb,
2361            ]))
2362            .register(Box::new(ConflictA))
2363            .register(Box::new(ConflictB))
2364            .build();
2365        assert_eq!(
2366            ctx.pid_router().route(19_001),
2367            Some("workflow-a"),
2368            "first module should win on PID conflict with explicit roles"
2369        );
2370    }
2371
2372    // ── Graceful shutdown ─────────────────────────────────────────────────────
2373
2374    /// Cancelling the token must make `run` return.
2375    ///
2376    /// A worker that does not read the token loops until the process exits,
2377    /// and dropping its `JoinHandle` does not abort a Tokio task — so the event
2378    /// store would close underneath a worker still running. An outbox
2379    /// `acknowledge` losing that race leaves the counterparty holding a message
2380    /// the outbox still shows as pending, and the next start delivers it
2381    /// again.
2382    #[tokio::test]
2383    async fn a_cancelled_outbox_worker_returns() {
2384        let worker = OutboxWorker {
2385            store: InMemoryOutboxStore::new(),
2386            sender: AlwaysDelivers,
2387            deadline_store: InMemoryDeadlineStore::new(),
2388            batch_size: 10,
2389            // Far longer than the timeout below: the point is that cancellation
2390            // interrupts the idle sleep rather than being noticed after it.
2391            poll_interval: std::time::Duration::from_secs(300),
2392            max_attempts: 48,
2393            max_retry_window: std::time::Duration::from_secs(72 * 3600),
2394            dead_letter_sink: std::sync::Arc::new(crate::dead_letter::LogDeadLetterSink),
2395            heartbeat: None,
2396            shutdown: None,
2397        };
2398        let token = tokio_util::sync::CancellationToken::new();
2399        let worker = worker.with_shutdown(token.clone());
2400
2401        let handle = tokio::spawn(worker.run());
2402        // Let it reach the sleep, then signal.
2403        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2404        token.cancel();
2405
2406        tokio::time::timeout(std::time::Duration::from_secs(5), handle)
2407            .await
2408            .expect("outbox worker must return promptly after cancellation")
2409            .expect("outbox worker must not panic");
2410    }
2411
2412    #[tokio::test]
2413    async fn a_cancelled_deadline_scheduler_returns() {
2414        let scheduler = DeadlineScheduler {
2415            store: InMemoryDeadlineStore::new(),
2416            dispatch: Box::new(|_| Box::pin(async { Ok(()) })),
2417            batch_size: 100,
2418            poll_interval: std::time::Duration::from_secs(300),
2419            heartbeat: None,
2420            shutdown: None,
2421        };
2422        let token = tokio_util::sync::CancellationToken::new();
2423        let scheduler = scheduler.with_shutdown(token.clone());
2424
2425        let handle = tokio::spawn(scheduler.run());
2426        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2427        token.cancel();
2428
2429        tokio::time::timeout(std::time::Duration::from_secs(5), handle)
2430            .await
2431            .expect("deadline scheduler must return promptly after cancellation")
2432            .expect("deadline scheduler must not panic");
2433    }
2434
2435    /// A token cancelled before the first poll must stop the worker without it
2436    /// touching the store at all — the case where shutdown arrives during boot.
2437    #[tokio::test]
2438    async fn a_worker_cancelled_before_it_starts_does_no_work() {
2439        let outbox = InMemoryOutboxStore::new();
2440        let stream_id = crate::ids::StreamId::new("gpke/shutdown-test");
2441        let msg = outbox_message(&stream_id, "UTILMD");
2442        outbox.enqueue(std::slice::from_ref(&msg)).await.unwrap();
2443
2444        let token = tokio_util::sync::CancellationToken::new();
2445        token.cancel();
2446
2447        let worker = OutboxWorker {
2448            store: outbox.clone(),
2449            sender: AlwaysDelivers,
2450            deadline_store: InMemoryDeadlineStore::new(),
2451            batch_size: 10,
2452            poll_interval: std::time::Duration::from_millis(5),
2453            max_attempts: 48,
2454            max_retry_window: std::time::Duration::from_secs(72 * 3600),
2455            dead_letter_sink: std::sync::Arc::new(crate::dead_letter::LogDeadLetterSink),
2456            heartbeat: None,
2457            shutdown: None,
2458        }
2459        .with_shutdown(token);
2460
2461        tokio::time::timeout(std::time::Duration::from_secs(5), worker.run())
2462            .await
2463            .expect("an already-cancelled worker must return immediately");
2464
2465        assert_eq!(
2466            outbox.pending_now(10).await.unwrap().len(),
2467            1,
2468            "the message must stay queued for the next start, not be delivered \
2469             by a worker that was told to stop",
2470        );
2471    }
2472
2473    // ── APERAK delivery-window discharge ──────────────────────────────────────
2474
2475    /// A sender that always succeeds, so the worker takes the delivery path.
2476    struct AlwaysDelivers;
2477    impl As4Sender for AlwaysDelivers {
2478        async fn send(&self, _msg: &crate::outbox::OutboxMessage) -> Result<(), EngineError> {
2479            Ok(())
2480        }
2481    }
2482
2483    fn outbox_message(
2484        stream_id: &crate::ids::StreamId,
2485        message_type: &str,
2486    ) -> crate::outbox::OutboxMessage {
2487        crate::outbox::OutboxMessage::new(
2488            stream_id.clone(),
2489            crate::ids::ProcessId::new(),
2490            TenantId::new(),
2491            crate::ids::CorrelationId::new(),
2492            crate::ids::ConversationId::new(),
2493            crate::ids::EventId::new(),
2494            message_type,
2495            "9900357000004",
2496            serde_json::json!({}),
2497        )
2498    }
2499
2500    fn deadline_on(
2501        stream_id: &crate::ids::StreamId,
2502        msg: &crate::outbox::OutboxMessage,
2503        label: &str,
2504    ) -> Deadline {
2505        Deadline::new(
2506            stream_id.clone(),
2507            msg.process_id,
2508            msg.tenant_id,
2509            WorkflowId::new("gpke-supplier-change", "FV2025-10-01"),
2510            label,
2511            time::OffsetDateTime::now_utc() + time::Duration::hours(6),
2512        )
2513    }
2514
2515    /// Deliver `msg` through the worker's real loop and return the labels that
2516    /// survive on its stream.
2517    ///
2518    /// Drives `run` rather than calling the discharge directly — the wiring is
2519    /// the thing under test, and calling the method straight passes even when
2520    /// `run` never invokes it.
2521    async fn labels_surviving_delivery(
2522        msg: &crate::outbox::OutboxMessage,
2523        stream_id: &crate::ids::StreamId,
2524        registered: &[&str],
2525    ) -> Vec<String> {
2526        let deadlines = InMemoryDeadlineStore::new();
2527        for label in registered {
2528            deadlines
2529                .register(&deadline_on(stream_id, msg, label))
2530                .await
2531                .unwrap();
2532        }
2533        let outbox = InMemoryOutboxStore::new();
2534        outbox.enqueue(std::slice::from_ref(msg)).await.unwrap();
2535
2536        let worker = OutboxWorker {
2537            store: outbox.clone(),
2538            sender: AlwaysDelivers,
2539            deadline_store: deadlines.clone(),
2540            batch_size: 10,
2541            poll_interval: std::time::Duration::from_millis(5),
2542            max_attempts: 48,
2543            max_retry_window: std::time::Duration::from_secs(72 * 3600),
2544            dead_letter_sink: std::sync::Arc::new(crate::dead_letter::LogDeadLetterSink),
2545            heartbeat: None,
2546            shutdown: None,
2547        };
2548        // `run` never returns; give it enough cycles to drain the one message.
2549        let _ = tokio::time::timeout(std::time::Duration::from_millis(300), worker.run()).await;
2550        assert!(
2551            outbox.pending_now(10).await.unwrap().is_empty(),
2552            "the message must have been delivered and acknowledged",
2553        );
2554
2555        let mut left: Vec<String> = deadlines
2556            .for_stream(stream_id)
2557            .await
2558            .unwrap()
2559            .iter()
2560            .map(|d| d.label().to_owned())
2561            .collect();
2562        left.sort();
2563        left
2564    }
2565
2566    /// Delivering a message must retire the window that was watching for it.
2567    ///
2568    /// These windows are registered when the message is enqueued and nothing
2569    /// else ever cancels them, so without the discharge they fire for **every**
2570    /// process — including every one that answered on time. The scheduler cannot
2571    /// tell those apart (a deadline reaching `due_now` is late by construction),
2572    /// so the miss counters would track processes started, not obligations
2573    /// missed.
2574    #[tokio::test]
2575    async fn delivering_a_message_discharges_its_delivery_window() {
2576        // (message type, the window it answers for)
2577        for (message_type, window) in [
2578            ("APERAK", mako_fristen::APERAK_STROM_WINDOW_LABEL),
2579            ("APERAK", mako_fristen::APERAK_GAS_FOLGEPROZESS_LABEL),
2580            ("APERAK", mako_fristen::APERAK_GAS_INITIALPROZESS_LABEL),
2581            ("CONTRL", mako_fristen::CONTRL_FRIST_LABEL),
2582        ] {
2583            let stream_id = crate::ids::StreamId::new("gpke-supplier-change-1");
2584            let msg = outbox_message(&stream_id, message_type);
2585            // A process-response deadline shares the stream and must survive:
2586            // it is waiting on the counterparty, not on our delivery.
2587            let left =
2588                labels_surviving_delivery(&msg, &stream_id, &[window, "gpke-response-window"])
2589                    .await;
2590            assert_eq!(
2591                left,
2592                vec!["gpke-response-window"],
2593                "delivering {message_type} must discharge `{window}` and leave \
2594                 every other deadline alone",
2595            );
2596        }
2597    }
2598
2599    /// A delivery must not discharge a *different* message's window.
2600    ///
2601    /// The CONTRL and APERAK obligations run concurrently on the same
2602    /// interchange. Acknowledging syntax (CONTRL) says nothing about whether the
2603    /// application-level APERAK went out, so discharging both on one delivery
2604    /// would silence a real violation.
2605    #[tokio::test]
2606    async fn a_delivery_does_not_discharge_another_messages_window() {
2607        let stream_id = crate::ids::StreamId::new("gpke-supplier-change-1");
2608        let contrl = outbox_message(&stream_id, "CONTRL");
2609
2610        let left = labels_surviving_delivery(
2611            &contrl,
2612            &stream_id,
2613            &[
2614                mako_fristen::CONTRL_FRIST_LABEL,
2615                mako_fristen::APERAK_STROM_WINDOW_LABEL,
2616            ],
2617        )
2618        .await;
2619
2620        assert_eq!(
2621            left,
2622            vec![mako_fristen::APERAK_STROM_WINDOW_LABEL.to_owned()],
2623            "a delivered CONTRL discharges only the CONTRL window; the APERAK \
2624             obligation is still outstanding",
2625        );
2626    }
2627
2628    /// Every delivery-window label must be discharged by the message it watches.
2629    ///
2630    /// This is the invariant the miss counters rest on. A window label that
2631    /// `discharges_delivery_window` does not recognise is never retired, so it
2632    /// fires on every process and is counted as a regulatory violation each
2633    /// time — which is precisely how `makod_aperak_missed_total` once came to
2634    /// count Strom processes rather than missed APERAKs.
2635    ///
2636    /// Adding a delivery window means adding a row here.
2637    #[test]
2638    fn every_delivery_window_label_is_discharged_by_its_message() {
2639        for (message_type, label) in [
2640            ("APERAK", mako_fristen::APERAK_STROM_WINDOW_LABEL),
2641            ("APERAK", mako_fristen::APERAK_GAS_FOLGEPROZESS_LABEL),
2642            ("APERAK", mako_fristen::APERAK_GAS_INITIALPROZESS_LABEL),
2643            ("CONTRL", mako_fristen::CONTRL_FRIST_LABEL),
2644        ] {
2645            assert!(
2646                mako_fristen::discharges_delivery_window(message_type, label),
2647                "delivering {message_type} must discharge `{label}`, or the window \
2648                 outlives its obligation and alerts on every process",
2649            );
2650        }
2651    }
2652
2653    // ── Retry-budget classification ───────────────────────────────────────────
2654
2655    /// Sink double that records every rejection's attempt count.
2656    #[derive(Default)]
2657    struct RecordingSink(std::sync::Mutex<Vec<u32>>);
2658    impl crate::dead_letter::DeadLetterSink for std::sync::Arc<RecordingSink> {
2659        fn reject(&self, reason: &crate::dead_letter::DeadLetterReason) {
2660            if let crate::dead_letter::DeadLetterReason::OutboxExhausted { attempts, .. } = reason {
2661                self.0
2662                    .lock()
2663                    .unwrap_or_else(std::sync::PoisonError::into_inner)
2664                    .push(*attempts);
2665            }
2666        }
2667    }
2668
2669    struct NoRenderer;
2670    impl As4Sender for NoRenderer {
2671        async fn send(&self, msg: &crate::outbox::OutboxMessage) -> Result<(), EngineError> {
2672            Err(EngineError::RendererNotImplemented {
2673                message_type: msg.message_type.as_ref().into(),
2674                message_id: msg.message_id.to_string().into(),
2675            })
2676        }
2677    }
2678
2679    async fn drive_worker<S: As4Sender>(
2680        sender: S,
2681        msg: crate::outbox::OutboxMessage,
2682    ) -> (InMemoryOutboxStore, std::sync::Arc<RecordingSink>) {
2683        let outbox = InMemoryOutboxStore::new();
2684        outbox.enqueue(std::slice::from_ref(&msg)).await.unwrap();
2685        let sink = std::sync::Arc::new(RecordingSink::default());
2686        let worker = OutboxWorker {
2687            store: outbox.clone(),
2688            sender,
2689            deadline_store: InMemoryDeadlineStore::new(),
2690            batch_size: 10,
2691            poll_interval: std::time::Duration::from_millis(5),
2692            max_attempts: 48,
2693            max_retry_window: std::time::Duration::from_secs(72 * 3600),
2694            dead_letter_sink: std::sync::Arc::new(std::sync::Arc::clone(&sink)),
2695            heartbeat: None,
2696            shutdown: None,
2697        };
2698        let _ = tokio::time::timeout(std::time::Duration::from_millis(300), worker.run()).await;
2699        (outbox, sink)
2700    }
2701
2702    /// `RendererNotImplemented` is documented as permanent — the worker must
2703    /// dead-letter it on the *first* attempt, not burn the retry budget on a
2704    /// failure that cannot heal between attempts. Until the permanent arm
2705    /// matched it, this promise was broken.
2706    #[tokio::test(start_paused = true)]
2707    async fn a_missing_renderer_dead_letters_without_retrying() {
2708        let stream_id = crate::ids::StreamId::new("test-renderer-missing");
2709        let msg = outbox_message(&stream_id, "MSCONS");
2710        let (outbox, sink) = drive_worker(NoRenderer, msg).await;
2711
2712        assert!(
2713            outbox.pending_now(10).await.unwrap().is_empty(),
2714            "the message must be acknowledged, not left for another attempt",
2715        );
2716        let rejections = sink
2717            .0
2718            .lock()
2719            .unwrap_or_else(std::sync::PoisonError::into_inner)
2720            .clone();
2721        assert_eq!(
2722            rejections,
2723            vec![0],
2724            "exactly one dead-letter, on the first attempt (attempt_count 0)",
2725        );
2726    }
2727
2728    /// The retry budget is a *window*, not a count: a message whose age has
2729    /// exceeded it after at least one attempt is dead-lettered even though the
2730    /// attempt belt is nowhere near exhausted — full-jitter backoff makes a
2731    /// count no proxy for the 72 h duty.
2732    #[tokio::test(start_paused = true)]
2733    async fn an_aged_message_with_a_prior_attempt_is_dead_lettered() {
2734        let stream_id = crate::ids::StreamId::new("test-window-exhausted");
2735        let mut msg = outbox_message(&stream_id, "UTILMD");
2736        msg.created_at = time::OffsetDateTime::now_utc() - time::Duration::hours(73);
2737        msg.attempt_count = 1;
2738        let (outbox, sink) = drive_worker(AlwaysDelivers, msg).await;
2739
2740        assert!(
2741            outbox.pending_now(10).await.unwrap().is_empty(),
2742            "the exhausted message must leave the outbox",
2743        );
2744        let rejections = sink
2745            .0
2746            .lock()
2747            .unwrap_or_else(std::sync::PoisonError::into_inner)
2748            .clone();
2749        assert_eq!(
2750            rejections,
2751            vec![1],
2752            "the window, not the attempt belt, must have dead-lettered it",
2753        );
2754    }
2755
2756    /// A message that aged past the window while the worker was down still
2757    /// gets its first try — the window is only consulted after an attempt, so
2758    /// downtime never buries a message unsent.
2759    #[tokio::test(start_paused = true)]
2760    async fn an_aged_message_with_no_attempts_is_still_tried_once() {
2761        let stream_id = crate::ids::StreamId::new("test-aged-first-try");
2762        let mut msg = outbox_message(&stream_id, "UTILMD");
2763        msg.created_at = time::OffsetDateTime::now_utc() - time::Duration::hours(200);
2764        let (outbox, sink) = drive_worker(AlwaysDelivers, msg).await;
2765
2766        assert!(
2767            outbox.pending_now(10).await.unwrap().is_empty(),
2768            "the message must have been delivered and acknowledged",
2769        );
2770        assert!(
2771            sink.0
2772                .lock()
2773                .unwrap_or_else(std::sync::PoisonError::into_inner)
2774                .is_empty(),
2775            "delivery, not dead-lettering: age alone must never bury a message",
2776        );
2777    }
2778}