mako_engine/builder.rs
1//! [`EngineModule`] trait, [`EngineBuilder`], and [`EngineContext`].
2//!
3// Allow using deprecated Noop stores as *type-level defaults* in EngineBuilder / EngineContext
4// generic parameters. The types are deprecated to prevent instantiation in production code,
5// but using them as default type parameters in struct definitions (not instantiating them) is
6// the intended pattern for the type-state builder API.
7#![allow(deprecated)]
8//!
9//! # Summary
10//!
11//! `EngineBuilder` assembles all engine infrastructure into a single
12//! [`EngineContext`] value. Domain modules (GPKE, WiM, GeLi Gas, …) register
13//! themselves at startup via the [`EngineModule`] trait, making their names
14//! visible in diagnostics and health checks.
15//!
16//! # Type-state guarantee
17//!
18//! [`EngineBuilder::build`] is only available when the event store type
19//! parameter `ES` implements [`EventStore`]. Forgetting to call
20//! [`with_event_store`] is a **compile-time error**, not a runtime panic.
21//!
22//! All other stores default to their respective `Noop` implementations:
23//!
24//! | Store | Default |
25//! |-------|---------|
26//! | Snapshot store | [`NoopSnapshotStore`] |
27//! | Outbox store | [`NoopOutboxStore`] |
28//! | Deadline store | [`NoopDeadlineStore`] |
29//! | Process registry | [`NoopProcessRegistry`] |
30//!
31//! # Assembly example
32//!
33//! ```rust,ignore
34//! use mako_engine::builder::{EngineBuilder, EngineModule};
35//! use mako_engine::event_store::InMemoryEventStore;
36//! use mako_engine::outbox::InMemoryOutboxStore;
37//! use mako_engine::deadline::InMemoryDeadlineStore;
38//! use mako_engine::registry::InMemoryProcessRegistry;
39//! use mako_engine::snapshot::InMemorySnapshotStore;
40//!
41//! struct GpkeModule;
42//! impl EngineModule for GpkeModule { fn name(&self) -> &'static str { "gpke" } }
43//!
44//! let ctx = EngineBuilder::new()
45//! .with_event_store(InMemoryEventStore::new())
46//! .with_snapshot_store(InMemorySnapshotStore::new())
47//! .with_outbox_store(InMemoryOutboxStore::new())
48//! .with_deadline_store(InMemoryDeadlineStore::new())
49//! .with_registry(InMemoryProcessRegistry::new())
50//! .register(Box::new(GpkeModule))
51//! .build();
52//!
53//! // Spawn a fresh process:
54//! let p = ctx.spawn::<SupplierChangeWorkflow>(tenant_id, workflow_id);
55//! p.execute(ReceiveUtilmd { .. }).await?;
56//!
57//! // Resume an existing process from a persisted identity:
58//! let identity = ctx.registry.lookup(&conv_id.to_string()).await?.unwrap();
59//! let p = ctx.resume::<SupplierChangeWorkflow>(identity);
60//!
61//! // Access stores for delivery workers / schedulers:
62//! let pending = ctx.outbox_store.pending_now(50).await?;
63//! let overdue = ctx.deadline_store.due_now(50).await?;
64//! ```
65//!
66//! [`with_event_store`]: EngineBuilder::with_event_store
67
68// Type-state generics can produce long signatures that trip up the
69// `type_complexity` lint; suppress it for this module only.
70#![allow(clippy::type_complexity)]
71
72// The Noop* types are marked #[deprecated] to guard against accidental
73// production use. The builder is the only place they're instantiated as
74// defaults; suppress the lint here explicitly.
75#[allow(deprecated)]
76use crate::{
77 dead_letter::{DeadLetterSink, LogDeadLetterSink},
78 deadline::{Deadline, DeadlineStore, NoopDeadlineStore},
79 error::EngineError,
80 event_store::EventStore,
81 ids::{ProcessIdentity, TenantId},
82 marktrolle::DeploymentRoles,
83 outbox::{NoopOutboxStore, OutboxMessage, OutboxStore},
84 pid_router::PidRouter,
85 process::Process,
86 registry::{NoopProcessRegistry, ProcessRegistry},
87 snapshot::{NoopSnapshotStore, SnapshotStore},
88 version::WorkflowId,
89 workflow::Workflow,
90};
91
92use std::sync::Arc;
93
94// ── EngineModule ──────────────────────────────────────────────────────────────
95
96/// A self-contained domain module that registers with the engine at startup.
97///
98/// Domain crates implement this trait to declare their presence in the engine.
99/// The module name is surfaced in [`EngineContext::registered_modules`] for
100/// diagnostics, health checks, and log output.
101///
102/// ## Startup validation
103///
104/// Override [`configure`] to perform adapter coverage checks at engine startup
105/// time. The engine calls [`configure`] for every registered module during
106/// [`EngineBuilder::build`] and panics with an actionable message if any
107/// module returns `Err`. This surfaces missing adapter registrations as a
108/// startup failure rather than a silent runtime error.
109///
110/// ## Example
111///
112/// ```rust,ignore
113/// pub struct GpkeModule;
114///
115/// impl EngineModule for GpkeModule {
116/// fn name(&self) -> &'static str { "gpke" }
117///
118/// fn configure(&self) -> Result<(), String> {
119/// // Validate that every known BDEW format version has an adapter:
120/// GPKE_ADAPTER_REGISTRY
121/// .validate_policy(&GpkeWorkflow::version_policy(), &KNOWN_FVS)
122/// .map_err(|uncovered| format!(
123/// "gpke: missing adapters for format versions: {:?}",
124/// uncovered
125/// ))
126/// }
127/// }
128///
129/// let ctx = EngineBuilder::new()
130/// .with_event_store(my_store)
131/// .register(Box::new(GpkeModule))
132/// .build(); // panics if GpkeModule::configure returns Err
133///
134/// assert_eq!(ctx.registered_modules(), &["gpke"]);
135/// ```
136///
137/// [`configure`]: EngineModule::configure
138pub trait EngineModule: Send + 'static {
139 /// Stable, unique name for this domain module.
140 ///
141 /// Used in diagnostics, health checks, and structured log output.
142 /// Choose a short lowercase identifier (e.g. `"gpke"`, `"wim"`,
143 /// `"geli"`).
144 fn name(&self) -> &'static str;
145
146 /// Register all PIDs this module handles into the shared [`PidRouter`].
147 ///
148 /// # Mutability contract
149 ///
150 /// This method is called **exactly once** by [`EngineBuilder::build`],
151 /// before the resulting [`EngineContext`] is handed to the caller. The
152 /// `&mut PidRouter` reference is only available here, at build time.
153 /// After `build` returns the router is **sealed** — the engine provides
154 /// only a shared `&PidRouter` reference, with no mutation path at runtime.
155 ///
156 /// Consequence: **all PIDs a module will ever need must be registered
157 /// here**. Do not attempt to register PIDs lazily from async handlers or
158 /// after the engine has started — there is no API for that by design.
159 ///
160 /// Duplicate registrations (same PID from two modules) silently overwrite
161 /// the previous mapping; the last module to register wins. Use
162 /// `cargo xtask validate-pruefids` to catch accidental PID conflicts
163 /// between modules before they reach production.
164 ///
165 /// For role-conditional registration (PIDs that should only be active for
166 /// specific BDEW Marktrollen), override [`register_pids_with_roles`] instead.
167 ///
168 /// # Example
169 ///
170 /// ```rust,ignore
171 /// fn register_pids(&self, router: &mut PidRouter) {
172 /// // GPKE Lieferantenwechsel / Lieferbeginn (BK6-22-024, PIDs 55001, 55002, 55017)
173 /// for &pid in &[55001_u32, 55002, 55017] {
174 /// router.register(pid, "gpke-supplier-change");
175 /// }
176 /// }
177 /// ```
178 ///
179 /// [`register_pids_with_roles`]: EngineModule::register_pids_with_roles
180 fn register_pids(&self, _router: &mut PidRouter) {}
181
182 /// Register PIDs with role-context awareness.
183 ///
184 /// This is the **preferred override** for modules that have role-conditional
185 /// PID registrations — PIDs that should only be active when this `makod`
186 /// instance holds a specific [`Marktrolle`].
187 ///
188 /// The default implementation calls [`register_pids`] (role-agnostic) so
189 /// existing modules that override `register_pids` continue to work without
190 /// changes.
191 ///
192 /// Override this method instead of `register_pids` when any PID registration
193 /// should be conditional on the deployment role:
194 ///
195 /// ```rust,ignore
196 /// use mako_engine::marktrolle::Marktrolle;
197 ///
198 /// fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
199 /// // Always register: 55001, 55002 (not role-specific)
200 /// for pid in [55001_u32, 55002] { router.register_with_module(pid, "gpke-supplier-change", self.name()); }
201 ///
202 /// // Only when NB role: 19001/19002 inbound ORDRSP from MSB
203 /// if roles.contains(Marktrolle::Nb) {
204 /// for pid in [19001_u32, 19002] { router.register_with_module(pid, "gpke-konfiguration", self.name()); }
205 /// }
206 /// }
207 /// ```
208 ///
209 /// # Conflict guard
210 ///
211 /// Use [`PidRouter::register_with_module`] (not `register`) inside this
212 /// method. The conflict guard panics at build time if two modules register
213 /// the same PID to different workflows — this makes role misconfigurations
214 /// visible at startup rather than silently misrouting messages.
215 ///
216 /// [`Marktrolle`]: crate::marktrolle::Marktrolle
217 /// [`register_pids`]: EngineModule::register_pids
218 fn register_pids_with_roles(&self, router: &mut PidRouter, _roles: &DeploymentRoles) {
219 self.register_pids(router);
220 }
221
222 /// Workflow names this module handles for deadline dispatch.
223 ///
224 /// Return the same name strings that [`register_pids`] maps PIDs to.
225 /// These names are stored in [`EngineContext::registered_workflows`] and
226 /// used to validate that every workflow that has deadlines scheduled is
227 /// covered by the deadline scheduler dispatch function at runtime.
228 ///
229 /// The default implementation returns an empty slice. Override it to
230 /// declare all workflow names that may fire deadlines:
231 ///
232 /// ```rust,ignore
233 /// fn workflow_names(&self) -> &'static [&'static str] {
234 /// &["gpke-supplier-change", "gpke-abrechnung"]
235 /// }
236 /// ```
237 ///
238 /// [`register_pids`]: EngineModule::register_pids
239 /// [`EngineContext::registered_workflows`]: crate::builder::EngineContext::registered_workflows
240 fn workflow_names(&self) -> &'static [&'static str] {
241 &[]
242 }
243
244 /// Declare the EDIFACT profile types this module requires at runtime.
245 ///
246 /// Returning a non-empty slice causes [`EngineBuilder::build`] to call the
247 /// registered profile validator for each requirement. If no active profile
248 /// exists for a required message type, `build` panics with an actionable
249 /// error so deployment fails fast rather than silently.
250 ///
251 /// Domain crates declare their format requirements here rather than
252 /// calling `edi_energy::registry::ReleaseRegistry::global()` inside
253 /// `configure()`, so `edi-energy` stays out of their production
254 /// `[dependencies]`.
255 ///
256 /// ```rust,ignore
257 /// fn profile_requirements(&self) -> &'static [ProfileRequirement] {
258 /// &[
259 /// ProfileRequirement { message_type: "UTILMD", label: "UTILMD Strom (GPKE)" },
260 /// ProfileRequirement { message_type: "INVOIC", label: "INVOIC Abrechnung (GPKE)" },
261 /// ]
262 /// }
263 /// ```
264 ///
265 /// [`ProfileRequirement`]: crate::profile::ProfileRequirement
266 fn profile_requirements(&self) -> &'static [crate::profile::ProfileRequirement] {
267 &[]
268 }
269
270 /// Validate adapter coverage and configuration at engine startup.
271 ///
272 /// Called by [`EngineBuilder::build`] after all modules are registered.
273 /// Return `Ok(())` when the module is fully configured. Return `Err(msg)`
274 /// with an actionable description when an adapter or configuration is
275 /// missing — the engine will panic with that message so the deployment
276 /// fails early rather than silently.
277 ///
278 /// The default implementation is a no-op (always returns `Ok(())`).
279 /// Override it in domain crates to call
280 /// [`AdapterRegistry::validate_policy`] and emit structured errors.
281 ///
282 /// Note: if your validation needs access to the edi-energy profile
283 /// registry, use [`profile_requirements`] instead — it does not require
284 /// importing `edi-energy` in domain crates.
285 ///
286 /// [`AdapterRegistry::validate_policy`]: crate::message_adapter::AdapterRegistry::validate_policy
287 /// [`profile_requirements`]: EngineModule::profile_requirements
288 ///
289 /// # Errors
290 ///
291 /// Returns a descriptive error string when the module's configuration is invalid.
292 fn configure(&self) -> Result<(), String> {
293 Ok(())
294 }
295}
296
297// ── EngineContext ─────────────────────────────────────────────────────────────
298
299/// Assembled engine infrastructure returned by [`EngineBuilder::build`].
300///
301/// `EngineContext` bundles all stores and the process registry into a single
302/// value. It is the root dependency for:
303///
304/// - Spawning new processes ([`spawn`])
305/// - Resuming existing processes ([`resume`])
306/// - Running outbox delivery workers (`outbox_store.pending_now(…)`)
307/// - Driving the deadline scheduler (`deadline_store.due_now(…)`)
308///
309/// ## Generic parameters
310///
311/// | Param | Role | Default |
312/// |-------|------|---------|
313/// | `ES` | [`EventStore`] backend | — (required) |
314/// | `SS` | [`SnapshotStore`] backend | [`NoopSnapshotStore`] |
315/// | `OS` | [`OutboxStore`] backend | [`NoopOutboxStore`] |
316/// | `DS` | [`DeadlineStore`] backend | [`NoopDeadlineStore`] |
317/// | `PR` | [`ProcessRegistry`] backend | [`NoopProcessRegistry`] |
318///
319/// In most codebases all type parameters are inferred from the builder calls.
320///
321/// [`spawn`]: EngineContext::spawn
322/// [`resume`]: EngineContext::resume
323pub struct EngineContext<
324 ES,
325 SS = NoopSnapshotStore,
326 OS = NoopOutboxStore,
327 DS = NoopDeadlineStore,
328 PR = NoopProcessRegistry,
329> {
330 event_store: Arc<ES>,
331 snapshot_store: SS,
332 outbox_store: OS,
333 deadline_store: DS,
334 registry: PR,
335 /// Dead-letter sink for unroutable or unprocessable inbound messages.
336 ///
337 /// Stored as `Arc<dyn DeadLetterSink>` so callers can share it across
338 /// tasks without an extra type parameter on `EngineContext`.
339 pub dead_letter_sink: Arc<dyn DeadLetterSink>,
340 /// PID-to-workflow routing table, populated from all registered modules.
341 pid_router: PidRouter,
342 registered_modules: Vec<&'static str>,
343 /// Workflow names declared by all registered modules via
344 /// [`EngineModule::workflow_names`]. Used to validate deadline scheduler
345 /// coverage at runtime (see [`EngineContext::registered_workflows`]).
346 registered_workflows: Vec<&'static str>,
347}
348
349// ── Type aliases ──────────────────────────────────────────────────────────────
350
351/// An [`EngineContext`] with all optional subsystems disabled.
352///
353/// Uses `NoopSnapshotStore` and, in `testing`-enabled builds, Noop
354/// implementations for outbox, deadline, and process registry. Suitable for
355/// tests and minimal deployments where only a durable event store is required.
356///
357/// All five type parameters are inferred from context when used with
358/// [`EngineBuilder`]:
359///
360/// ```rust,ignore
361/// // Only available in test / testing-feature builds:
362/// use mako_engine::builder::{EngineBuilder, MinimalEngine};
363/// use mako_engine::event_store::InMemoryEventStore;
364///
365/// let ctx: MinimalEngine<InMemoryEventStore> = EngineBuilder::new()
366/// .with_event_store(InMemoryEventStore::new())
367/// .build();
368/// ```
369pub type MinimalEngine<ES> = EngineContext<ES>;
370
371impl<ES, SS, OS, DS, PR> EngineContext<ES, SS, OS, DS, PR>
372where
373 ES: EventStore,
374{
375 /// Spawn a new process and return a typed `Process<W, Arc<ES>>` handle.
376 ///
377 /// No `ES: Clone` bound is required — the engine stores the event store
378 /// behind an `Arc` so spawning is always a cheap pointer clone.
379 ///
380 /// ```rust,ignore
381 /// let p = ctx.spawn::<SupplierChangeWorkflow>(tenant_id, workflow_id);
382 /// p.execute(ReceiveUtilmd { .. }).await?;
383 /// ```
384 #[must_use]
385 pub fn spawn<W: Workflow>(
386 &self,
387 tenant_id: TenantId,
388 workflow_id: WorkflowId,
389 ) -> Process<W, Arc<ES>> {
390 Process::new(Arc::clone(&self.event_store), tenant_id, workflow_id)
391 }
392
393 /// Resume an existing process from a [`ProcessIdentity`].
394 ///
395 /// ```rust,ignore
396 /// let identity = ctx.registry()
397 /// .lookup(tenant_id, &conv_id.to_string())
398 /// .await?
399 /// .ok_or(EngineError::Registry("unknown conversation".into()))?;
400 /// let p = ctx.resume::<SupplierChangeWorkflow>(identity);
401 /// p.execute(HandleAperak { .. }).await?;
402 /// ```
403 #[must_use]
404 pub fn resume<W: Workflow>(&self, identity: ProcessIdentity) -> Process<W, Arc<ES>> {
405 Process::from_identity(Arc::clone(&self.event_store), identity)
406 }
407
408 /// Names of all domain modules registered with the builder, in
409 /// registration order.
410 #[must_use]
411 pub fn registered_modules(&self) -> &[&'static str] {
412 &self.registered_modules
413 }
414
415 /// Workflow names declared by all registered modules, in registration order.
416 ///
417 /// Use this in the deadline scheduler dispatch function to detect unknown
418 /// workflow names at startup. If a deadline fires for a workflow name that
419 /// is not in this list, the scheduler's dispatch function should emit an
420 /// error rather than silently dropping the deadline:
421 ///
422 /// ```rust,ignore
423 /// let known = ctx.registered_workflows().iter().copied().collect::<HashSet<_>>();
424 /// let scheduler = ctx.run_deadline_scheduler(
425 /// move |deadline| {
426 /// let wf = deadline.workflow_id().name.as_ref();
427 /// if !known.contains(wf) {
428 /// tracing::error!(workflow = %wf, "deadline fired for unregistered workflow");
429 /// return Box::pin(async { Ok(()) });
430 /// }
431 /// // dispatch by workflow name …
432 /// Box::pin(async { Ok(()) })
433 /// },
434 /// 100,
435 /// Duration::from_secs(30),
436 /// );
437 /// ```
438 #[must_use]
439 pub fn registered_workflows(&self) -> &[&'static str] {
440 &self.registered_workflows
441 }
442
443 /// The event store backend (behind an `Arc`).
444 #[must_use]
445 pub fn event_store(&self) -> &Arc<ES> {
446 &self.event_store
447 }
448
449 /// The snapshot store backend.
450 #[must_use]
451 pub fn snapshot_store(&self) -> &SS {
452 &self.snapshot_store
453 }
454
455 /// The outbox store backend.
456 ///
457 /// Poll `outbox_store().pending_now(limit)` in a background task to drain
458 /// the delivery queue.
459 #[must_use]
460 pub fn outbox_store(&self) -> &OS {
461 &self.outbox_store
462 }
463
464 /// The deadline store backend.
465 ///
466 /// Poll `deadline_store().due_now(limit)` in a background scheduler to
467 /// fire overdue process timers.
468 #[must_use]
469 pub fn deadline_store(&self) -> &DS {
470 &self.deadline_store
471 }
472
473 /// The process routing registry.
474 ///
475 /// Register a [`ProcessIdentity`] under a `(tenant_id, key)` pair at
476 /// process creation, then `lookup` it when routing inbound messages.
477 #[must_use]
478 pub fn registry(&self) -> &PR {
479 &self.registry
480 }
481
482 /// The dead-letter sink for unroutable or unprocessable messages.
483 ///
484 /// Call [`DeadLetterSink::reject`] when an inbound message cannot be
485 /// dispatched to any workflow. The default sink emits `tracing::warn!`
486 /// so rejections are always visible in the log output.
487 #[must_use]
488 pub fn dead_letter_sink(&self) -> &Arc<dyn DeadLetterSink> {
489 &self.dead_letter_sink
490 }
491
492 /// Assert that no Noop store is active — call this during production startup.
493 ///
494 /// Checks the type names of `OS`, `DS`, and `PR` against the string `"Noop"`.
495 /// Panics with a human-readable message if any match, directing the operator
496 /// to configure a persistent backend.
497 ///
498 /// # When to call
499 ///
500 /// Call this early in `makod`'s startup path (and `--check` mode) to catch
501 /// deployments where a Noop store was accidentally wired — e.g. the
502 /// `[outbox]`, `[deadline]`, or `[registry]` configuration section was
503 /// omitted from `makod.toml`. The check is defence-in-depth: in release
504 /// builds without the `testing` feature, Noop stores cannot implement the
505 /// required traits at all and the compiler would have already rejected them.
506 ///
507 /// # Panics
508 ///
509 /// Panics when any of `OS`, `DS`, or `PR` is a Noop implementation.
510 pub fn assert_production_stores(&self) {
511 let checks: &[(&str, &str)] = &[
512 ("OutboxStore", std::any::type_name::<OS>()),
513 ("DeadlineStore", std::any::type_name::<DS>()),
514 ("ProcessRegistry", std::any::type_name::<PR>()),
515 ];
516 for (trait_name, type_name) in checks {
517 assert!(
518 !type_name.contains("Noop"),
519 "makod: Noop{trait_name} is active — \
520 configure a persistent {trait_name} backend in makod.toml. \
521 Type resolved to: {type_name}"
522 );
523 }
524 }
525
526 /// The PID-to-workflow routing table.
527 ///
528 /// Populated **once** during [`EngineBuilder::build`] by calling
529 /// [`EngineModule::register_pids`] on every registered module in
530 /// registration order. After `build` returns the table is **sealed** —
531 /// it is read-only for the lifetime of the `EngineContext` and may be
532 /// freely shared across async tasks without synchronisation.
533 ///
534 /// # Mutability contract
535 ///
536 /// There is intentionally no `pid_router_mut()` accessor. Adding PIDs
537 /// after the engine is built would create a TOCTOU race between the
538 /// dispatch path (which calls `route(pid)`) and any hypothetical
539 /// concurrent mutator. Instead, register all PIDs during the build phase
540 /// via `EngineModule::register_pids`.
541 ///
542 /// If a new process family needs to be added without restarting the
543 /// binary, rebuild and restart `makod` — hot-swap of PID routing is not
544 /// supported.
545 ///
546 /// # Example — dispatch at the AS4 reception boundary
547 ///
548 /// ```rust,ignore
549 /// let workflow_name = ctx.pid_router().route(pid)
550 /// .ok_or_else(|| EngineError::Workflow(WorkflowError::InvalidCommand(
551 /// format!("no workflow registered for PID {pid}").into()
552 /// )))?;
553 ///
554 /// match workflow_name {
555 /// "gpke-supplier-change" => dispatch::<GpkeSupplierChangeWorkflow>(&ctx, pid, payload).await,
556 /// "wim-device-change" => dispatch::<WimDeviceChangeWorkflow>(&ctx, pid, payload).await,
557 /// other => Err(EngineError::Workflow(WorkflowError::InvalidCommand(
558 /// format!("unhandled workflow name: {other}").into()
559 /// ))),
560 /// }
561 /// ```
562 #[must_use]
563 pub fn pid_router(&self) -> &PidRouter {
564 &self.pid_router
565 }
566}
567
568// ── As4Sender ─────────────────────────────────────────────────────────────────
569
570/// Sends a single AS4 / EDIINT-over-HTTP outbound message.
571///
572/// Implement this trait for your AS4 gateway client and pass it to
573/// [`EngineContext::run_outbox_worker`].
574///
575/// # Contract
576///
577/// Return `Ok(())` only after the message has been **durably accepted** by the
578/// receiving MSH. Return `Err(…)` on transient or permanent failure — the
579/// outbox worker calls [`OutboxStore::reschedule`] so the message is retried.
580pub trait As4Sender: Send + Sync + 'static {
581 /// Transmit `msg` and return when the remote MSH has accepted it.
582 fn send(
583 &self,
584 msg: &OutboxMessage,
585 ) -> impl std::future::Future<Output = Result<(), EngineError>> + Send;
586}
587
588// ── OutboxWorker ──────────────────────────────────────────────────────────────
589
590/// A background worker that drains the outbox by polling pending
591/// [`OutboxMessage`]s and dispatching them via an [`As4Sender`].
592///
593/// Obtain via [`EngineContext::run_outbox_worker`] and drive by spawning
594/// [`OutboxWorker::run`] in a Tokio task.
595///
596/// # Polling behaviour
597///
598/// When the poll returns an empty batch the worker sleeps for `poll_interval`
599/// before polling again. Non-empty batches are processed immediately.
600///
601/// # Error handling
602///
603/// Successful sends are acknowledged via [`OutboxStore::acknowledge`].
604/// Failed sends are rescheduled via [`OutboxStore::reschedule`] using
605/// **full-jitter exponential backoff**: `delay = rand(0, min(MAX, BASE * 2^n))`
606/// where `n = attempt_count`. This avoids thundering-herd when multiple
607/// `makod` instances restart simultaneously after a receiver outage.
608///
609/// When `attempt_count >= max_attempts`, the message is **acknowledged** (removed
610/// from the outbox) and a [`DeadLetterReason::OutboxExhausted`] record is written
611/// to the dead-letter sink. This prevents permanently-undeliverable messages
612/// from clogging the outbox forever.
613///
614/// All errors are emitted as structured `tracing` events at `warn` / `error`
615/// level rather than `eprintln!`, so they appear in the application's log
616/// pipeline with full context (message_id, error).
617///
618/// # Example
619///
620/// ```rust,ignore
621/// use std::time::Duration;
622///
623/// let worker = ctx.run_outbox_worker(my_sender, 50, Duration::from_secs(1));
624/// tokio::spawn(async move { worker.run().await });
625/// ```
626///
627/// [`DeadLetterReason::OutboxExhausted`]: crate::dead_letter::DeadLetterReason::OutboxExhausted
628pub struct OutboxWorker<OS: OutboxStore, S: As4Sender, DS: DeadlineStore> {
629 store: OS,
630 sender: S,
631 /// Used to discharge a delivery-window deadline once the message it was
632 /// watching has actually been sent — see [`OutboxWorker::run`].
633 deadline_store: DS,
634 batch_size: usize,
635 poll_interval: std::time::Duration,
636 /// Maximum total delivery attempts before a message is dead-lettered — a
637 /// runaway belt, not the budget. The budget is [`Self::max_retry_window`]:
638 /// the backoff is full-jitter, so an attempt *count* cannot promise a
639 /// retry *duration*, and the BDEW retry duty is stated in hours.
640 max_attempts: u32,
641 /// Maximum age (from `created_at`) a message is retried for before it is
642 /// dead-lettered. This is what honours a time-stated retry duty (BDEW AS4
643 /// Kommunikationshandbuch: 72 h for unacknowledged messages) — see
644 /// `mako_as4::constants::MAX_RETRY_DURATION_SECS`.
645 ///
646 /// Checked only after at least one attempt: a message that aged in a
647 /// stopped worker still gets its first try rather than being buried
648 /// unsent.
649 max_retry_window: std::time::Duration,
650 /// Sink for messages that exceed `max_attempts` or `max_retry_window`.
651 dead_letter_sink: std::sync::Arc<dyn crate::dead_letter::DeadLetterSink>,
652 /// Optional liveness heartbeat — stores the current UTC Unix timestamp
653 /// (seconds) after each poll cycle so health probes can detect stale workers.
654 heartbeat: Option<std::sync::Arc<std::sync::atomic::AtomicI64>>,
655 /// Graceful-shutdown signal. When cancelled the worker finishes the message
656 /// it is delivering, then returns from [`OutboxWorker::run`] — see
657 /// [`OutboxWorker::with_shutdown`].
658 shutdown: Option<tokio_util::sync::CancellationToken>,
659}
660
661/// Sleep for `dur`, returning early if `token` is cancelled.
662///
663/// Returns `true` when the sleep completed and the caller should keep looping,
664/// `false` when the token was cancelled and the caller must return.
665///
666/// A worker that sleeps on a bare `tokio::time::sleep` cannot observe a
667/// shutdown until its poll interval elapses. For the deadline scheduler that is
668/// 30 seconds by default — longer than a typical container termination grace
669/// period, which turns a graceful drain into a SIGKILL.
670///
671/// Public so that binaries running their own poll-loop workers alongside the
672/// engine's (projection catch-up, webhook delivery, retention purges) can honour
673/// the same token and stop before the store is closed.
674pub async fn sleep_or_cancel(
675 dur: std::time::Duration,
676 token: Option<&tokio_util::sync::CancellationToken>,
677) -> bool {
678 let Some(t) = token else {
679 tokio::time::sleep(dur).await;
680 return true;
681 };
682 tokio::select! {
683 () = tokio::time::sleep(dur) => true,
684 () = t.cancelled() => false,
685 }
686}
687
688/// Compute a full-jitter exponential backoff delay.
689///
690/// `attempt` is the number of prior attempts (0 = first retry).
691/// `entropy` provides randomness; derive from a stable message identifier
692/// (e.g. hash of `message_id`) rather than the current timestamp — a
693/// timestamp-derived value is deterministic within a single batch, which
694/// defeats jitter when multiple messages fail simultaneously.
695///
696/// | attempt | window (s) | expected delay (s) |
697/// |---------|------------|-------------------|
698/// | 0 | 5 | 2.5 |
699/// | 1 | 10 | 5 |
700/// | 2 | 20 | 10 |
701/// | 3 | 40 | 20 |
702/// | 4 | 80 | 40 |
703/// | 5+ | 300 (cap) | 150 |
704fn backoff_delay(attempt: u32, entropy: u64) -> std::time::Duration {
705 const BASE_SECS: u64 = 5;
706 const MAX_SECS: u64 = 300;
707 // Exponential window: BASE * 2^attempt, capped at MAX.
708 let window = BASE_SECS
709 .saturating_mul(1u64.wrapping_shl(attempt.min(5)))
710 .min(MAX_SECS);
711 // Full jitter: uniform random in [0, window).
712 let jitter_secs = if window == 0 { 0 } else { entropy % window };
713 std::time::Duration::from_secs(jitter_secs)
714}
715
716impl<OS: OutboxStore, S: As4Sender, DS: DeadlineStore> OutboxWorker<OS, S, DS> {
717 /// Run the outbox drain loop until the shutdown token is cancelled.
718 ///
719 /// Without a token (see [`OutboxWorker::with_shutdown`]) the loop runs until
720 /// the task is aborted or the process exits. With one, cancellation is
721 /// observed between messages and during the idle sleep, so an in-flight
722 /// delivery is always finished and acknowledged before the worker returns —
723 /// dropping it mid-`send` would risk a duplicate AS4 delivery on restart.
724 ///
725 /// # Panics
726 ///
727 /// Panics if `time::Duration::try_from(delay)` overflows (unreachable for
728 /// the delay values produced by `backoff_delay`).
729 #[allow(clippy::too_many_lines)]
730 pub async fn run(self) {
731 loop {
732 if self
733 .shutdown
734 .as_ref()
735 .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
736 {
737 tracing::info!("outbox worker: shutdown signalled; stopping");
738 return;
739 }
740 // Tick liveness at the *start* of every poll cycle, ahead of the
741 // early-`continue` paths below. An idle worker (empty outbox) and
742 // one retrying after a store error are both alive and must keep
743 // ticking; only a worker genuinely hung inside an `.await` stops.
744 if let Some(ref hb) = self.heartbeat {
745 hb.store(
746 time::OffsetDateTime::now_utc().unix_timestamp(),
747 std::sync::atomic::Ordering::Relaxed,
748 );
749 }
750
751 let batch = match self.store.pending_now(self.batch_size).await {
752 Ok(b) => b,
753 Err(e) => {
754 tracing::warn!(error = %e, "outbox worker: store error polling pending messages (will retry)");
755 if !sleep_or_cancel(self.poll_interval, self.shutdown.as_ref()).await {
756 return;
757 }
758 continue;
759 }
760 };
761
762 if batch.is_empty() {
763 if !sleep_or_cancel(self.poll_interval, self.shutdown.as_ref()).await {
764 return;
765 }
766 continue;
767 }
768
769 for msg in batch {
770 // Between messages, not inside one: a `send` that is already in
771 // flight must run to its `acknowledge`, or the counterparty
772 // receives a message the outbox still believes is pending and
773 // redelivers it after the restart.
774 if self
775 .shutdown
776 .as_ref()
777 .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
778 {
779 tracing::info!(
780 "outbox worker: shutdown signalled mid-batch; \
781 remaining messages stay queued for the next start"
782 );
783 return;
784 }
785 // ── Retry budget ──────────────────────────────────────
786 // `attempt_count` starts at 0 and is incremented on each
787 // `reschedule` call. The message is permanently undeliverable
788 // when the retry *window* has elapsed (the BDEW duty is stated
789 // in hours, and full-jitter backoff makes a count no proxy for
790 // a duration) or the attempt belt is exhausted: acknowledge it
791 // (remove from outbox) and dead-letter it so the regulatory
792 // audit trail is preserved. The window is only consulted after
793 // a first attempt, so a message that aged while the worker was
794 // down still gets tried once.
795 let age = time::OffsetDateTime::now_utc() - msg.created_at;
796 let window_elapsed = msg.attempt_count > 0
797 && age
798 >= time::Duration::try_from(self.max_retry_window)
799 .unwrap_or(time::Duration::hours(72));
800 if msg.attempt_count >= self.max_attempts || window_elapsed {
801 tracing::error!(
802 message_id = %msg.message_id,
803 message_type = %msg.message_type,
804 recipient = %msg.recipient,
805 attempts = msg.attempt_count,
806 max_attempts = self.max_attempts,
807 age_secs = age.whole_seconds(),
808 window_secs = self.max_retry_window.as_secs(),
809 "outbox worker: retry budget exhausted; dead-lettering message",
810 );
811 self.dead_letter_sink.reject(
812 &crate::dead_letter::DeadLetterReason::OutboxExhausted {
813 message_id: msg.message_id,
814 message_type: msg.message_type.to_string(),
815 recipient: msg.recipient.to_string(),
816 last_error: format!(
817 "delivery exhausted after {} attempts",
818 msg.attempt_count
819 ),
820 attempts: msg.attempt_count,
821 },
822 );
823 if let Err(e) = self.store.acknowledge(msg.message_id).await {
824 tracing::error!(
825 message_id = %msg.message_id,
826 error = %e,
827 "outbox worker: acknowledge after exhaust failed; message may reappear",
828 );
829 }
830 continue;
831 }
832
833 match self.sender.send(&msg).await {
834 Ok(()) => {
835 if let Err(e) = self.store.acknowledge(msg.message_id).await {
836 tracing::warn!(
837 message_id = %msg.message_id,
838 error = %e,
839 "outbox worker: acknowledge failed",
840 );
841 }
842 // CONTRL AHB 1.0 §1.2: the CONTRL must be delivered
843 // within 6 wall-clock hours of interchange receipt.
844 // `msg.created_at` is when the PendingOutbox was
845 // materialised (which should equal the ingest timestamp
846 // for transport-layer CONTRL obligations).
847 if msg.message_type.as_ref() == "CONTRL" {
848 let elapsed = time::OffsetDateTime::now_utc() - msg.created_at;
849 if elapsed > time::Duration::hours(mako_fristen::CONTRL_FRIST_HOURS) {
850 tracing::warn!(
851 message_id = %msg.message_id,
852 elapsed_secs = elapsed.whole_seconds(),
853 max_secs = mako_fristen::CONTRL_FRIST_HOURS * 3600,
854 "outbox worker: CONTRL delivered OUTSIDE the 6h Übertragungsfrist \
855 (CONTRL AHB 1.0 §1.2) — this is a BNetzA compliance violation"
856 );
857 }
858 }
859 // APERAK AHB 1.0 §2.4.1: Strom UTILMD/ORDERS APERAK must be
860 // delivered within 45 minutes on weekdays, or by Sunday 12:00
861 // if received on Saturday. Log a compliance warning if the
862 // delivery window was missed so operators can investigate.
863 if msg.message_type.as_ref() == "APERAK" {
864 let elapsed = time::OffsetDateTime::now_utc() - msg.created_at;
865 if elapsed
866 > time::Duration::minutes(
867 mako_fristen::APERAK_STROM_WEEKDAY_MINUTES,
868 )
869 {
870 tracing::warn!(
871 message_id = %msg.message_id,
872 elapsed_mins = elapsed.whole_minutes(),
873 "outbox worker: APERAK delivered after the 45-minute Strom \
874 sending window (APERAK AHB 1.0 §2.4.1) — \
875 check OutboxWorker and AS4 transport health"
876 );
877 }
878 }
879 // The message is out, so any delivery window that was
880 // watching for it has been answered — retire it.
881 //
882 // Nothing else cancels these. A monitoring deadline that
883 // outlives the obligation it monitors fires for every
884 // process, including every one that answered on time,
885 // and the scheduler cannot tell those apart because a
886 // deadline reaching `due_now` is late by construction.
887 // Leaving them registered turns the miss counters into
888 // counts of *processes started*.
889 self.discharge_delivery_window(&msg).await;
890 }
891 // Permanent error: dead-letter immediately without retrying.
892 // PartnerUnknown requires operator intervention (add --as4-partner);
893 // Serialization errors will never succeed on retry; a missing
894 // wire-format renderer cannot appear between attempts — its own
895 // documentation promises immediate dead-lettering, and until this
896 // arm matched it, that promise was broken and the message burned
897 // the whole retry budget first.
898 Err(ref e)
899 if e.is_partner_unknown()
900 || e.is_renderer_not_implemented()
901 || matches!(e, EngineError::Serialization(_)) =>
902 {
903 tracing::error!(
904 message_id = %msg.message_id,
905 message_type = %msg.message_type,
906 recipient = %msg.recipient,
907 error = %e,
908 "outbox worker: permanent send failure; dead-lettering without retry",
909 );
910 self.dead_letter_sink.reject(
911 &crate::dead_letter::DeadLetterReason::OutboxExhausted {
912 message_id: msg.message_id,
913 message_type: msg.message_type.to_string(),
914 recipient: msg.recipient.to_string(),
915 last_error: e.to_string(),
916 attempts: msg.attempt_count,
917 },
918 );
919 if let Err(re) = self.store.acknowledge(msg.message_id).await {
920 tracing::error!(
921 message_id = %msg.message_id,
922 error = %re,
923 "outbox worker: acknowledge after permanent failure failed",
924 );
925 }
926 }
927 Err(e) => {
928 // Stable jitter entropy derived from the UUID bytes of
929 // `message_id`. Using the last 8 bytes as a `u64` gives
930 // uniform entropy across message IDs (UUIDs are random in
931 // all 128 bits for v4) and is stable across Rust versions —
932 // unlike `DefaultHasher`, whose algorithm is explicitly
933 // documented as unstable.
934 let entropy = {
935 let uuid = msg.message_id.as_uuid();
936 let bytes = uuid.as_bytes();
937 u64::from_le_bytes(bytes[8..16].try_into().unwrap())
938 };
939 let delay = backoff_delay(msg.attempt_count, entropy);
940 let retry_at = time::OffsetDateTime::now_utc()
941 + time::Duration::try_from(delay).unwrap_or(time::Duration::minutes(5));
942 tracing::warn!(
943 message_id = %msg.message_id,
944 attempt = msg.attempt_count,
945 max_attempts = self.max_attempts,
946 retry_in = ?delay,
947 error = %e,
948 "outbox worker: send failed; rescheduling with backoff",
949 );
950 if let Err(re) = self.store.reschedule(msg.message_id, retry_at).await {
951 tracing::error!(
952 message_id = %msg.message_id,
953 error = %re,
954 "outbox worker: reschedule failed; message may be stuck",
955 );
956 }
957 }
958 }
959 }
960 }
961 }
962}
963
964impl<ES, SS, OS, DS, PR> EngineContext<ES, SS, OS, DS, PR>
965where
966 ES: EventStore,
967 OS: OutboxStore + Clone,
968{
969 /// Construct an [`OutboxWorker`] that drains the outbox via `sender`.
970 ///
971 /// `batch_size` — messages fetched per poll cycle.
972 /// `poll_interval` — sleep duration when the batch is empty.
973 ///
974 /// `max_attempts` — attempt belt against runaway loops; the real budget is
975 /// `max_retry_window`, the message age after which delivery is abandoned.
976 /// The BDEW AS4 retry duty is stated in *hours* (72 h for unacknowledged
977 /// messages — `mako_as4::constants::MAX_RETRY_DURATION_SECS`), and the
978 /// full-jitter backoff makes an attempt count no proxy for a duration, so
979 /// both are taken and either exhausts the message.
980 ///
981 /// ```rust,ignore
982 /// use std::time::Duration;
983 ///
984 /// let worker = ctx.run_outbox_worker(
985 /// my_sender, 50, Duration::from_secs(1),
986 /// 10_000, Duration::from_secs(72 * 3600),
987 /// );
988 /// tokio::spawn(async move { worker.run().await });
989 /// ```
990 #[must_use]
991 pub fn run_outbox_worker<S: As4Sender>(
992 &self,
993 sender: S,
994 batch_size: usize,
995 poll_interval: std::time::Duration,
996 max_attempts: u32,
997 max_retry_window: std::time::Duration,
998 ) -> OutboxWorker<OS, S, DS>
999 where
1000 DS: DeadlineStore + Clone,
1001 {
1002 OutboxWorker {
1003 store: self.outbox_store.clone(),
1004 sender,
1005 deadline_store: self.deadline_store.clone(),
1006 batch_size,
1007 poll_interval,
1008 max_attempts,
1009 max_retry_window,
1010 dead_letter_sink: self.dead_letter_sink.clone(),
1011 heartbeat: None,
1012 shutdown: None,
1013 }
1014 }
1015}
1016
1017impl<OS: OutboxStore, S: As4Sender, DS: DeadlineStore> OutboxWorker<OS, S, DS> {
1018 /// Attach a liveness heartbeat to this worker.
1019 ///
1020 /// The worker will store the current UTC Unix timestamp (seconds) into
1021 /// `heartbeat` at the end of every poll cycle. Pass the same
1022 /// `Arc<AtomicI64>` to the health endpoint so it can detect stale workers.
1023 #[must_use]
1024 pub fn with_heartbeat(
1025 mut self,
1026 heartbeat: std::sync::Arc<std::sync::atomic::AtomicI64>,
1027 ) -> Self {
1028 self.heartbeat = Some(heartbeat);
1029 self
1030 }
1031
1032 /// Attach a graceful-shutdown token.
1033 ///
1034 /// Cancelling it makes [`OutboxWorker::run`] return at the next message
1035 /// boundary or immediately out of its idle sleep. Await the worker's
1036 /// `JoinHandle` afterwards: the point of the token is that the caller can
1037 /// close the event store *after* the worker has stopped writing to it.
1038 #[must_use]
1039 pub fn with_shutdown(mut self, shutdown: tokio_util::sync::CancellationToken) -> Self {
1040 self.shutdown = Some(shutdown);
1041 self
1042 }
1043
1044 /// Retire the delivery window `msg` was being watched by, if one is open.
1045 ///
1046 /// A delivery-window deadline exists to answer one question: *did this
1047 /// message go out in time?* Once it has gone out the question is settled,
1048 /// and leaving the deadline registered only guarantees a false alarm later.
1049 /// [`fristen::discharges_delivery_window`] decides which labels a given
1050 /// message type answers for; deadlines that merely share the stream (a
1051 /// process-response window, say) are left alone.
1052 ///
1053 /// Best-effort: a failure here costs a spurious alert at the window's close,
1054 /// never a lost or duplicated message, so it is logged rather than
1055 /// propagated — the delivery itself has already been acknowledged.
1056 ///
1057 /// [`fristen::discharges_delivery_window`]: mako_fristen::discharges_delivery_window
1058 async fn discharge_delivery_window(&self, msg: &crate::outbox::OutboxMessage) {
1059 let open = match self.deadline_store.for_stream(&msg.stream_id).await {
1060 Ok(deadlines) => deadlines,
1061 Err(e) => {
1062 tracing::warn!(
1063 message_id = %msg.message_id,
1064 message_type = %msg.message_type,
1065 error = %e,
1066 "outbox worker: could not read deadlines to discharge the delivery \
1067 window; it may fire a spurious regulatory alert",
1068 );
1069 return;
1070 }
1071 };
1072
1073 for deadline in open
1074 .iter()
1075 .filter(|d| mako_fristen::discharges_delivery_window(&msg.message_type, d.label()))
1076 {
1077 if let Err(e) = self.deadline_store.cancel(deadline.deadline_id()).await {
1078 tracing::warn!(
1079 message_id = %msg.message_id,
1080 deadline_id = %deadline.deadline_id(),
1081 label = %deadline.label(),
1082 error = %e,
1083 "outbox worker: could not discharge the delivery window; \
1084 it may fire a spurious regulatory alert",
1085 );
1086 } else {
1087 tracing::debug!(
1088 message_id = %msg.message_id,
1089 message_type = %msg.message_type,
1090 deadline_id = %deadline.deadline_id(),
1091 label = %deadline.label(),
1092 "outbox worker: message delivered — delivery window discharged",
1093 );
1094 }
1095 }
1096 }
1097}
1098
1099impl<ES, SS, OS, DS, PR> std::fmt::Debug for EngineContext<ES, SS, OS, DS, PR>
1100where
1101 ES: std::fmt::Debug,
1102 SS: std::fmt::Debug,
1103 OS: std::fmt::Debug,
1104 DS: std::fmt::Debug,
1105 PR: std::fmt::Debug,
1106{
1107 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1108 f.debug_struct("EngineContext")
1109 .field("registered_modules", &self.registered_modules)
1110 .field("registered_workflows", &self.registered_workflows)
1111 .field("pid_router_len", &self.pid_router.len())
1112 .finish_non_exhaustive()
1113 }
1114}
1115
1116// ── NoopAs4Sender / LogAs4Sender ──────────────────────────────────────────────
1117
1118/// An [`As4Sender`] that succeeds immediately without sending anything.
1119///
1120/// Use in tests and environments where outbound AS4 delivery is not yet
1121/// wired. All outbox messages are acknowledged (removed from the queue)
1122/// without being transmitted.
1123///
1124/// # ⚠️ Data loss warning
1125///
1126/// Every outbox message is **silently discarded** — no EDIFACT message is
1127/// sent to any counterparty. Do not use in production.
1128#[derive(Debug, Clone, Copy, Default)]
1129#[must_use = "NoopAs4Sender discards all outbound messages silently — use a real AS4 gateway in production"]
1130#[cfg_attr(
1131 not(any(test, feature = "testing")),
1132 deprecated = "NoopAs4Sender must not be wired in production builds; every \
1133 outbound EDIFACT message would be silently discarded. Use \
1134 a real As4Sender implementation instead."
1135)]
1136pub struct NoopAs4Sender;
1137
1138// The trait impl is test/testing-only: a release build without the `testing`
1139// feature cannot wire NoopAs4Sender into an outbox worker at all.
1140#[cfg(any(test, feature = "testing"))]
1141impl As4Sender for NoopAs4Sender {
1142 async fn send(&self, _msg: &OutboxMessage) -> Result<(), EngineError> {
1143 Ok(())
1144 }
1145}
1146
1147/// An [`As4Sender`] that logs every outbound message at `warn` level and
1148/// succeeds without transmitting.
1149///
1150/// Useful for development and integration-testing environments where the
1151/// full AS4 stack is not yet available but message visibility is desired.
1152/// All outbox messages are acknowledged (removed from the queue) after logging.
1153///
1154/// # ⚠️ Data loss warning
1155///
1156/// No EDIFACT message is sent to any counterparty. Do not use in production.
1157#[derive(Debug, Clone, Copy, Default)]
1158#[must_use = "LogAs4Sender discards all outbound messages — use a real AS4 gateway in production"]
1159pub struct LogAs4Sender;
1160
1161impl As4Sender for LogAs4Sender {
1162 async fn send(&self, msg: &OutboxMessage) -> Result<(), EngineError> {
1163 tracing::warn!(
1164 message_id = %msg.message_id,
1165 message_type = %msg.message_type,
1166 recipient = %msg.recipient,
1167 "LogAs4Sender: outbox message dropped — configure a real AS4 gateway for production",
1168 );
1169 Ok(())
1170 }
1171}
1172
1173// ── DeadlineScheduler ─────────────────────────────────────────────────────────
1174
1175/// A background task that polls [`DeadlineStore::due_now`] and dispatches
1176/// deadline commands to the owning processes via a caller-supplied function.
1177///
1178/// Obtain via [`EngineContext::run_deadline_scheduler`] and drive by spawning
1179/// [`DeadlineScheduler::run`] in a Tokio task.
1180///
1181/// # Dispatch function
1182///
1183/// The `dispatch` function receives a fired [`Deadline`] and returns a future
1184/// that dispatches the appropriate timeout command to the process. The function
1185/// is responsible for resuming the correct workflow and calling `execute`.
1186/// After the future completes, the scheduler cancels the deadline from the
1187/// store regardless of the dispatch outcome (to prevent re-firing).
1188///
1189/// ```rust,ignore
1190/// use std::time::Duration;
1191///
1192/// let scheduler = ctx.run_deadline_scheduler(
1193/// |deadline| async move {
1194/// tracing::warn!(
1195/// deadline_id = %deadline.deadline_id(),
1196/// label = %deadline.label(),
1197/// "deadline fired",
1198/// );
1199/// Ok(())
1200/// },
1201/// 100,
1202/// Duration::from_secs(30),
1203/// );
1204/// tokio::spawn(async move { scheduler.run().await });
1205/// ```
1206pub struct DeadlineScheduler<DS: DeadlineStore> {
1207 store: DS,
1208 dispatch: Box<
1209 dyn Fn(
1210 Deadline,
1211 ) -> std::pin::Pin<
1212 Box<dyn std::future::Future<Output = Result<(), EngineError>> + Send>,
1213 > + Send
1214 + Sync,
1215 >,
1216 batch_size: usize,
1217 poll_interval: std::time::Duration,
1218 /// Optional liveness heartbeat — stores the current UTC Unix timestamp
1219 /// (seconds) after each poll cycle.
1220 heartbeat: Option<std::sync::Arc<std::sync::atomic::AtomicI64>>,
1221 /// Graceful-shutdown signal — see [`DeadlineScheduler::with_shutdown`].
1222 shutdown: Option<tokio_util::sync::CancellationToken>,
1223}
1224
1225impl<DS: DeadlineStore> DeadlineScheduler<DS> {
1226 /// Run the deadline poll loop until the shutdown token is cancelled.
1227 ///
1228 /// Cancellation is observed between deadlines and during the idle sleep, so
1229 /// a deadline already being dispatched runs to completion. A deadline left
1230 /// undispatched stays registered and fires on the next start — it is due, so
1231 /// the next `due_now` returns it again.
1232 pub async fn run(self) {
1233 loop {
1234 if self
1235 .shutdown
1236 .as_ref()
1237 .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
1238 {
1239 tracing::info!("deadline scheduler: shutdown signalled; stopping");
1240 return;
1241 }
1242 // Tick liveness at the *start* of every poll cycle, ahead of the
1243 // early-`continue` paths below. An idle scheduler (no due
1244 // deadlines) is alive and must keep ticking; only one genuinely
1245 // hung inside an `.await` stops.
1246 if let Some(ref hb) = self.heartbeat {
1247 hb.store(
1248 time::OffsetDateTime::now_utc().unix_timestamp(),
1249 std::sync::atomic::Ordering::Relaxed,
1250 );
1251 }
1252
1253 let result = match self.store.due_now(self.batch_size).await {
1254 Ok(r) => r,
1255 Err(e) => {
1256 tracing::warn!(
1257 error = %e,
1258 "deadline scheduler: store error polling due deadlines (will retry)",
1259 );
1260 if !sleep_or_cancel(self.poll_interval, self.shutdown.as_ref()).await {
1261 return;
1262 }
1263 continue;
1264 }
1265 };
1266
1267 if result.deadlines.is_empty() {
1268 if !sleep_or_cancel(self.poll_interval, self.shutdown.as_ref()).await {
1269 return;
1270 }
1271 continue;
1272 }
1273
1274 for deadline in result.deadlines {
1275 // Between deadlines, not inside one: a dispatch already running
1276 // must finish so its events and outbox entries commit together.
1277 if self
1278 .shutdown
1279 .as_ref()
1280 .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
1281 {
1282 tracing::info!(
1283 "deadline scheduler: shutdown signalled mid-batch; \
1284 undispatched deadlines remain due and fire on the next start"
1285 );
1286 return;
1287 }
1288 let id = deadline.deadline_id();
1289 let label = deadline.label().to_owned();
1290
1291 // An APERAK delivery window that reaches this point is a
1292 // regulatory violation under APERAK AHB 1.0 §2.4.1 (Strom
1293 // 45 min) / §2.3.1 (Gas 1 Werktag): the outbox worker discharges
1294 // the window the moment the APERAK goes out, so one that is
1295 // still registered when it comes due was never answered.
1296 //
1297 // Do NOT re-test `now > due_at` here. `due_now` selects on
1298 // `due_at <= now`, so that comparison is true by construction and
1299 // says nothing about compliance — it was the reason this counter
1300 // once tracked "Strom processes started" rather than "APERAKs
1301 // missed". The discharge is what carries the meaning.
1302 if label.starts_with(mako_fristen::APERAK_WINDOW_LABEL_PREFIX) {
1303 let now = time::OffsetDateTime::now_utc();
1304 crate::metrics::EngineMetrics::global().aperak_missed(&label);
1305 tracing::error!(
1306 deadline_id = %id,
1307 label = %label,
1308 due_at = %deadline.due_at(),
1309 fired_at = %now,
1310 overdue_secs = (now - deadline.due_at()).whole_seconds(),
1311 "APERAK delivery window closed with no delivery — regulatory \
1312 violation (APERAK AHB 1.0 §2.4.1 Strom / §2.3.1 Gas). \
1313 Counter: makod_aperak_missed_total",
1314 );
1315 }
1316
1317 let should_cancel = match (self.dispatch)(deadline).await {
1318 Ok(()) => true,
1319 Err(ref e) if e.is_version_conflict() => {
1320 // The process was modified concurrently; the timeout
1321 // command will be retried on the next poll cycle.
1322 // Do NOT cancel — let the deadline remain due so it
1323 // fires again until a non-conflict dispatch succeeds.
1324 tracing::warn!(
1325 deadline_id = %id,
1326 label = %label,
1327 "deadline scheduler: VersionConflict; will retry on next poll",
1328 );
1329 false
1330 }
1331 Err(e) => {
1332 tracing::warn!(
1333 deadline_id = %id,
1334 label = %label,
1335 error = %e,
1336 "deadline scheduler: dispatch failed (permanent); cancelling",
1337 );
1338 true
1339 }
1340 };
1341 if should_cancel && let Err(e) = self.store.cancel(id).await {
1342 tracing::error!(
1343 deadline_id = %id,
1344 error = %e,
1345 "deadline scheduler: cancel failed; deadline may fire again",
1346 );
1347 }
1348 }
1349
1350 // If has_more, loop immediately to drain the batch.
1351 }
1352 }
1353}
1354
1355impl<DS: DeadlineStore> DeadlineScheduler<DS> {
1356 /// Attach a liveness heartbeat to this scheduler.
1357 ///
1358 /// The scheduler will store the current UTC Unix timestamp (seconds) into
1359 /// `heartbeat` at the end of every poll cycle.
1360 #[must_use]
1361 pub fn with_heartbeat(
1362 mut self,
1363 heartbeat: std::sync::Arc<std::sync::atomic::AtomicI64>,
1364 ) -> Self {
1365 self.heartbeat = Some(heartbeat);
1366 self
1367 }
1368
1369 /// Attach a graceful-shutdown token.
1370 ///
1371 /// Cancelling it makes [`DeadlineScheduler::run`] return at the next
1372 /// deadline boundary or immediately out of its idle sleep, so the caller can
1373 /// close the event store once the scheduler has stopped writing to it.
1374 #[must_use]
1375 pub fn with_shutdown(mut self, shutdown: tokio_util::sync::CancellationToken) -> Self {
1376 self.shutdown = Some(shutdown);
1377 self
1378 }
1379}
1380
1381impl<ES, SS, OS, DS, PR> EngineContext<ES, SS, OS, DS, PR>
1382where
1383 ES: EventStore,
1384 DS: DeadlineStore + Clone,
1385{
1386 /// Construct a [`DeadlineScheduler`] that polls the deadline store and
1387 /// dispatches fired deadlines via `dispatch`.
1388 ///
1389 /// The `dispatch` function is called for every fired deadline. It should
1390 /// resume the owning process and execute the appropriate timeout command.
1391 ///
1392 /// `batch_size` — deadlines fetched per poll cycle.
1393 /// `poll_interval` — sleep duration when no deadlines are due.
1394 ///
1395 /// ```rust,ignore
1396 /// use std::time::Duration;
1397 ///
1398 /// let scheduler = ctx.run_deadline_scheduler(
1399 /// |d| async move {
1400 /// tracing::info!(label = %d.label(), "firing deadline");
1401 /// Ok(())
1402 /// },
1403 /// 100,
1404 /// Duration::from_secs(30),
1405 /// );
1406 /// tokio::spawn(async move { scheduler.run().await });
1407 /// ```
1408 #[must_use]
1409 pub fn run_deadline_scheduler<F, Fut>(
1410 &self,
1411 dispatch: F,
1412 batch_size: usize,
1413 poll_interval: std::time::Duration,
1414 ) -> DeadlineScheduler<DS>
1415 where
1416 F: Fn(Deadline) -> Fut + Send + Sync + 'static,
1417 Fut: std::future::Future<Output = Result<(), EngineError>> + Send + 'static,
1418 {
1419 DeadlineScheduler {
1420 store: self.deadline_store.clone(),
1421 dispatch: Box::new(move |d| Box::pin(dispatch(d))),
1422 batch_size,
1423 poll_interval,
1424 heartbeat: None,
1425 shutdown: None,
1426 }
1427 }
1428}
1429
1430// ── EngineBuilder ─────────────────────────────────────────────────────────────
1431
1432/// Assembles engine infrastructure and produces an [`EngineContext`].
1433///
1434/// Uses type-state to enforce that an event store is provided before
1435/// [`build`] can be called. All other stores default to `Noop`
1436/// implementations.
1437///
1438/// ## Quick start
1439///
1440/// ```rust,ignore
1441/// // Minimal — event store only, all others are Noop:
1442/// let ctx = EngineBuilder::new()
1443/// .with_event_store(InMemoryEventStore::new())
1444/// .build();
1445///
1446/// // Full infrastructure:
1447/// let ctx = EngineBuilder::new()
1448/// .with_event_store(InMemoryEventStore::new())
1449/// .with_snapshot_store(InMemorySnapshotStore::new())
1450/// .with_outbox_store(InMemoryOutboxStore::new())
1451/// .with_deadline_store(InMemoryDeadlineStore::new())
1452/// .with_registry(InMemoryProcessRegistry::new())
1453/// .register(Box::new(GpkeModule))
1454/// .build();
1455/// ```
1456///
1457/// [`build`]: EngineBuilder::build
1458pub struct EngineBuilder<
1459 ES = (),
1460 SS = NoopSnapshotStore,
1461 OS = NoopOutboxStore,
1462 DS = NoopDeadlineStore,
1463 PR = NoopProcessRegistry,
1464> {
1465 event_store: ES,
1466 snapshot_store: SS,
1467 outbox_store: OS,
1468 deadline_store: DS,
1469 registry: PR,
1470 dead_letter_sink: Arc<dyn DeadLetterSink>,
1471 modules: Vec<Box<dyn EngineModule>>,
1472 /// Active [`DeploymentRoles`] for this engine instance.
1473 ///
1474 /// Controls role-conditional PID registration via
1475 /// [`EngineModule::register_pids_with_roles`]. Defaults to
1476 /// [`DeploymentRoles::all()`] for backward compatibility.
1477 deployment_roles: DeploymentRoles,
1478 /// Optional profile validator injected by `makod` or callers that have
1479 /// access to `edi-energy`. When `Some`, called for each
1480 /// [`ProfileRequirement`] declared by registered modules. When `None`,
1481 /// profile requirements are not validated (safe in unit tests).
1482 ///
1483 /// Signature: `fn(message_type: &str) -> bool`
1484 ///
1485 /// [`ProfileRequirement`]: crate::profile::ProfileRequirement
1486 profile_validator: Option<Box<dyn Fn(&str) -> bool + Send + Sync>>,
1487}
1488#[cfg(any(test, feature = "testing"))]
1489impl Default
1490 for EngineBuilder<
1491 (),
1492 NoopSnapshotStore,
1493 NoopOutboxStore,
1494 NoopDeadlineStore,
1495 NoopProcessRegistry,
1496 >
1497{
1498 fn default() -> Self {
1499 Self {
1500 event_store: (),
1501 snapshot_store: NoopSnapshotStore,
1502 outbox_store: NoopOutboxStore,
1503 deadline_store: NoopDeadlineStore,
1504 registry: NoopProcessRegistry,
1505 dead_letter_sink: Arc::new(LogDeadLetterSink),
1506 modules: Vec::new(),
1507 deployment_roles: DeploymentRoles::all(),
1508 profile_validator: None,
1509 }
1510 }
1511}
1512
1513#[cfg(any(test, feature = "testing"))]
1514impl EngineBuilder {
1515 /// Create a new builder with all `Noop` defaults.
1516 ///
1517 /// Only available in `#[cfg(test)]` or with the `testing` feature enabled,
1518 /// because the Noop defaults silently discard outbox messages, deadlines,
1519 /// and process registry entries. Production binaries must wire real stores
1520 /// via the `with_*` builder methods.
1521 ///
1522 /// Call [`with_event_store`] before [`build`] — the event store is
1523 /// **required**.
1524 ///
1525 /// [`with_event_store`]: EngineBuilder::with_event_store
1526 /// [`build`]: EngineBuilder::build
1527 #[must_use]
1528 pub fn new() -> Self {
1529 Self::default()
1530 }
1531}
1532
1533impl<OS, DS, PR> EngineBuilder<(), NoopSnapshotStore, OS, DS, PR>
1534where
1535 OS: OutboxStore,
1536 DS: DeadlineStore,
1537 PR: ProcessRegistry,
1538{
1539 /// Create a production-ready builder with explicit stores for outbox,
1540 /// deadline, and process registry.
1541 ///
1542 /// This constructor is available in all build configurations including
1543 /// production binaries. It enforces that the three stores that can cause
1544 /// silent data loss (`OutboxStore`, `DeadlineStore`, `ProcessRegistry`)
1545 /// are provided explicitly — there is no Noop fallback.
1546 ///
1547 /// `NoopSnapshotStore` is used as the snapshot default because it is safe
1548 /// for production: skipping snapshots means full replay, but no data loss.
1549 /// Override with [`with_snapshot_store`] to enable snapshot-accelerated
1550 /// replay.
1551 ///
1552 /// Call [`with_event_store`] before [`build`] — the event store is
1553 /// **required**.
1554 ///
1555 /// ```rust,ignore
1556 /// let ctx = EngineBuilder::with_stores(outbox, deadline, registry)
1557 /// .with_event_store(store.clone())
1558 /// .with_snapshot_store(InMemorySnapshotStore::new())
1559 /// .build();
1560 /// ```
1561 ///
1562 /// [`with_snapshot_store`]: EngineBuilder::with_snapshot_store
1563 /// [`with_event_store`]: EngineBuilder::with_event_store
1564 /// [`build`]: EngineBuilder::build
1565 #[must_use]
1566 pub fn with_stores(outbox_store: OS, deadline_store: DS, registry: PR) -> Self {
1567 Self {
1568 event_store: (),
1569 snapshot_store: NoopSnapshotStore,
1570 outbox_store,
1571 deadline_store,
1572 registry,
1573 dead_letter_sink: Arc::new(LogDeadLetterSink),
1574 modules: Vec::new(),
1575 deployment_roles: DeploymentRoles::all(),
1576 profile_validator: None,
1577 }
1578 }
1579}
1580
1581impl<ES, SS, OS, DS, PR> EngineBuilder<ES, SS, OS, DS, PR> {
1582 /// Set the event store. **Required** — `build()` is only available once
1583 /// this has been called with a type that implements [`EventStore`].
1584 ///
1585 /// Replaces any previously set event store (type-state transition).
1586 #[must_use]
1587 pub fn with_event_store<ES2: EventStore>(
1588 self,
1589 store: ES2,
1590 ) -> EngineBuilder<ES2, SS, OS, DS, PR> {
1591 EngineBuilder {
1592 event_store: store,
1593 snapshot_store: self.snapshot_store,
1594 outbox_store: self.outbox_store,
1595 deadline_store: self.deadline_store,
1596 registry: self.registry,
1597 dead_letter_sink: self.dead_letter_sink,
1598 modules: self.modules,
1599 deployment_roles: self.deployment_roles,
1600 profile_validator: self.profile_validator,
1601 }
1602 }
1603
1604 /// Set the snapshot store (default: [`NoopSnapshotStore`]).
1605 ///
1606 /// ## Default: `NoopSnapshotStore`
1607 ///
1608 /// Without calling this method the builder uses [`NoopSnapshotStore`],
1609 /// which silently discards all snapshot writes and returns `None` for
1610 /// every snapshot read. The engine still functions correctly — every
1611 /// command handling call replays the full event log from the beginning
1612 /// instead of starting from a stored snapshot. For low-volume processes
1613 /// this is fine; for long-lived processes with many events the replay cost
1614 /// can become significant.
1615 ///
1616 /// Enable snapshotting in production by providing a real [`SnapshotStore`]
1617 /// implementation (e.g. the SlateDB-backed store in `makod`). In tests,
1618 /// `InMemorySnapshotStore` is available behind the `testing` feature flag.
1619 ///
1620 /// Note: [`Process::state_with_snapshot`][crate::process::Process::state_with_snapshot]
1621 /// is a compile-time no-op when the snapshot store is `NoopSnapshotStore`
1622 /// — it never calls the store and always returns `None`, so no snapshot is
1623 /// ever saved or loaded.
1624 #[must_use]
1625 pub fn with_snapshot_store<SS2: SnapshotStore>(
1626 self,
1627 store: SS2,
1628 ) -> EngineBuilder<ES, SS2, OS, DS, PR> {
1629 EngineBuilder {
1630 event_store: self.event_store,
1631 snapshot_store: store,
1632 outbox_store: self.outbox_store,
1633 deadline_store: self.deadline_store,
1634 registry: self.registry,
1635 dead_letter_sink: self.dead_letter_sink,
1636 modules: self.modules,
1637 deployment_roles: self.deployment_roles,
1638 profile_validator: self.profile_validator,
1639 }
1640 }
1641
1642 /// Set the outbox store (default: [`NoopOutboxStore`]).
1643 #[must_use]
1644 pub fn with_outbox_store<OS2: OutboxStore>(
1645 self,
1646 store: OS2,
1647 ) -> EngineBuilder<ES, SS, OS2, DS, PR> {
1648 EngineBuilder {
1649 event_store: self.event_store,
1650 snapshot_store: self.snapshot_store,
1651 outbox_store: store,
1652 deadline_store: self.deadline_store,
1653 registry: self.registry,
1654 dead_letter_sink: self.dead_letter_sink,
1655 modules: self.modules,
1656 deployment_roles: self.deployment_roles,
1657 profile_validator: self.profile_validator,
1658 }
1659 }
1660
1661 /// Set the deadline store (default: [`NoopDeadlineStore`]).
1662 #[must_use]
1663 pub fn with_deadline_store<DS2: DeadlineStore>(
1664 self,
1665 store: DS2,
1666 ) -> EngineBuilder<ES, SS, OS, DS2, PR> {
1667 EngineBuilder {
1668 event_store: self.event_store,
1669 snapshot_store: self.snapshot_store,
1670 outbox_store: self.outbox_store,
1671 deadline_store: store,
1672 registry: self.registry,
1673 dead_letter_sink: self.dead_letter_sink,
1674 modules: self.modules,
1675 deployment_roles: self.deployment_roles,
1676 profile_validator: self.profile_validator,
1677 }
1678 }
1679
1680 /// Set the process registry (default: [`NoopProcessRegistry`]).
1681 #[must_use]
1682 pub fn with_registry<PR2: ProcessRegistry>(
1683 self,
1684 registry: PR2,
1685 ) -> EngineBuilder<ES, SS, OS, DS, PR2> {
1686 EngineBuilder {
1687 event_store: self.event_store,
1688 snapshot_store: self.snapshot_store,
1689 outbox_store: self.outbox_store,
1690 deadline_store: self.deadline_store,
1691 registry,
1692 dead_letter_sink: self.dead_letter_sink,
1693 modules: self.modules,
1694 deployment_roles: self.deployment_roles,
1695 profile_validator: self.profile_validator,
1696 }
1697 }
1698
1699 /// Set the dead-letter sink (default: [`LogDeadLetterSink`]).
1700 ///
1701 /// The dead-letter sink receives every message that cannot be routed to a
1702 /// workflow. The default [`LogDeadLetterSink`] emits `tracing::warn!`
1703 /// events, making rejections visible in log output without configuration.
1704 ///
1705 /// Override with a persistent DLQ implementation in production:
1706 ///
1707 /// ```rust,ignore
1708 /// use mako_engine::dead_letter::LogDeadLetterSink;
1709 ///
1710 /// let ctx = EngineBuilder::new()
1711 /// .with_event_store(my_store)
1712 /// .with_dead_letter_sink(MyPersistentDlq::new())
1713 /// .build();
1714 /// ```
1715 ///
1716 /// [`LogDeadLetterSink`]: crate::dead_letter::LogDeadLetterSink
1717 #[must_use]
1718 pub fn with_dead_letter_sink(mut self, sink: impl DeadLetterSink) -> Self {
1719 self.dead_letter_sink = Arc::new(sink);
1720 self
1721 }
1722
1723 /// Register an `edi-energy` profile validator for startup profile checks.
1724 ///
1725 /// The closure receives a message-type string (e.g. `"UTILMD"`) and must
1726 /// return `true` if at least one active profile for that message type is
1727 /// registered for today's date.
1728 ///
1729 /// Wire this in `makod` using the `edi-energy` global registry:
1730 ///
1731 /// ```rust,ignore
1732 /// use edi_energy::registry::ReleaseRegistry;
1733 ///
1734 /// let today = time::OffsetDateTime::now_utc().date();
1735 /// builder.with_profile_validator(move |msg_type| {
1736 /// ReleaseRegistry::global()
1737 /// .profiles_for_str(msg_type)
1738 /// .any(|p| match (p.valid_from(), p.valid_until()) {
1739 /// (Some(f), Some(u)) => f <= today && today <= u,
1740 /// (Some(f), None) => f <= today,
1741 /// (None, _) => true,
1742 /// })
1743 /// })
1744 /// ```
1745 ///
1746 /// Domain crates do **not** need to call this — they only declare
1747 /// [`profile_requirements`].
1748 ///
1749 /// [`profile_requirements`]: EngineModule::profile_requirements
1750 #[must_use]
1751 pub fn with_profile_validator(
1752 mut self,
1753 validator: impl Fn(&str) -> bool + Send + Sync + 'static,
1754 ) -> Self {
1755 self.profile_validator = Some(Box::new(validator));
1756 self
1757 }
1758
1759 /// Register a domain module.
1760 ///
1761 /// The module name becomes visible in
1762 /// [`EngineContext::registered_modules`] after [`build`] is called.
1763 ///
1764 /// [`build`]: EngineBuilder::build
1765 #[must_use]
1766 pub fn register(mut self, module: Box<dyn EngineModule>) -> Self {
1767 self.modules.push(module);
1768 self
1769 }
1770
1771 /// Register multiple [`EngineModule`]s at once from a pre-built `Vec`.
1772 ///
1773 /// Equivalent to calling [`register`] in a loop. Useful when the set of
1774 /// modules is assembled conditionally (e.g. via `#[cfg]`-gated pushes to a
1775 /// `Vec<Box<dyn EngineModule>>`) before the builder chain starts.
1776 ///
1777 /// [`register`]: EngineBuilder::register
1778 #[must_use]
1779 pub fn register_many(mut self, modules: Vec<Box<dyn EngineModule>>) -> Self {
1780 self.modules.extend(modules);
1781 self
1782 }
1783
1784 /// Set the active [`DeploymentRoles`] for this engine instance.
1785 ///
1786 /// Controls role-conditional PID registration in [`EngineModule::register_pids_with_roles`].
1787 ///
1788 /// The default is [`DeploymentRoles::all()`], which registers every PID unconditionally
1789 /// — identical to the pre-role-aware behavior. Providing an explicit role set
1790 /// restricts role-conditional blocks to only the declared roles:
1791 ///
1792 /// - **NB-only** (`DeploymentRoles::nb()`): 19001/19002 route to `gpke-konfiguration`;
1793 /// WiM nMSB blocks are skipped.
1794 /// - **nMSB-only** (`DeploymentRoles::nmsb()`): 19001/19002 route to `wim-geraeteubernahme`;
1795 /// GPKE NB blocks are skipped.
1796 /// - **NB + gMSB** (`DeploymentRoles::nb_msb()`): most common Stadtwerke combination.
1797 ///
1798 /// # Conflict guard
1799 ///
1800 /// When two modules would register the same PID to **different** workflows, the
1801 /// engine panics during [`build`]. Set explicit roles to prevent both modules from
1802 /// activating the same PID simultaneously:
1803 ///
1804 /// ```rust,ignore
1805 /// use mako_engine::marktrolle::DeploymentRoles;
1806 ///
1807 /// let ctx = EngineBuilder::with_stores(outbox, deadline, registry)
1808 /// .with_event_store(store)
1809 /// .with_deployment_roles(DeploymentRoles::nb()) // only NB: GPKE gets 19001/19002
1810 /// .register(Box::new(GpkeModule))
1811 /// .register(Box::new(WimModule)) // nMSB block skipped — no conflict
1812 /// .build();
1813 /// ```
1814 ///
1815 /// [`build`]: EngineBuilder::build
1816 #[must_use]
1817 pub fn with_deployment_roles(mut self, roles: DeploymentRoles) -> Self {
1818 self.deployment_roles = roles;
1819 self
1820 }
1821}
1822
1823impl<ES, SS, OS, DS, PR> EngineBuilder<ES, SS, OS, DS, PR>
1824where
1825 ES: EventStore,
1826 SS: SnapshotStore,
1827 OS: OutboxStore,
1828 DS: DeadlineStore,
1829 PR: ProcessRegistry,
1830{
1831 /// Build the [`EngineContext`].
1832 ///
1833 /// Consumes the builder. All registered modules and configured stores are
1834 /// moved into the returned [`EngineContext`].
1835 ///
1836 /// This method is only available when `ES` implements [`EventStore`].
1837 /// If you have not called [`with_event_store`], this will not compile.
1838 ///
1839 /// # Panics
1840 ///
1841 /// Panics when any registered module returns `Err` from
1842 /// [`EngineModule::configure`]. The panic message includes the module
1843 /// name and the error string so the deployment failure is actionable.
1844 ///
1845 /// [`with_event_store`]: EngineBuilder::with_event_store
1846 #[must_use]
1847 #[allow(clippy::too_many_lines)]
1848 pub fn build(self) -> EngineContext<ES, SS, OS, DS, PR> {
1849 // ── Noop store safety checks ──────────────────────────────────────────
1850 //
1851 // Noop stores lose data silently: NoopDeadlineStore drops every APERAK
1852 // deadline (BNetzA violation), NoopOutboxStore discards all outbound
1853 // messages, NoopProcessRegistry loses conversation routing on restart.
1854 //
1855 // In production builds (no `testing` feature, not running under
1856 // `#[test]`), the Noop constructors are cfg-gated out so this branch
1857 // is dead code and compiles away. In test/testing/tracing builds we
1858 // emit warnings so test harnesses see the configuration in log output.
1859 //
1860 // IMPORTANT: if you are reading this because a panic fired in production,
1861 // it means the `testing` feature was accidentally enabled in the binary.
1862 // Remove it from the production Cargo.toml feature list immediately.
1863 {
1864 let os_name = std::any::type_name::<OS>();
1865 let ds_name = std::any::type_name::<DS>();
1866 let pr_name = std::any::type_name::<PR>();
1867
1868 // Regulatory-critical stores: panic in any build context if these
1869 // are noop. OutboxStore and DeadlineStore must be durable in
1870 // production; ProcessRegistry must survive restarts.
1871 #[cfg(not(any(test, feature = "testing")))]
1872 {
1873 assert!(
1874 !ds_name.contains("NoopDeadlineStore"),
1875 "EngineBuilder::build: NoopDeadlineStore is active in a \
1876 non-testing build. This silently discards all APERAK deadlines, \
1877 which is an immediately reportable BNetzA violation \
1878 (BK6-22-024 §5, BK7-24-01-009). \
1879 Call .with_deadline_store(SlateDbStore::as_deadline_store()) \
1880 in your production engine assembly. \
1881 If this is a test, enable the 'testing' feature."
1882 );
1883 assert!(
1884 !os_name.contains("NoopOutboxStore"),
1885 "EngineBuilder::build: NoopOutboxStore is active in a \
1886 non-testing build. This silently discards all outbound \
1887 APERAK, CONTRL, and UTILMD messages. \
1888 Call .with_outbox_store(SlateDbStore::as_outbox_store()) \
1889 in your production engine assembly. \
1890 If this is a test, enable the 'testing' feature."
1891 );
1892 assert!(
1893 !pr_name.contains("NoopProcessRegistry"),
1894 "EngineBuilder::build: NoopProcessRegistry is active in a \
1895 non-testing build. This means conversation routing \
1896 (PID → stream_id lookup) is lost on every restart, \
1897 breaking all WiM, GeLi Gas, and GPKE in-flight processes. \
1898 Call .with_registry(SlateDbStore::as_process_registry()) \
1899 in your production engine assembly. \
1900 If this is a test, enable the 'testing' feature."
1901 );
1902 }
1903
1904 // In test/testing/tracing builds: emit warnings instead of panicking.
1905 #[cfg(any(test, feature = "testing", feature = "tracing"))]
1906 {
1907 let ss_name = std::any::type_name::<SS>();
1908 if ss_name.contains("NoopSnapshotStore") {
1909 tracing::warn!(
1910 store = ss_name,
1911 "EngineBuilder: NoopSnapshotStore is active — snapshots will not be \
1912 persisted. Use SlateDbStore::as_snapshot_store() in production."
1913 );
1914 }
1915 if os_name.contains("NoopOutboxStore") {
1916 tracing::warn!(
1917 store = os_name,
1918 "EngineBuilder: NoopOutboxStore is active — outbound messages will be \
1919 silently discarded. Use SlateDbStore::as_outbox_store() in production."
1920 );
1921 }
1922 if ds_name.contains("NoopDeadlineStore") {
1923 tracing::warn!(
1924 store = ds_name,
1925 "EngineBuilder: NoopDeadlineStore is active — scheduled deadlines will \
1926 not fire after restart. Use SlateDbStore::as_deadline_store() in production."
1927 );
1928 }
1929 if pr_name.contains("NoopProcessRegistry") {
1930 tracing::warn!(
1931 store = pr_name,
1932 "EngineBuilder: NoopProcessRegistry is active — process routing will be \
1933 lost on restart. Use SlateDbStore::as_process_registry() in production."
1934 );
1935 }
1936 }
1937 }
1938 // Validate every module before assembling the context.
1939 // A missing adapter or misconfigured module fails at startup (not at
1940 // first inbound message), making deployment failures observable immediately.
1941 for module in &self.modules {
1942 if let Err(msg) = module.configure() {
1943 panic!(
1944 "EngineBuilder::build: module '{}' failed configuration validation: {}",
1945 module.name(),
1946 msg
1947 );
1948 }
1949 // Validate profile requirements via the injected validator.
1950 // Domain crates declare requirements; only the binary crate (makod)
1951 // injects the edi-energy registry — domain crates need no edi-energy
1952 // import for this check.
1953 if let Some(ref validator) = self.profile_validator {
1954 for req in module.profile_requirements() {
1955 assert!(
1956 validator(req.message_type),
1957 "EngineBuilder::build: module '{}' requires an active edi-energy \
1958 profile for '{}' ({}) but none is registered for today's date. \
1959 Run `cargo xtask codegen` to add the missing profile.",
1960 module.name(),
1961 req.message_type,
1962 req.label,
1963 );
1964 }
1965 }
1966 }
1967 // Build the PID router from all registered modules.
1968 // Also assert that no two modules claim the same PID — a PID overlap
1969 // is always a configuration error: one module's messages would be
1970 // silently swallowed by another's workflow, producing missing-process
1971 // errors or incorrect audit trails.
1972 let mut pid_router = PidRouter::new();
1973 let mut pid_owners: std::collections::HashMap<u32, &str> = std::collections::HashMap::new();
1974 // Keep each module's scratch router so we can build `pid_router` from
1975 // them in a second pass with the resolved ownership table.
1976 let mut module_scratches: Vec<PidRouter> = Vec::with_capacity(self.modules.len());
1977
1978 // Pass 1 — detect conflicts, determine PID ownership (first-wins for
1979 // explicit roles, last-wins for DeploymentRoles::all()).
1980 for module in &self.modules {
1981 // Temporarily build a scratch router to read this module's PIDs
1982 // for cross-module overlap detection (module-ownership level).
1983 let mut scratch = PidRouter::new();
1984 module.register_pids_with_roles(&mut scratch, &self.deployment_roles);
1985 for pid in scratch.registered_pids() {
1986 if let Some(prev) = pid_owners.insert(pid, module.name()) {
1987 if self.deployment_roles.is_all() {
1988 // With DeploymentRoles::all() (the default), role-conditional PIDs
1989 // are registered by all modules that claim them, producing last-wins
1990 // semantics. This is acceptable for single-role and dev/test deployments.
1991 //
1992 // In production multi-role deployments where both an NB and nMSB role
1993 // are served by the same instance, set explicit roles via
1994 // `EngineBuilder::with_deployment_roles` to prevent silent misrouting.
1995 //
1996 // We emit a debug-level log here (not warn) because the vast majority
1997 // of deployments are single-role and this overlap is expected/harmless.
1998 #[cfg(feature = "tracing")]
1999 tracing::debug!(
2000 pid,
2001 previous_module = prev,
2002 current_module = module.name(),
2003 "PID registered by multiple modules with DeploymentRoles::all(); \
2004 last module wins (use with_deployment_roles for strict routing)",
2005 );
2006 let _ = prev; // suppress unused-variable warning when tracing is off
2007 } else {
2008 // Explicit roles: the FIRST module to register a PID retains ownership.
2009 // Restore the previous (first) owner and emit a warning so the operator
2010 // can investigate. A panic would be too strict: some shared PIDs
2011 // (e.g. REMADV 33001/33002) are legitimately claimed by both GPKE and
2012 // WiM billing; conversation-ID routing is the long-term solution, but
2013 // first-wins gives correct behaviour for all current deployments.
2014 pid_owners.insert(pid, prev); // restore first owner
2015 #[cfg(feature = "tracing")]
2016 tracing::warn!(
2017 pid,
2018 first_module = prev,
2019 second_module = module.name(),
2020 "PID {pid} claimed by both '{prev}' and '{}' with explicit \
2021 DeploymentRoles; first module ('{prev}') retains ownership. \
2022 Verify PID registration is correct for this deployment.",
2023 module.name(),
2024 );
2025 #[cfg(not(feature = "tracing"))]
2026 let _ = prev; // suppress unused-variable warning when tracing is off
2027 }
2028 }
2029 }
2030 module_scratches.push(scratch);
2031 }
2032
2033 // Pass 2 — build the real `pid_router` from the scratch pads, respecting
2034 // the ownership table built in pass 1.
2035 for (module, scratch) in self.modules.iter().zip(module_scratches.iter()) {
2036 // Unambiguous (Sparte-agnostic) entries: only register if this module
2037 // owns the PID in the resolved ownership table.
2038 for pid in scratch.registered_pids() {
2039 if pid_owners.get(&pid).copied() == Some(module.name())
2040 && let Some(wf) = scratch.route(pid)
2041 {
2042 pid_router.register(pid, wf);
2043 }
2044 }
2045 // Commodity (Sparte-qualified) entries use distinct (pid, Sparte) keys
2046 // and never conflict across modules; register them all unconditionally.
2047 for (pid, sparte, wf) in scratch.registered_commodity_entries() {
2048 pid_router.register_with_sparte(pid, sparte, wf);
2049 }
2050 }
2051 let registered_modules = self.modules.iter().map(|m| m.name()).collect();
2052 let registered_workflows = self
2053 .modules
2054 .iter()
2055 .flat_map(|m| m.workflow_names().iter().copied())
2056 .collect();
2057 EngineContext {
2058 event_store: Arc::new(self.event_store),
2059 snapshot_store: self.snapshot_store,
2060 outbox_store: self.outbox_store,
2061 deadline_store: self.deadline_store,
2062 registry: self.registry,
2063 dead_letter_sink: self.dead_letter_sink,
2064 pid_router,
2065 registered_modules,
2066 registered_workflows,
2067 }
2068 }
2069}
2070
2071#[cfg(test)]
2072mod tests {
2073 use super::*;
2074 use crate::{
2075 deadline::InMemoryDeadlineStore,
2076 error::WorkflowError,
2077 event_store::InMemoryEventStore,
2078 ids::TenantId,
2079 outbox::InMemoryOutboxStore,
2080 pid_router::PidRouter,
2081 registry::InMemoryProcessRegistry,
2082 snapshot::InMemorySnapshotStore,
2083 version::WorkflowId,
2084 workflow::{CommandPayload, EventPayload, Workflow},
2085 };
2086
2087 // ── Minimal workflow for spawn/resume tests ───────────────────────────────
2088
2089 #[derive(serde::Serialize, serde::Deserialize)]
2090 struct PingEvent;
2091
2092 impl EventPayload for PingEvent {
2093 fn event_type(&self) -> &'static str {
2094 "Ping"
2095 }
2096 }
2097
2098 struct PingCommand;
2099
2100 impl CommandPayload for PingCommand {}
2101
2102 #[derive(Default, Clone)]
2103 struct PingState;
2104
2105 struct PingWorkflow;
2106
2107 impl Workflow for PingWorkflow {
2108 type State = PingState;
2109 type Event = PingEvent;
2110 type Command = PingCommand;
2111
2112 fn apply(state: PingState, _: &PingEvent) -> PingState {
2113 state
2114 }
2115
2116 fn handle(
2117 _: &PingState,
2118 _: PingCommand,
2119 ) -> Result<crate::workflow::WorkflowOutput<PingEvent>, WorkflowError> {
2120 Ok(vec![PingEvent].into())
2121 }
2122 }
2123
2124 struct TestModule;
2125
2126 impl EngineModule for TestModule {
2127 fn name(&self) -> &'static str {
2128 "test-module"
2129 }
2130 }
2131
2132 // ── Tests ─────────────────────────────────────────────────────────────────
2133
2134 #[test]
2135 fn build_with_event_store_only() {
2136 let ctx = EngineBuilder::new()
2137 .with_event_store(InMemoryEventStore::new())
2138 .build();
2139 assert!(ctx.registered_modules().is_empty());
2140 }
2141
2142 #[test]
2143 fn build_with_all_stores_and_module() {
2144 let ctx = EngineBuilder::new()
2145 .with_event_store(InMemoryEventStore::new())
2146 .with_snapshot_store(InMemorySnapshotStore::new())
2147 .with_outbox_store(InMemoryOutboxStore::new())
2148 .with_deadline_store(InMemoryDeadlineStore::new())
2149 .with_registry(InMemoryProcessRegistry::new())
2150 .register(Box::new(TestModule))
2151 .build();
2152 assert_eq!(ctx.registered_modules(), &["test-module"]);
2153 }
2154
2155 #[test]
2156 fn multiple_modules_ordered() {
2157 struct ModA;
2158 impl EngineModule for ModA {
2159 fn name(&self) -> &'static str {
2160 "mod-a"
2161 }
2162 }
2163 struct ModB;
2164 impl EngineModule for ModB {
2165 fn name(&self) -> &'static str {
2166 "mod-b"
2167 }
2168 }
2169
2170 let ctx = EngineBuilder::new()
2171 .with_event_store(InMemoryEventStore::new())
2172 .register(Box::new(ModA))
2173 .register(Box::new(ModB))
2174 .build();
2175 assert_eq!(ctx.registered_modules(), &["mod-a", "mod-b"]);
2176 }
2177
2178 #[tokio::test]
2179 async fn spawn_creates_independent_processes() {
2180 let ctx = EngineBuilder::new()
2181 .with_event_store(InMemoryEventStore::new())
2182 .build();
2183 let wf_id = WorkflowId::new("ping", "FV2024-10-01");
2184
2185 let p1 = ctx.spawn::<PingWorkflow>(TenantId::new(), wf_id.clone());
2186 let p2 = ctx.spawn::<PingWorkflow>(TenantId::new(), wf_id);
2187
2188 assert_ne!(p1.process_id(), p2.process_id());
2189 }
2190
2191 #[tokio::test]
2192 async fn resume_sees_previously_appended_events() {
2193 let store = InMemoryEventStore::new();
2194 let ctx = EngineBuilder::new().with_event_store(store).build();
2195
2196 let p = ctx.spawn::<PingWorkflow>(TenantId::new(), WorkflowId::new("ping", "FV2024-10-01"));
2197 p.execute(PingCommand).await.unwrap();
2198
2199 let identity = p.identity();
2200 let resumed = ctx.resume::<PingWorkflow>(identity);
2201 assert_eq!(resumed.event_count().await.unwrap(), 1);
2202 }
2203
2204 #[tokio::test]
2205 async fn registry_routes_process_via_conversation_key() {
2206 use crate::registry::RegistryKey;
2207 let ctx = EngineBuilder::new()
2208 .with_event_store(InMemoryEventStore::new())
2209 .with_registry(InMemoryProcessRegistry::new())
2210 .build();
2211
2212 let p = ctx.spawn::<PingWorkflow>(TenantId::new(), WorkflowId::new("ping", "FV2024-10-01"));
2213 let tenant = p.tenant_id();
2214 let conv_key = RegistryKey::parse("conv:test-conversation-123").expect("valid key");
2215 ctx.registry()
2216 .register(tenant, &conv_key, p.identity())
2217 .await
2218 .unwrap();
2219
2220 let found = ctx
2221 .registry()
2222 .lookup(tenant, &conv_key)
2223 .await
2224 .unwrap()
2225 .expect("must be registered");
2226 let resumed = ctx.resume::<PingWorkflow>(found);
2227 assert_eq!(resumed.process_id(), p.process_id());
2228 }
2229
2230 #[test]
2231 fn pid_router_populated_by_module_register_pids() {
2232 struct PidModule;
2233 impl EngineModule for PidModule {
2234 fn name(&self) -> &'static str {
2235 "pid-module"
2236 }
2237 fn register_pids(&self, router: &mut PidRouter) {
2238 router.register(55001, "gpke-supplier-change");
2239 router.register(55002, "gpke-supplier-change");
2240 }
2241 }
2242
2243 let ctx = EngineBuilder::new()
2244 .with_event_store(InMemoryEventStore::new())
2245 .register(Box::new(PidModule))
2246 .build();
2247
2248 assert_eq!(ctx.pid_router().route(55001), Some("gpke-supplier-change"));
2249 assert_eq!(ctx.pid_router().route(55002), Some("gpke-supplier-change"));
2250 assert!(ctx.pid_router().route(99999).is_none());
2251 assert_eq!(ctx.pid_router().len(), 2);
2252 }
2253
2254 /// Verify that `register_pids_with_roles` gates PIDs behind role checks.
2255 ///
2256 /// Scenario: two modules share PID 19001.
2257 /// - ModuleA registers 19001 → "workflow-a" when role `Nb` is present.
2258 /// - ModuleB registers 19001 → "workflow-b" when role `Nmsb` is explicitly set
2259 /// (not on `all()`).
2260 ///
2261 /// - `all()`: ModuleA fires (Nb ∈ all), ModuleB does NOT (is_all → skip).
2262 /// → 19001 routes to "workflow-a".
2263 /// - `from_roles([Nb])`: ModuleA fires, ModuleB skips.
2264 /// → 19001 routes to "workflow-a".
2265 /// - `from_roles([Nmsb])`: ModuleA skips, ModuleB fires.
2266 /// → 19001 routes to "workflow-b".
2267 #[test]
2268 fn register_pids_with_roles_gates_pids_correctly() {
2269 use crate::marktrolle::{DeploymentRoles, Marktrolle};
2270
2271 struct ModuleA;
2272 impl EngineModule for ModuleA {
2273 fn name(&self) -> &'static str {
2274 "module-a"
2275 }
2276 fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
2277 if roles.contains(Marktrolle::Nb) {
2278 router.register(19_001, "workflow-a");
2279 }
2280 }
2281 }
2282
2283 struct ModuleB;
2284 impl EngineModule for ModuleB {
2285 fn name(&self) -> &'static str {
2286 "module-b"
2287 }
2288 fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
2289 // Only fires on explicit Nmsb, not on all() (backward-compat sentinel).
2290 if !roles.is_all() && roles.contains(Marktrolle::Nmsb) {
2291 router.register(19_001, "workflow-b");
2292 router.register(19_015, "workflow-b");
2293 }
2294 }
2295 }
2296
2297 let build = |roles: DeploymentRoles| {
2298 EngineBuilder::new()
2299 .with_event_store(InMemoryEventStore::new())
2300 .with_deployment_roles(roles)
2301 .register(Box::new(ModuleA))
2302 .register(Box::new(ModuleB))
2303 .build()
2304 };
2305
2306 // all() → backward compat: ModuleA registers 19001 (Nb ∈ all), ModuleB skips.
2307 let ctx = build(DeploymentRoles::all());
2308 assert_eq!(ctx.pid_router().route(19_001), Some("workflow-a"));
2309 assert!(ctx.pid_router().route(19_015).is_none());
2310
2311 // Explicit Nb → same result: ModuleA registers, ModuleB (nMSB) skips.
2312 let ctx = build(DeploymentRoles::nb());
2313 assert_eq!(ctx.pid_router().route(19_001), Some("workflow-a"));
2314 assert!(ctx.pid_router().route(19_015).is_none());
2315
2316 // Explicit Nmsb → ModuleA skips (Nb ∉ roles), ModuleB registers.
2317 let ctx = build(DeploymentRoles::nmsb());
2318 assert_eq!(ctx.pid_router().route(19_001), Some("workflow-b"));
2319 assert_eq!(ctx.pid_router().route(19_015), Some("workflow-b"));
2320 }
2321
2322 /// Verify that explicit roles with two conflicting modules use first-wins semantics
2323 /// (the first module to register a PID retains ownership; the second is silently skipped).
2324 #[test]
2325 fn register_pids_with_roles_conflict_uses_first_wins_with_explicit_roles() {
2326 use crate::marktrolle::{DeploymentRoles, Marktrolle};
2327
2328 struct ConflictA;
2329 impl EngineModule for ConflictA {
2330 fn name(&self) -> &'static str {
2331 "conflict-a"
2332 }
2333 fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
2334 if roles.contains(Marktrolle::Nb) {
2335 router.register(19_001, "workflow-a");
2336 }
2337 }
2338 }
2339
2340 struct ConflictB;
2341 impl EngineModule for ConflictB {
2342 fn name(&self) -> &'static str {
2343 "conflict-b"
2344 }
2345 fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
2346 if !roles.is_all() && roles.contains(Marktrolle::Nmsb) {
2347 router.register(19_001, "workflow-b"); // same PID, different workflow
2348 }
2349 }
2350 }
2351
2352 // from_roles([Nb, Nmsb]): both modules fire for PID 19_001.
2353 // First-wins: ConflictA (registered first) retains ownership → "workflow-a".
2354 let ctx = EngineBuilder::new()
2355 .with_event_store(InMemoryEventStore::new())
2356 .with_deployment_roles(DeploymentRoles::from_roles([
2357 Marktrolle::Nb,
2358 Marktrolle::Nmsb,
2359 ]))
2360 .register(Box::new(ConflictA))
2361 .register(Box::new(ConflictB))
2362 .build();
2363 assert_eq!(
2364 ctx.pid_router().route(19_001),
2365 Some("workflow-a"),
2366 "first module should win on PID conflict with explicit roles"
2367 );
2368 }
2369
2370 // ── Graceful shutdown ─────────────────────────────────────────────────────
2371
2372 /// Cancelling the token must make `run` return.
2373 ///
2374 /// The workers used to loop until the process exited: the shutdown path
2375 /// cancelled a token nobody read, dropped their `JoinHandle`s — which does
2376 /// not abort a Tokio task — and then closed the event store underneath
2377 /// them. An outbox `acknowledge` losing that race leaves the counterparty
2378 /// holding a message the outbox still shows as pending, and the next start
2379 /// delivers it again.
2380 #[tokio::test]
2381 async fn a_cancelled_outbox_worker_returns() {
2382 let worker = OutboxWorker {
2383 store: InMemoryOutboxStore::new(),
2384 sender: AlwaysDelivers,
2385 deadline_store: InMemoryDeadlineStore::new(),
2386 batch_size: 10,
2387 // Far longer than the timeout below: the point is that cancellation
2388 // interrupts the idle sleep rather than being noticed after it.
2389 poll_interval: std::time::Duration::from_secs(300),
2390 max_attempts: 48,
2391 max_retry_window: std::time::Duration::from_secs(72 * 3600),
2392 dead_letter_sink: std::sync::Arc::new(crate::dead_letter::LogDeadLetterSink),
2393 heartbeat: None,
2394 shutdown: None,
2395 };
2396 let token = tokio_util::sync::CancellationToken::new();
2397 let worker = worker.with_shutdown(token.clone());
2398
2399 let handle = tokio::spawn(worker.run());
2400 // Let it reach the sleep, then signal.
2401 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2402 token.cancel();
2403
2404 tokio::time::timeout(std::time::Duration::from_secs(5), handle)
2405 .await
2406 .expect("outbox worker must return promptly after cancellation")
2407 .expect("outbox worker must not panic");
2408 }
2409
2410 #[tokio::test]
2411 async fn a_cancelled_deadline_scheduler_returns() {
2412 let scheduler = DeadlineScheduler {
2413 store: InMemoryDeadlineStore::new(),
2414 dispatch: Box::new(|_| Box::pin(async { Ok(()) })),
2415 batch_size: 100,
2416 poll_interval: std::time::Duration::from_secs(300),
2417 heartbeat: None,
2418 shutdown: None,
2419 };
2420 let token = tokio_util::sync::CancellationToken::new();
2421 let scheduler = scheduler.with_shutdown(token.clone());
2422
2423 let handle = tokio::spawn(scheduler.run());
2424 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2425 token.cancel();
2426
2427 tokio::time::timeout(std::time::Duration::from_secs(5), handle)
2428 .await
2429 .expect("deadline scheduler must return promptly after cancellation")
2430 .expect("deadline scheduler must not panic");
2431 }
2432
2433 /// A token cancelled before the first poll must stop the worker without it
2434 /// touching the store at all — the case where shutdown arrives during boot.
2435 #[tokio::test]
2436 async fn a_worker_cancelled_before_it_starts_does_no_work() {
2437 let outbox = InMemoryOutboxStore::new();
2438 let stream_id = crate::ids::StreamId::new("gpke/shutdown-test");
2439 let msg = outbox_message(&stream_id, "UTILMD");
2440 outbox.enqueue(std::slice::from_ref(&msg)).await.unwrap();
2441
2442 let token = tokio_util::sync::CancellationToken::new();
2443 token.cancel();
2444
2445 let worker = OutboxWorker {
2446 store: outbox.clone(),
2447 sender: AlwaysDelivers,
2448 deadline_store: InMemoryDeadlineStore::new(),
2449 batch_size: 10,
2450 poll_interval: std::time::Duration::from_millis(5),
2451 max_attempts: 48,
2452 max_retry_window: std::time::Duration::from_secs(72 * 3600),
2453 dead_letter_sink: std::sync::Arc::new(crate::dead_letter::LogDeadLetterSink),
2454 heartbeat: None,
2455 shutdown: None,
2456 }
2457 .with_shutdown(token);
2458
2459 tokio::time::timeout(std::time::Duration::from_secs(5), worker.run())
2460 .await
2461 .expect("an already-cancelled worker must return immediately");
2462
2463 assert_eq!(
2464 outbox.pending_now(10).await.unwrap().len(),
2465 1,
2466 "the message must stay queued for the next start, not be delivered \
2467 by a worker that was told to stop",
2468 );
2469 }
2470
2471 // ── APERAK delivery-window discharge ──────────────────────────────────────
2472
2473 /// A sender that always succeeds, so the worker takes the delivery path.
2474 struct AlwaysDelivers;
2475 impl As4Sender for AlwaysDelivers {
2476 async fn send(&self, _msg: &crate::outbox::OutboxMessage) -> Result<(), EngineError> {
2477 Ok(())
2478 }
2479 }
2480
2481 fn outbox_message(
2482 stream_id: &crate::ids::StreamId,
2483 message_type: &str,
2484 ) -> crate::outbox::OutboxMessage {
2485 crate::outbox::OutboxMessage::new(
2486 stream_id.clone(),
2487 crate::ids::ProcessId::new(),
2488 TenantId::new(),
2489 crate::ids::CorrelationId::new(),
2490 crate::ids::ConversationId::new(),
2491 crate::ids::EventId::new(),
2492 message_type,
2493 "9900357000004",
2494 serde_json::json!({}),
2495 )
2496 }
2497
2498 fn deadline_on(
2499 stream_id: &crate::ids::StreamId,
2500 msg: &crate::outbox::OutboxMessage,
2501 label: &str,
2502 ) -> Deadline {
2503 Deadline::new(
2504 stream_id.clone(),
2505 msg.process_id,
2506 msg.tenant_id,
2507 WorkflowId::new("gpke-supplier-change", "FV2025-10-01"),
2508 label,
2509 time::OffsetDateTime::now_utc() + time::Duration::hours(6),
2510 )
2511 }
2512
2513 /// Deliver `msg` through the worker's real loop and return the labels that
2514 /// survive on its stream.
2515 ///
2516 /// Drives `run` rather than calling the discharge directly — the wiring is
2517 /// the thing under test, and calling the method straight passes even when
2518 /// `run` never invokes it.
2519 async fn labels_surviving_delivery(
2520 msg: &crate::outbox::OutboxMessage,
2521 stream_id: &crate::ids::StreamId,
2522 registered: &[&str],
2523 ) -> Vec<String> {
2524 let deadlines = InMemoryDeadlineStore::new();
2525 for label in registered {
2526 deadlines
2527 .register(&deadline_on(stream_id, msg, label))
2528 .await
2529 .unwrap();
2530 }
2531 let outbox = InMemoryOutboxStore::new();
2532 outbox.enqueue(std::slice::from_ref(msg)).await.unwrap();
2533
2534 let worker = OutboxWorker {
2535 store: outbox.clone(),
2536 sender: AlwaysDelivers,
2537 deadline_store: deadlines.clone(),
2538 batch_size: 10,
2539 poll_interval: std::time::Duration::from_millis(5),
2540 max_attempts: 48,
2541 max_retry_window: std::time::Duration::from_secs(72 * 3600),
2542 dead_letter_sink: std::sync::Arc::new(crate::dead_letter::LogDeadLetterSink),
2543 heartbeat: None,
2544 shutdown: None,
2545 };
2546 // `run` never returns; give it enough cycles to drain the one message.
2547 let _ = tokio::time::timeout(std::time::Duration::from_millis(300), worker.run()).await;
2548 assert!(
2549 outbox.pending_now(10).await.unwrap().is_empty(),
2550 "the message must have been delivered and acknowledged",
2551 );
2552
2553 let mut left: Vec<String> = deadlines
2554 .for_stream(stream_id)
2555 .await
2556 .unwrap()
2557 .iter()
2558 .map(|d| d.label().to_owned())
2559 .collect();
2560 left.sort();
2561 left
2562 }
2563
2564 /// Delivering a message must retire the window that was watching for it.
2565 ///
2566 /// These windows are registered when the message is enqueued and nothing
2567 /// else ever cancels them, so without the discharge they fire for **every**
2568 /// process — including every one that answered on time. The scheduler cannot
2569 /// tell those apart (a deadline reaching `due_now` is late by construction),
2570 /// so the miss counters would track processes started, not obligations
2571 /// missed.
2572 #[tokio::test]
2573 async fn delivering_a_message_discharges_its_delivery_window() {
2574 // (message type, the window it answers for)
2575 for (message_type, window) in [
2576 ("APERAK", mako_fristen::APERAK_STROM_WINDOW_LABEL),
2577 ("APERAK", mako_fristen::APERAK_GAS_FOLGEPROZESS_LABEL),
2578 ("APERAK", mako_fristen::APERAK_GAS_INITIALPROZESS_LABEL),
2579 ("CONTRL", mako_fristen::CONTRL_FRIST_LABEL),
2580 ] {
2581 let stream_id = crate::ids::StreamId::new("gpke-supplier-change-1");
2582 let msg = outbox_message(&stream_id, message_type);
2583 // A process-response deadline shares the stream and must survive:
2584 // it is waiting on the counterparty, not on our delivery.
2585 let left =
2586 labels_surviving_delivery(&msg, &stream_id, &[window, "gpke-response-window"])
2587 .await;
2588 assert_eq!(
2589 left,
2590 vec!["gpke-response-window"],
2591 "delivering {message_type} must discharge `{window}` and leave \
2592 every other deadline alone",
2593 );
2594 }
2595 }
2596
2597 /// A delivery must not discharge a *different* message's window.
2598 ///
2599 /// The CONTRL and APERAK obligations run concurrently on the same
2600 /// interchange. Acknowledging syntax (CONTRL) says nothing about whether the
2601 /// application-level APERAK went out, so discharging both on one delivery
2602 /// would silence a real violation.
2603 #[tokio::test]
2604 async fn a_delivery_does_not_discharge_another_messages_window() {
2605 let stream_id = crate::ids::StreamId::new("gpke-supplier-change-1");
2606 let contrl = outbox_message(&stream_id, "CONTRL");
2607
2608 let left = labels_surviving_delivery(
2609 &contrl,
2610 &stream_id,
2611 &[
2612 mako_fristen::CONTRL_FRIST_LABEL,
2613 mako_fristen::APERAK_STROM_WINDOW_LABEL,
2614 ],
2615 )
2616 .await;
2617
2618 assert_eq!(
2619 left,
2620 vec![mako_fristen::APERAK_STROM_WINDOW_LABEL.to_owned()],
2621 "a delivered CONTRL discharges only the CONTRL window; the APERAK \
2622 obligation is still outstanding",
2623 );
2624 }
2625
2626 /// Every delivery-window label must be discharged by the message it watches.
2627 ///
2628 /// This is the invariant the miss counters rest on. A window label that
2629 /// `discharges_delivery_window` does not recognise is never retired, so it
2630 /// fires on every process and is counted as a regulatory violation each
2631 /// time — which is precisely how `makod_aperak_missed_total` once came to
2632 /// count Strom processes rather than missed APERAKs.
2633 ///
2634 /// Adding a delivery window means adding a row here.
2635 #[test]
2636 fn every_delivery_window_label_is_discharged_by_its_message() {
2637 for (message_type, label) in [
2638 ("APERAK", mako_fristen::APERAK_STROM_WINDOW_LABEL),
2639 ("APERAK", mako_fristen::APERAK_GAS_FOLGEPROZESS_LABEL),
2640 ("APERAK", mako_fristen::APERAK_GAS_INITIALPROZESS_LABEL),
2641 ("CONTRL", mako_fristen::CONTRL_FRIST_LABEL),
2642 ] {
2643 assert!(
2644 mako_fristen::discharges_delivery_window(message_type, label),
2645 "delivering {message_type} must discharge `{label}`, or the window \
2646 outlives its obligation and alerts on every process",
2647 );
2648 }
2649 }
2650
2651 // ── Retry-budget classification ───────────────────────────────────────────
2652
2653 /// Sink double that records every rejection's attempt count.
2654 #[derive(Default)]
2655 struct RecordingSink(std::sync::Mutex<Vec<u32>>);
2656 impl crate::dead_letter::DeadLetterSink for std::sync::Arc<RecordingSink> {
2657 fn reject(&self, reason: &crate::dead_letter::DeadLetterReason) {
2658 if let crate::dead_letter::DeadLetterReason::OutboxExhausted { attempts, .. } = reason {
2659 self.0
2660 .lock()
2661 .unwrap_or_else(std::sync::PoisonError::into_inner)
2662 .push(*attempts);
2663 }
2664 }
2665 }
2666
2667 struct NoRenderer;
2668 impl As4Sender for NoRenderer {
2669 async fn send(&self, msg: &crate::outbox::OutboxMessage) -> Result<(), EngineError> {
2670 Err(EngineError::RendererNotImplemented {
2671 message_type: msg.message_type.as_ref().into(),
2672 message_id: msg.message_id.to_string().into(),
2673 })
2674 }
2675 }
2676
2677 async fn drive_worker<S: As4Sender>(
2678 sender: S,
2679 msg: crate::outbox::OutboxMessage,
2680 ) -> (InMemoryOutboxStore, std::sync::Arc<RecordingSink>) {
2681 let outbox = InMemoryOutboxStore::new();
2682 outbox.enqueue(std::slice::from_ref(&msg)).await.unwrap();
2683 let sink = std::sync::Arc::new(RecordingSink::default());
2684 let worker = OutboxWorker {
2685 store: outbox.clone(),
2686 sender,
2687 deadline_store: InMemoryDeadlineStore::new(),
2688 batch_size: 10,
2689 poll_interval: std::time::Duration::from_millis(5),
2690 max_attempts: 48,
2691 max_retry_window: std::time::Duration::from_secs(72 * 3600),
2692 dead_letter_sink: std::sync::Arc::new(std::sync::Arc::clone(&sink)),
2693 heartbeat: None,
2694 shutdown: None,
2695 };
2696 let _ = tokio::time::timeout(std::time::Duration::from_millis(300), worker.run()).await;
2697 (outbox, sink)
2698 }
2699
2700 /// `RendererNotImplemented` is documented as permanent — the worker must
2701 /// dead-letter it on the *first* attempt, not burn the retry budget on a
2702 /// failure that cannot heal between attempts. Until the permanent arm
2703 /// matched it, this promise was broken.
2704 #[tokio::test(start_paused = true)]
2705 async fn a_missing_renderer_dead_letters_without_retrying() {
2706 let stream_id = crate::ids::StreamId::new("test-renderer-missing");
2707 let msg = outbox_message(&stream_id, "MSCONS");
2708 let (outbox, sink) = drive_worker(NoRenderer, msg).await;
2709
2710 assert!(
2711 outbox.pending_now(10).await.unwrap().is_empty(),
2712 "the message must be acknowledged, not left for another attempt",
2713 );
2714 let rejections = sink
2715 .0
2716 .lock()
2717 .unwrap_or_else(std::sync::PoisonError::into_inner)
2718 .clone();
2719 assert_eq!(
2720 rejections,
2721 vec![0],
2722 "exactly one dead-letter, on the first attempt (attempt_count 0)",
2723 );
2724 }
2725
2726 /// The retry budget is a *window*, not a count: a message whose age has
2727 /// exceeded it after at least one attempt is dead-lettered even though the
2728 /// attempt belt is nowhere near exhausted — full-jitter backoff makes a
2729 /// count no proxy for the 72 h duty.
2730 #[tokio::test(start_paused = true)]
2731 async fn an_aged_message_with_a_prior_attempt_is_dead_lettered() {
2732 let stream_id = crate::ids::StreamId::new("test-window-exhausted");
2733 let mut msg = outbox_message(&stream_id, "UTILMD");
2734 msg.created_at = time::OffsetDateTime::now_utc() - time::Duration::hours(73);
2735 msg.attempt_count = 1;
2736 let (outbox, sink) = drive_worker(AlwaysDelivers, msg).await;
2737
2738 assert!(
2739 outbox.pending_now(10).await.unwrap().is_empty(),
2740 "the exhausted message must leave the outbox",
2741 );
2742 let rejections = sink
2743 .0
2744 .lock()
2745 .unwrap_or_else(std::sync::PoisonError::into_inner)
2746 .clone();
2747 assert_eq!(
2748 rejections,
2749 vec![1],
2750 "the window, not the attempt belt, must have dead-lettered it",
2751 );
2752 }
2753
2754 /// A message that aged past the window while the worker was down still
2755 /// gets its first try — the window is only consulted after an attempt, so
2756 /// downtime never buries a message unsent.
2757 #[tokio::test(start_paused = true)]
2758 async fn an_aged_message_with_no_attempts_is_still_tried_once() {
2759 let stream_id = crate::ids::StreamId::new("test-aged-first-try");
2760 let mut msg = outbox_message(&stream_id, "UTILMD");
2761 msg.created_at = time::OffsetDateTime::now_utc() - time::Duration::hours(200);
2762 let (outbox, sink) = drive_worker(AlwaysDelivers, msg).await;
2763
2764 assert!(
2765 outbox.pending_now(10).await.unwrap().is_empty(),
2766 "the message must have been delivered and acknowledged",
2767 );
2768 assert!(
2769 sink.0
2770 .lock()
2771 .unwrap_or_else(std::sync::PoisonError::into_inner)
2772 .is_empty(),
2773 "delivery, not dead-lettering: age alone must never bury a message",
2774 );
2775 }
2776}