mako_engine/builder.rs
1//! [`EngineModule`] trait, [`EngineBuilder`], and [`EngineContext`].
2//!
3// Allow using deprecated Noop stores as *type-level defaults* in EngineBuilder / EngineContext
4// generic parameters. The types are deprecated to prevent instantiation in production code,
5// but using them as default type parameters in struct definitions (not instantiating them) is
6// the intended pattern for the type-state builder API.
7#![allow(deprecated)]
8//!
9//! # Summary
10//!
11//! `EngineBuilder` assembles all engine infrastructure into a single
12//! [`EngineContext`] value. Domain modules (GPKE, WiM, GeLi Gas, …) register
13//! themselves at startup via the [`EngineModule`] trait, making their names
14//! visible in diagnostics and health checks.
15//!
16//! # Type-state guarantee
17//!
18//! [`EngineBuilder::build`] is only available when the event store type
19//! parameter `ES` implements [`EventStore`]. Forgetting to call
20//! [`with_event_store`] is a **compile-time error**, not a runtime panic.
21//!
22//! All other stores default to their respective `Noop` implementations:
23//!
24//! | Store | Default |
25//! |-------|---------|
26//! | Snapshot store | [`NoopSnapshotStore`] |
27//! | Outbox store | [`NoopOutboxStore`] |
28//! | Deadline store | [`NoopDeadlineStore`] |
29//! | Process registry | [`NoopProcessRegistry`] |
30//!
31//! # Assembly example
32//!
33//! ```rust,ignore
34//! use mako_engine::builder::{EngineBuilder, EngineModule};
35//! use mako_engine::event_store::InMemoryEventStore;
36//! use mako_engine::outbox::InMemoryOutboxStore;
37//! use mako_engine::deadline::InMemoryDeadlineStore;
38//! use mako_engine::registry::InMemoryProcessRegistry;
39//! use mako_engine::snapshot::InMemorySnapshotStore;
40//!
41//! struct GpkeModule;
42//! impl EngineModule for GpkeModule { fn name(&self) -> &'static str { "gpke" } }
43//!
44//! let ctx = EngineBuilder::new()
45//! .with_event_store(InMemoryEventStore::new())
46//! .with_snapshot_store(InMemorySnapshotStore::new())
47//! .with_outbox_store(InMemoryOutboxStore::new())
48//! .with_deadline_store(InMemoryDeadlineStore::new())
49//! .with_registry(InMemoryProcessRegistry::new())
50//! .register(Box::new(GpkeModule))
51//! .build();
52//!
53//! // Spawn a fresh process:
54//! let p = ctx.spawn::<SupplierChangeWorkflow>(tenant_id, workflow_id);
55//! p.execute(ReceiveUtilmd { .. }).await?;
56//!
57//! // Resume an existing process from a persisted identity:
58//! let identity = ctx.registry.lookup(&conv_id.to_string()).await?.unwrap();
59//! let p = ctx.resume::<SupplierChangeWorkflow>(identity);
60//!
61//! // Access stores for delivery workers / schedulers:
62//! let pending = ctx.outbox_store.pending_now(50).await?;
63//! let overdue = ctx.deadline_store.due_now(50).await?;
64//! ```
65//!
66//! [`with_event_store`]: EngineBuilder::with_event_store
67
68// Type-state generics can produce long signatures that trip up the
69// `type_complexity` lint; suppress it for this module only.
70#![allow(clippy::type_complexity)]
71
72// The Noop* types are marked #[deprecated] to guard against accidental
73// production use. The builder is the only place they're instantiated as
74// defaults; suppress the lint here explicitly.
75#[allow(deprecated)]
76use crate::{
77 dead_letter::{DeadLetterSink, LogDeadLetterSink},
78 deadline::{Deadline, DeadlineStore, NoopDeadlineStore},
79 error::EngineError,
80 event_store::EventStore,
81 ids::{ProcessIdentity, TenantId},
82 marktrolle::DeploymentRoles,
83 outbox::{NoopOutboxStore, OutboxMessage, OutboxStore},
84 pid_router::PidRouter,
85 process::Process,
86 registry::{NoopProcessRegistry, ProcessRegistry},
87 snapshot::{NoopSnapshotStore, SnapshotStore},
88 version::WorkflowId,
89 workflow::Workflow,
90};
91
92use std::sync::Arc;
93
94// ── EngineModule ──────────────────────────────────────────────────────────────
95
96/// A self-contained domain module that registers with the engine at startup.
97///
98/// Domain crates implement this trait to declare their presence in the engine.
99/// The module name is surfaced in [`EngineContext::registered_modules`] for
100/// diagnostics, health checks, and log output.
101///
102/// ## Startup validation
103///
104/// Override [`configure`] to perform adapter coverage checks at engine startup
105/// time. The engine calls [`configure`] for every registered module during
106/// [`EngineBuilder::build`] and panics with an actionable message if any
107/// module returns `Err`. This surfaces missing adapter registrations as a
108/// startup failure rather than a silent runtime error.
109///
110/// ## Example
111///
112/// ```rust,ignore
113/// pub struct GpkeModule;
114///
115/// impl EngineModule for GpkeModule {
116/// fn name(&self) -> &'static str { "gpke" }
117///
118/// fn configure(&self) -> Result<(), String> {
119/// // Validate that every known BDEW format version has an adapter:
120/// GPKE_ADAPTER_REGISTRY
121/// .validate_policy(&GpkeWorkflow::version_policy(), &KNOWN_FVS)
122/// .map_err(|uncovered| format!(
123/// "gpke: missing adapters for format versions: {:?}",
124/// uncovered
125/// ))
126/// }
127/// }
128///
129/// let ctx = EngineBuilder::new()
130/// .with_event_store(my_store)
131/// .register(Box::new(GpkeModule))
132/// .build(); // panics if GpkeModule::configure returns Err
133///
134/// assert_eq!(ctx.registered_modules(), &["gpke"]);
135/// ```
136///
137/// [`configure`]: EngineModule::configure
138pub trait EngineModule: Send + 'static {
139 /// Stable, unique name for this domain module.
140 ///
141 /// Used in diagnostics, health checks, and structured log output.
142 /// Choose a short lowercase identifier (e.g. `"gpke"`, `"wim"`,
143 /// `"geli"`).
144 fn name(&self) -> &'static str;
145
146 /// Register all PIDs this module handles into the shared [`PidRouter`].
147 ///
148 /// # Mutability contract
149 ///
150 /// This method is called **exactly once** by [`EngineBuilder::build`],
151 /// before the resulting [`EngineContext`] is handed to the caller. The
152 /// `&mut PidRouter` reference is only available here, at build time.
153 /// After `build` returns the router is **sealed** — the engine provides
154 /// only a shared `&PidRouter` reference, with no mutation path at runtime.
155 ///
156 /// Consequence: **all PIDs a module will ever need must be registered
157 /// here**. Do not attempt to register PIDs lazily from async handlers or
158 /// after the engine has started — there is no API for that by design.
159 ///
160 /// Duplicate registrations (same PID from two modules) silently overwrite
161 /// the previous mapping; the last module to register wins. Use
162 /// `cargo xtask validate-pruefids` to catch accidental PID conflicts
163 /// between modules before they reach production.
164 ///
165 /// For role-conditional registration (PIDs that should only be active for
166 /// specific BDEW Marktrollen), override [`register_pids_with_roles`] instead.
167 ///
168 /// # Example
169 ///
170 /// ```rust,ignore
171 /// fn register_pids(&self, router: &mut PidRouter) {
172 /// // GPKE Lieferantenwechsel / Lieferbeginn (BK6-22-024, PIDs 55001, 55002, 55017)
173 /// for &pid in &[55001_u32, 55002, 55017] {
174 /// router.register(pid, "gpke-supplier-change");
175 /// }
176 /// }
177 /// ```
178 ///
179 /// [`register_pids_with_roles`]: EngineModule::register_pids_with_roles
180 fn register_pids(&self, _router: &mut PidRouter) {}
181
182 /// Register PIDs with role-context awareness.
183 ///
184 /// This is the **preferred override** for modules that have role-conditional
185 /// PID registrations — PIDs that should only be active when this `makod`
186 /// instance holds a specific [`Marktrolle`].
187 ///
188 /// The default implementation calls [`register_pids`] (role-agnostic) so
189 /// existing modules that override `register_pids` continue to work without
190 /// changes.
191 ///
192 /// Override this method instead of `register_pids` when any PID registration
193 /// should be conditional on the deployment role:
194 ///
195 /// ```rust,ignore
196 /// use mako_engine::marktrolle::Marktrolle;
197 ///
198 /// fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
199 /// // Always register: 55001, 55002 (not role-specific)
200 /// for pid in [55001_u32, 55002] { router.register_with_module(pid, "gpke-supplier-change", self.name()); }
201 ///
202 /// // Only when NB role: 19001/19002 inbound ORDRSP from MSB
203 /// if roles.contains(Marktrolle::Nb) {
204 /// for pid in [19001_u32, 19002] { router.register_with_module(pid, "gpke-konfiguration", self.name()); }
205 /// }
206 /// }
207 /// ```
208 ///
209 /// # Conflict guard
210 ///
211 /// Use [`PidRouter::register_with_module`] (not `register`) inside this
212 /// method. The conflict guard panics at build time if two modules register
213 /// the same PID to different workflows — this makes role misconfigurations
214 /// visible at startup rather than silently misrouting messages.
215 ///
216 /// [`Marktrolle`]: crate::marktrolle::Marktrolle
217 /// [`register_pids`]: EngineModule::register_pids
218 fn register_pids_with_roles(&self, router: &mut PidRouter, _roles: &DeploymentRoles) {
219 self.register_pids(router);
220 }
221
222 /// Workflow names this module handles for deadline dispatch.
223 ///
224 /// Return the same name strings that [`register_pids`] maps PIDs to.
225 /// These names are stored in [`EngineContext::registered_workflows`] and
226 /// used to validate that every workflow that has deadlines scheduled is
227 /// covered by the deadline scheduler dispatch function at runtime.
228 ///
229 /// The default implementation returns an empty slice. Override it to
230 /// declare all workflow names that may fire deadlines:
231 ///
232 /// ```rust,ignore
233 /// fn workflow_names(&self) -> &'static [&'static str] {
234 /// &["gpke-supplier-change", "gpke-abrechnung"]
235 /// }
236 /// ```
237 ///
238 /// [`register_pids`]: EngineModule::register_pids
239 /// [`EngineContext::registered_workflows`]: crate::builder::EngineContext::registered_workflows
240 fn workflow_names(&self) -> &'static [&'static str] {
241 &[]
242 }
243
244 /// Declare the EDIFACT profile types this module requires at runtime.
245 ///
246 /// Returning a non-empty slice causes [`EngineBuilder::build`] to call the
247 /// registered profile validator for each requirement. If no active profile
248 /// exists for a required message type, `build` panics with an actionable
249 /// error so deployment fails fast rather than silently.
250 ///
251 /// Domain crates declare their format requirements here rather than
252 /// calling `edi_energy::registry::ReleaseRegistry::global()` inside
253 /// `configure()`, so `edi-energy` stays out of their production
254 /// `[dependencies]`.
255 ///
256 /// ```rust,ignore
257 /// fn profile_requirements(&self) -> &'static [ProfileRequirement] {
258 /// &[
259 /// ProfileRequirement { message_type: "UTILMD", label: "UTILMD Strom (GPKE)" },
260 /// ProfileRequirement { message_type: "INVOIC", label: "INVOIC Abrechnung (GPKE)" },
261 /// ]
262 /// }
263 /// ```
264 ///
265 /// [`ProfileRequirement`]: crate::profile::ProfileRequirement
266 fn profile_requirements(&self) -> &'static [crate::profile::ProfileRequirement] {
267 &[]
268 }
269
270 /// Validate adapter coverage and configuration at engine startup.
271 ///
272 /// Called by [`EngineBuilder::build`] after all modules are registered.
273 /// Return `Ok(())` when the module is fully configured. Return `Err(msg)`
274 /// with an actionable description when an adapter or configuration is
275 /// missing — the engine will panic with that message so the deployment
276 /// fails early rather than silently.
277 ///
278 /// The default implementation is a no-op (always returns `Ok(())`).
279 /// Override it in domain crates to call
280 /// [`AdapterRegistry::validate_policy`] and emit structured errors.
281 ///
282 /// Note: if your validation needs access to the edi-energy profile
283 /// registry, use [`profile_requirements`] instead — it does not require
284 /// importing `edi-energy` in domain crates.
285 ///
286 /// [`AdapterRegistry::validate_policy`]: crate::message_adapter::AdapterRegistry::validate_policy
287 /// [`profile_requirements`]: EngineModule::profile_requirements
288 ///
289 /// # Errors
290 ///
291 /// Returns a descriptive error string when the module's configuration is invalid.
292 fn configure(&self) -> Result<(), String> {
293 Ok(())
294 }
295}
296
297// ── EngineContext ─────────────────────────────────────────────────────────────
298
299/// Assembled engine infrastructure returned by [`EngineBuilder::build`].
300///
301/// `EngineContext` bundles all stores and the process registry into a single
302/// value. It is the root dependency for:
303///
304/// - Spawning new processes ([`spawn`])
305/// - Resuming existing processes ([`resume`])
306/// - Running outbox delivery workers (`outbox_store.pending_now(…)`)
307/// - Driving the deadline scheduler (`deadline_store.due_now(…)`)
308///
309/// ## Generic parameters
310///
311/// | Param | Role | Default |
312/// |-------|------|---------|
313/// | `ES` | [`EventStore`] backend | — (required) |
314/// | `SS` | [`SnapshotStore`] backend | [`NoopSnapshotStore`] |
315/// | `OS` | [`OutboxStore`] backend | [`NoopOutboxStore`] |
316/// | `DS` | [`DeadlineStore`] backend | [`NoopDeadlineStore`] |
317/// | `PR` | [`ProcessRegistry`] backend | [`NoopProcessRegistry`] |
318///
319/// In most codebases all type parameters are inferred from the builder calls.
320///
321/// [`spawn`]: EngineContext::spawn
322/// [`resume`]: EngineContext::resume
323pub struct EngineContext<
324 ES,
325 SS = NoopSnapshotStore,
326 OS = NoopOutboxStore,
327 DS = NoopDeadlineStore,
328 PR = NoopProcessRegistry,
329> {
330 event_store: Arc<ES>,
331 snapshot_store: SS,
332 outbox_store: OS,
333 deadline_store: DS,
334 registry: PR,
335 /// Dead-letter sink for unroutable or unprocessable inbound messages.
336 ///
337 /// Stored as `Arc<dyn DeadLetterSink>` so callers can share it across
338 /// tasks without an extra type parameter on `EngineContext`.
339 pub dead_letter_sink: Arc<dyn DeadLetterSink>,
340 /// PID-to-workflow routing table, populated from all registered modules.
341 pid_router: PidRouter,
342 registered_modules: Vec<&'static str>,
343 /// Workflow names declared by all registered modules via
344 /// [`EngineModule::workflow_names`]. Used to validate deadline scheduler
345 /// coverage at runtime (see [`EngineContext::registered_workflows`]).
346 registered_workflows: Vec<&'static str>,
347}
348
349// ── Type aliases ──────────────────────────────────────────────────────────────
350
351/// An [`EngineContext`] with all optional subsystems disabled.
352///
353/// Uses `NoopSnapshotStore` and, in `testing`-enabled builds, Noop
354/// implementations for outbox, deadline, and process registry. Suitable for
355/// tests and minimal deployments where only a durable event store is required.
356///
357/// All five type parameters are inferred from context when used with
358/// [`EngineBuilder`]:
359///
360/// ```rust,ignore
361/// // Only available in test / testing-feature builds:
362/// use mako_engine::builder::{EngineBuilder, MinimalEngine};
363/// use mako_engine::event_store::InMemoryEventStore;
364///
365/// let ctx: MinimalEngine<InMemoryEventStore> = EngineBuilder::new()
366/// .with_event_store(InMemoryEventStore::new())
367/// .build();
368/// ```
369pub type MinimalEngine<ES> = EngineContext<ES>;
370
371impl<ES, SS, OS, DS, PR> EngineContext<ES, SS, OS, DS, PR>
372where
373 ES: EventStore,
374{
375 /// Spawn a new process and return a typed `Process<W, Arc<ES>>` handle.
376 ///
377 /// No `ES: Clone` bound is required — the engine stores the event store
378 /// behind an `Arc` so spawning is always a cheap pointer clone.
379 ///
380 /// ```rust,ignore
381 /// let p = ctx.spawn::<SupplierChangeWorkflow>(tenant_id, workflow_id);
382 /// p.execute(ReceiveUtilmd { .. }).await?;
383 /// ```
384 #[must_use]
385 pub fn spawn<W: Workflow>(
386 &self,
387 tenant_id: TenantId,
388 workflow_id: WorkflowId,
389 ) -> Process<W, Arc<ES>> {
390 Process::new(Arc::clone(&self.event_store), tenant_id, workflow_id)
391 }
392
393 /// Resume an existing process from a [`ProcessIdentity`].
394 ///
395 /// ```rust,ignore
396 /// let identity = ctx.registry()
397 /// .lookup(tenant_id, &conv_id.to_string())
398 /// .await?
399 /// .ok_or(EngineError::Registry("unknown conversation".into()))?;
400 /// let p = ctx.resume::<SupplierChangeWorkflow>(identity);
401 /// p.execute(HandleAperak { .. }).await?;
402 /// ```
403 #[must_use]
404 pub fn resume<W: Workflow>(&self, identity: ProcessIdentity) -> Process<W, Arc<ES>> {
405 Process::from_identity(Arc::clone(&self.event_store), identity)
406 }
407
408 /// Names of all domain modules registered with the builder, in
409 /// registration order.
410 #[must_use]
411 pub fn registered_modules(&self) -> &[&'static str] {
412 &self.registered_modules
413 }
414
415 /// Workflow names declared by all registered modules, in registration order.
416 ///
417 /// Use this in the deadline scheduler dispatch function to detect unknown
418 /// workflow names at startup. If a deadline fires for a workflow name that
419 /// is not in this list, the scheduler's dispatch function should emit an
420 /// error rather than silently dropping the deadline:
421 ///
422 /// ```rust,ignore
423 /// let known = ctx.registered_workflows().iter().copied().collect::<HashSet<_>>();
424 /// let scheduler = ctx.run_deadline_scheduler(
425 /// move |deadline| {
426 /// let wf = deadline.workflow_id().name.as_ref();
427 /// if !known.contains(wf) {
428 /// tracing::error!(workflow = %wf, "deadline fired for unregistered workflow");
429 /// return Box::pin(async { Ok(()) });
430 /// }
431 /// // dispatch by workflow name …
432 /// Box::pin(async { Ok(()) })
433 /// },
434 /// 100,
435 /// Duration::from_secs(30),
436 /// );
437 /// ```
438 #[must_use]
439 pub fn registered_workflows(&self) -> &[&'static str] {
440 &self.registered_workflows
441 }
442
443 /// The event store backend (behind an `Arc`).
444 #[must_use]
445 pub fn event_store(&self) -> &Arc<ES> {
446 &self.event_store
447 }
448
449 /// The snapshot store backend.
450 #[must_use]
451 pub fn snapshot_store(&self) -> &SS {
452 &self.snapshot_store
453 }
454
455 /// The outbox store backend.
456 ///
457 /// Poll `outbox_store().pending_now(limit)` in a background task to drain
458 /// the delivery queue.
459 #[must_use]
460 pub fn outbox_store(&self) -> &OS {
461 &self.outbox_store
462 }
463
464 /// The deadline store backend.
465 ///
466 /// Poll `deadline_store().due_now(limit)` in a background scheduler to
467 /// fire overdue process timers.
468 #[must_use]
469 pub fn deadline_store(&self) -> &DS {
470 &self.deadline_store
471 }
472
473 /// The process routing registry.
474 ///
475 /// Register a [`ProcessIdentity`] under a `(tenant_id, key)` pair at
476 /// process creation, then `lookup` it when routing inbound messages.
477 #[must_use]
478 pub fn registry(&self) -> &PR {
479 &self.registry
480 }
481
482 /// The dead-letter sink for unroutable or unprocessable messages.
483 ///
484 /// Call [`DeadLetterSink::reject`] when an inbound message cannot be
485 /// dispatched to any workflow. The default sink emits `tracing::warn!`
486 /// so rejections are always visible in the log output.
487 #[must_use]
488 pub fn dead_letter_sink(&self) -> &Arc<dyn DeadLetterSink> {
489 &self.dead_letter_sink
490 }
491
492 /// Assert that no Noop store is active — call this during production startup.
493 ///
494 /// Checks the type names of `OS`, `DS`, and `PR` against the string `"Noop"`.
495 /// Panics with a human-readable message if any match, directing the operator
496 /// to configure a persistent backend.
497 ///
498 /// # When to call
499 ///
500 /// Call this early in `makod`'s startup path (and `--check` mode) to catch
501 /// deployments where a Noop store was accidentally wired — e.g. the
502 /// `[outbox]`, `[deadline]`, or `[registry]` configuration section was
503 /// omitted from `makod.toml`. The check is defence-in-depth: in release
504 /// builds without the `testing` feature, Noop stores cannot implement the
505 /// required traits at all and the compiler would have already rejected them.
506 ///
507 /// # Panics
508 ///
509 /// Panics when any of `OS`, `DS`, or `PR` is a Noop implementation.
510 pub fn assert_production_stores(&self) {
511 let checks: &[(&str, &str)] = &[
512 ("OutboxStore", std::any::type_name::<OS>()),
513 ("DeadlineStore", std::any::type_name::<DS>()),
514 ("ProcessRegistry", std::any::type_name::<PR>()),
515 ];
516 for (trait_name, type_name) in checks {
517 assert!(
518 !type_name.contains("Noop"),
519 "makod: Noop{trait_name} is active — \
520 configure a persistent {trait_name} backend in makod.toml. \
521 Type resolved to: {type_name}"
522 );
523 }
524 }
525
526 /// The PID-to-workflow routing table.
527 ///
528 /// Populated **once** during [`EngineBuilder::build`] by calling
529 /// [`EngineModule::register_pids`] on every registered module in
530 /// registration order. After `build` returns the table is **sealed** —
531 /// it is read-only for the lifetime of the `EngineContext` and may be
532 /// freely shared across async tasks without synchronisation.
533 ///
534 /// # Mutability contract
535 ///
536 /// There is intentionally no `pid_router_mut()` accessor. Adding PIDs
537 /// after the engine is built would create a TOCTOU race between the
538 /// dispatch path (which calls `route(pid)`) and any hypothetical
539 /// concurrent mutator. Instead, register all PIDs during the build phase
540 /// via `EngineModule::register_pids`.
541 ///
542 /// If a new process family needs to be added without restarting the
543 /// binary, rebuild and restart `makod` — hot-swap of PID routing is not
544 /// supported.
545 ///
546 /// # Example — dispatch at the AS4 reception boundary
547 ///
548 /// ```rust,ignore
549 /// let workflow_name = ctx.pid_router().route(pid)
550 /// .ok_or_else(|| EngineError::Workflow(WorkflowError::InvalidCommand(
551 /// format!("no workflow registered for PID {pid}").into()
552 /// )))?;
553 ///
554 /// match workflow_name {
555 /// "gpke-supplier-change" => dispatch::<GpkeSupplierChangeWorkflow>(&ctx, pid, payload).await,
556 /// "wim-device-change" => dispatch::<WimDeviceChangeWorkflow>(&ctx, pid, payload).await,
557 /// other => Err(EngineError::Workflow(WorkflowError::InvalidCommand(
558 /// format!("unhandled workflow name: {other}").into()
559 /// ))),
560 /// }
561 /// ```
562 #[must_use]
563 pub fn pid_router(&self) -> &PidRouter {
564 &self.pid_router
565 }
566}
567
568// ── As4Sender ─────────────────────────────────────────────────────────────────
569
570/// Sends a single AS4 / EDIINT-over-HTTP outbound message.
571///
572/// Implement this trait for your AS4 gateway client and pass it to
573/// [`EngineContext::run_outbox_worker`].
574///
575/// # Contract
576///
577/// Return `Ok(())` only after the message has been **durably accepted** by the
578/// receiving MSH. Return `Err(…)` on transient or permanent failure — the
579/// outbox worker calls [`OutboxStore::reschedule`] so the message is retried.
580pub trait As4Sender: Send + Sync + 'static {
581 /// Transmit `msg` and return when the remote MSH has accepted it.
582 fn send(
583 &self,
584 msg: &OutboxMessage,
585 ) -> impl std::future::Future<Output = Result<(), EngineError>> + Send;
586}
587
588// ── OutboxWorker ──────────────────────────────────────────────────────────────
589
590/// A background worker that drains the outbox by polling pending
591/// [`OutboxMessage`]s and dispatching them via an [`As4Sender`].
592///
593/// Obtain via [`EngineContext::run_outbox_worker`] and drive by spawning
594/// [`OutboxWorker::run`] in a Tokio task.
595///
596/// # Polling behaviour
597///
598/// When the poll returns an empty batch the worker sleeps for `poll_interval`
599/// before polling again. Non-empty batches are processed immediately.
600///
601/// # Error handling
602///
603/// Successful sends are acknowledged via [`OutboxStore::acknowledge`].
604/// Failed sends are rescheduled via [`OutboxStore::reschedule`] using
605/// **full-jitter exponential backoff**: `delay = rand(0, min(MAX, BASE * 2^n))`
606/// where `n = attempt_count`. This avoids thundering-herd when multiple
607/// `makod` instances restart simultaneously after a receiver outage.
608///
609/// When `attempt_count >= max_attempts`, the message is **acknowledged** (removed
610/// from the outbox) and a [`DeadLetterReason::OutboxExhausted`] record is written
611/// to the dead-letter sink. This prevents permanently-undeliverable messages
612/// from clogging the outbox forever.
613///
614/// All errors are emitted as structured `tracing` events at `warn` / `error`
615/// level rather than `eprintln!`, so they appear in the application's log
616/// pipeline with full context (message_id, error).
617///
618/// # Example
619///
620/// ```rust,ignore
621/// use std::time::Duration;
622///
623/// let worker = ctx.run_outbox_worker(my_sender, 50, Duration::from_secs(1));
624/// tokio::spawn(async move { worker.run().await });
625/// ```
626///
627/// [`DeadLetterReason::OutboxExhausted`]: crate::dead_letter::DeadLetterReason::OutboxExhausted
628pub struct OutboxWorker<OS: OutboxStore, S: As4Sender, DS: DeadlineStore> {
629 store: OS,
630 sender: S,
631 /// Used to discharge a delivery-window deadline once the message it was
632 /// watching has actually been sent — see [`OutboxWorker::run`].
633 deadline_store: DS,
634 batch_size: usize,
635 poll_interval: std::time::Duration,
636 /// Maximum total delivery attempts before a message is dead-lettered — a
637 /// runaway belt, not the budget. The budget is [`Self::max_retry_window`]:
638 /// the backoff is full-jitter, so an attempt *count* cannot promise a
639 /// retry *duration*, and the BDEW retry duty is stated in hours.
640 max_attempts: u32,
641 /// Maximum age (from `created_at`) a message is retried for before it is
642 /// dead-lettered. This is what honours a time-stated retry duty (BDEW AS4
643 /// Kommunikationshandbuch: 72 h for unacknowledged messages) — see
644 /// `mako_as4::constants::MAX_RETRY_DURATION_SECS`.
645 ///
646 /// Checked only after at least one attempt: a message that aged in a
647 /// stopped worker still gets its first try rather than being buried
648 /// unsent.
649 max_retry_window: std::time::Duration,
650 /// Sink for messages that exceed `max_attempts` or `max_retry_window`.
651 dead_letter_sink: std::sync::Arc<dyn crate::dead_letter::DeadLetterSink>,
652 /// Optional liveness heartbeat — stores the current UTC Unix timestamp
653 /// (seconds) after each poll cycle so health probes can detect stale workers.
654 heartbeat: Option<std::sync::Arc<std::sync::atomic::AtomicI64>>,
655 /// Graceful-shutdown signal. When cancelled the worker finishes the message
656 /// it is delivering, then returns from [`OutboxWorker::run`] — see
657 /// [`OutboxWorker::with_shutdown`].
658 shutdown: Option<tokio_util::sync::CancellationToken>,
659}
660
661/// Sleep for `dur`, returning early if `token` is cancelled.
662///
663/// Returns `true` when the sleep completed and the caller should keep looping,
664/// `false` when the token was cancelled and the caller must return.
665///
666/// A worker that sleeps on a bare `tokio::time::sleep` cannot observe a
667/// shutdown until its poll interval elapses. For the deadline scheduler that is
668/// 30 seconds by default — longer than a typical container termination grace
669/// period, which turns a graceful drain into a SIGKILL.
670///
671/// Public so that binaries running their own poll-loop workers alongside the
672/// engine's (projection catch-up, webhook delivery, retention purges) can honour
673/// the same token and stop before the store is closed.
674pub async fn sleep_or_cancel(
675 dur: std::time::Duration,
676 token: Option<&tokio_util::sync::CancellationToken>,
677) -> bool {
678 let Some(t) = token else {
679 tokio::time::sleep(dur).await;
680 return true;
681 };
682 tokio::select! {
683 () = tokio::time::sleep(dur) => true,
684 () = t.cancelled() => false,
685 }
686}
687
688/// Compute a full-jitter exponential backoff delay.
689///
690/// `attempt` is the number of prior attempts (0 = first retry).
691/// `entropy` provides randomness; derive from a stable message identifier
692/// (e.g. hash of `message_id`) rather than the current timestamp — a
693/// timestamp-derived value is deterministic within a single batch, which
694/// defeats jitter when multiple messages fail simultaneously.
695///
696/// | attempt | window (s) | expected delay (s) |
697/// |---------|------------|-------------------|
698/// | 0 | 5 | 2.5 |
699/// | 1 | 10 | 5 |
700/// | 2 | 20 | 10 |
701/// | 3 | 40 | 20 |
702/// | 4 | 80 | 40 |
703/// | 5+ | 300 (cap) | 150 |
704fn backoff_delay(attempt: u32, entropy: u64) -> std::time::Duration {
705 const BASE_SECS: u64 = 5;
706 const MAX_SECS: u64 = 300;
707 // Exponential window: BASE * 2^attempt, capped at MAX.
708 let window = BASE_SECS
709 .saturating_mul(1u64.wrapping_shl(attempt.min(5)))
710 .min(MAX_SECS);
711 // Full jitter: uniform random in [0, window).
712 let jitter_secs = if window == 0 { 0 } else { entropy % window };
713 std::time::Duration::from_secs(jitter_secs)
714}
715
716impl<OS: OutboxStore, S: As4Sender, DS: DeadlineStore> OutboxWorker<OS, S, DS> {
717 /// Run the outbox drain loop until the shutdown token is cancelled.
718 ///
719 /// Without a token (see [`OutboxWorker::with_shutdown`]) the loop runs until
720 /// the task is aborted or the process exits. With one, cancellation is
721 /// observed between messages and during the idle sleep, so an in-flight
722 /// delivery is always finished and acknowledged before the worker returns —
723 /// dropping it mid-`send` would risk a duplicate AS4 delivery on restart.
724 ///
725 /// # Panics
726 ///
727 /// Panics if `time::Duration::try_from(delay)` overflows (unreachable for
728 /// the delay values produced by `backoff_delay`).
729 #[allow(clippy::too_many_lines)]
730 pub async fn run(self) {
731 loop {
732 if self
733 .shutdown
734 .as_ref()
735 .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
736 {
737 tracing::info!("outbox worker: shutdown signalled; stopping");
738 return;
739 }
740 // Tick liveness at the *start* of every poll cycle, ahead of the
741 // early-`continue` paths below. An idle worker (empty outbox) and
742 // one retrying after a store error are both alive and must keep
743 // ticking; only a worker genuinely hung inside an `.await` stops.
744 if let Some(ref hb) = self.heartbeat {
745 hb.store(
746 time::OffsetDateTime::now_utc().unix_timestamp(),
747 std::sync::atomic::Ordering::Relaxed,
748 );
749 }
750
751 let batch = match self.store.pending_now(self.batch_size).await {
752 Ok(b) => b,
753 Err(e) => {
754 tracing::warn!(error = %e, "outbox worker: store error polling pending messages (will retry)");
755 if !sleep_or_cancel(self.poll_interval, self.shutdown.as_ref()).await {
756 return;
757 }
758 continue;
759 }
760 };
761
762 if batch.is_empty() {
763 if !sleep_or_cancel(self.poll_interval, self.shutdown.as_ref()).await {
764 return;
765 }
766 continue;
767 }
768
769 for msg in batch {
770 // Between messages, not inside one: a `send` that is already in
771 // flight must run to its `acknowledge`, or the counterparty
772 // receives a message the outbox still believes is pending and
773 // redelivers it after the restart.
774 if self
775 .shutdown
776 .as_ref()
777 .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
778 {
779 tracing::info!(
780 "outbox worker: shutdown signalled mid-batch; \
781 remaining messages stay queued for the next start"
782 );
783 return;
784 }
785 // ── Retry budget ──────────────────────────────────────
786 // `attempt_count` starts at 0 and is incremented on each
787 // `reschedule` call. The message is permanently undeliverable
788 // when the retry *window* has elapsed (the BDEW duty is stated
789 // in hours, and full-jitter backoff makes a count no proxy for
790 // a duration) or the attempt belt is exhausted: acknowledge it
791 // (remove from outbox) and dead-letter it so the regulatory
792 // audit trail is preserved. The window is only consulted after
793 // a first attempt, so a message that aged while the worker was
794 // down still gets tried once.
795 let age = time::OffsetDateTime::now_utc() - msg.created_at;
796 let window_elapsed = msg.attempt_count > 0
797 && age
798 >= time::Duration::try_from(self.max_retry_window)
799 .unwrap_or(time::Duration::hours(72));
800 if msg.attempt_count >= self.max_attempts || window_elapsed {
801 tracing::error!(
802 message_id = %msg.message_id,
803 message_type = %msg.message_type,
804 recipient = %msg.recipient,
805 attempts = msg.attempt_count,
806 max_attempts = self.max_attempts,
807 age_secs = age.whole_seconds(),
808 window_secs = self.max_retry_window.as_secs(),
809 "outbox worker: retry budget exhausted; dead-lettering message",
810 );
811 self.dead_letter_sink.reject(
812 &crate::dead_letter::DeadLetterReason::OutboxExhausted {
813 message_id: msg.message_id,
814 message_type: msg.message_type.to_string(),
815 recipient: msg.recipient.to_string(),
816 last_error: format!(
817 "delivery exhausted after {} attempts",
818 msg.attempt_count
819 ),
820 attempts: msg.attempt_count,
821 },
822 );
823 if let Err(e) = self.store.acknowledge(msg.message_id).await {
824 tracing::error!(
825 message_id = %msg.message_id,
826 error = %e,
827 "outbox worker: acknowledge after exhaust failed; message may reappear",
828 );
829 }
830 continue;
831 }
832
833 match self.sender.send(&msg).await {
834 Ok(()) => {
835 if let Err(e) = self.store.acknowledge(msg.message_id).await {
836 tracing::warn!(
837 message_id = %msg.message_id,
838 error = %e,
839 "outbox worker: acknowledge failed",
840 );
841 }
842 // CONTRL AHB 1.0 §1.2: the CONTRL must be delivered
843 // within 6 wall-clock hours of interchange receipt.
844 // `msg.created_at` is when the PendingOutbox was
845 // materialised (which should equal the ingest timestamp
846 // for transport-layer CONTRL obligations).
847 if msg.message_type.as_ref() == "CONTRL" {
848 let elapsed = time::OffsetDateTime::now_utc() - msg.created_at;
849 if elapsed > time::Duration::hours(mako_fristen::CONTRL_FRIST_HOURS) {
850 tracing::warn!(
851 message_id = %msg.message_id,
852 elapsed_secs = elapsed.whole_seconds(),
853 max_secs = mako_fristen::CONTRL_FRIST_HOURS * 3600,
854 "outbox worker: CONTRL delivered OUTSIDE the 6h Übertragungsfrist \
855 (CONTRL AHB 1.0 §1.2) — this is a BNetzA compliance violation"
856 );
857 }
858 }
859 // APERAK AHB 1.0 §2.4.1: Strom UTILMD/ORDERS APERAK must be
860 // delivered within 45 minutes on weekdays, or by Sunday 12:00
861 // if received on Saturday. Log a compliance warning if the
862 // delivery window was missed so operators can investigate.
863 if msg.message_type.as_ref() == "APERAK" {
864 let elapsed = time::OffsetDateTime::now_utc() - msg.created_at;
865 if elapsed
866 > time::Duration::minutes(
867 mako_fristen::APERAK_STROM_WEEKDAY_MINUTES,
868 )
869 {
870 tracing::warn!(
871 message_id = %msg.message_id,
872 elapsed_mins = elapsed.whole_minutes(),
873 "outbox worker: APERAK delivered after the 45-minute Strom \
874 sending window (APERAK AHB 1.0 §2.4.1) — \
875 check OutboxWorker and AS4 transport health"
876 );
877 }
878 }
879 // The message is out, so any delivery window that was
880 // watching for it has been answered — retire it.
881 //
882 // Nothing else cancels these. A monitoring deadline that
883 // outlives the obligation it monitors fires for every
884 // process, including every one that answered on time,
885 // and the scheduler cannot tell those apart because a
886 // deadline reaching `due_now` is late by construction.
887 // Leaving them registered turns the miss counters into
888 // counts of *processes started*.
889 self.discharge_delivery_window(&msg).await;
890 }
891 // Permanent error: dead-letter immediately without retrying.
892 // PartnerUnknown requires operator intervention (add --as4-partner);
893 // Serialization errors will never succeed on retry; a missing
894 // wire-format renderer cannot appear between attempts — its own
895 // documentation promises immediate dead-lettering, and until this
896 // arm matched it, that promise was broken and the message burned
897 // the whole retry budget first.
898 Err(ref e)
899 if e.is_partner_unknown()
900 || e.is_renderer_not_implemented()
901 || matches!(e, EngineError::Serialization(_)) =>
902 {
903 tracing::error!(
904 message_id = %msg.message_id,
905 message_type = %msg.message_type,
906 recipient = %msg.recipient,
907 error = %e,
908 "outbox worker: permanent send failure; dead-lettering without retry",
909 );
910 self.dead_letter_sink.reject(
911 &crate::dead_letter::DeadLetterReason::OutboxExhausted {
912 message_id: msg.message_id,
913 message_type: msg.message_type.to_string(),
914 recipient: msg.recipient.to_string(),
915 last_error: e.to_string(),
916 attempts: msg.attempt_count,
917 },
918 );
919 if let Err(re) = self.store.acknowledge(msg.message_id).await {
920 tracing::error!(
921 message_id = %msg.message_id,
922 error = %re,
923 "outbox worker: acknowledge after permanent failure failed",
924 );
925 }
926 }
927 Err(e) => {
928 // Stable jitter entropy derived from the UUID bytes of
929 // `message_id`. Using the last 8 bytes as a `u64` gives
930 // uniform entropy across message IDs (UUIDs are random in
931 // all 128 bits for v4) and is stable across Rust versions —
932 // unlike `DefaultHasher`, whose algorithm is explicitly
933 // documented as unstable.
934 let entropy = {
935 let uuid = msg.message_id.as_uuid();
936 let bytes = uuid.as_bytes();
937 u64::from_le_bytes(bytes[8..16].try_into().unwrap())
938 };
939 let delay = backoff_delay(msg.attempt_count, entropy);
940 let retry_at = time::OffsetDateTime::now_utc()
941 + time::Duration::try_from(delay).unwrap_or(time::Duration::minutes(5));
942 tracing::warn!(
943 message_id = %msg.message_id,
944 attempt = msg.attempt_count,
945 max_attempts = self.max_attempts,
946 retry_in = ?delay,
947 error = %e,
948 "outbox worker: send failed; rescheduling with backoff",
949 );
950 if let Err(re) = self.store.reschedule(msg.message_id, retry_at).await {
951 tracing::error!(
952 message_id = %msg.message_id,
953 error = %re,
954 "outbox worker: reschedule failed; message may be stuck",
955 );
956 }
957 }
958 }
959 }
960 }
961 }
962}
963
964impl<ES, SS, OS, DS, PR> EngineContext<ES, SS, OS, DS, PR>
965where
966 ES: EventStore,
967 OS: OutboxStore + Clone,
968{
969 /// Construct an [`OutboxWorker`] that drains the outbox via `sender`.
970 ///
971 /// `batch_size` — messages fetched per poll cycle.
972 /// `poll_interval` — sleep duration when the batch is empty.
973 ///
974 /// `max_attempts` — attempt belt against runaway loops; the real budget is
975 /// `max_retry_window`, the message age after which delivery is abandoned.
976 /// The BDEW AS4 retry duty is stated in *hours* (72 h for unacknowledged
977 /// messages — `mako_as4::constants::MAX_RETRY_DURATION_SECS`), and the
978 /// full-jitter backoff makes an attempt count no proxy for a duration, so
979 /// both are taken and either exhausts the message.
980 ///
981 /// ```rust,ignore
982 /// use std::time::Duration;
983 ///
984 /// let worker = ctx.run_outbox_worker(
985 /// my_sender, 50, Duration::from_secs(1),
986 /// 10_000, Duration::from_secs(72 * 3600),
987 /// );
988 /// tokio::spawn(async move { worker.run().await });
989 /// ```
990 #[must_use]
991 pub fn run_outbox_worker<S: As4Sender>(
992 &self,
993 sender: S,
994 batch_size: usize,
995 poll_interval: std::time::Duration,
996 max_attempts: u32,
997 max_retry_window: std::time::Duration,
998 ) -> OutboxWorker<OS, S, DS>
999 where
1000 DS: DeadlineStore + Clone,
1001 {
1002 OutboxWorker {
1003 store: self.outbox_store.clone(),
1004 sender,
1005 deadline_store: self.deadline_store.clone(),
1006 batch_size,
1007 poll_interval,
1008 max_attempts,
1009 max_retry_window,
1010 dead_letter_sink: self.dead_letter_sink.clone(),
1011 heartbeat: None,
1012 shutdown: None,
1013 }
1014 }
1015}
1016
1017impl<OS: OutboxStore, S: As4Sender, DS: DeadlineStore> OutboxWorker<OS, S, DS> {
1018 /// Attach a liveness heartbeat to this worker.
1019 ///
1020 /// The worker will store the current UTC Unix timestamp (seconds) into
1021 /// `heartbeat` at the end of every poll cycle. Pass the same
1022 /// `Arc<AtomicI64>` to the health endpoint so it can detect stale workers.
1023 #[must_use]
1024 pub fn with_heartbeat(
1025 mut self,
1026 heartbeat: std::sync::Arc<std::sync::atomic::AtomicI64>,
1027 ) -> Self {
1028 self.heartbeat = Some(heartbeat);
1029 self
1030 }
1031
1032 /// Attach a graceful-shutdown token.
1033 ///
1034 /// Cancelling it makes [`OutboxWorker::run`] return at the next message
1035 /// boundary or immediately out of its idle sleep. Await the worker's
1036 /// `JoinHandle` afterwards: the point of the token is that the caller can
1037 /// close the event store *after* the worker has stopped writing to it.
1038 #[must_use]
1039 pub fn with_shutdown(mut self, shutdown: tokio_util::sync::CancellationToken) -> Self {
1040 self.shutdown = Some(shutdown);
1041 self
1042 }
1043
1044 /// Retire the delivery window `msg` was being watched by, if one is open.
1045 ///
1046 /// A delivery-window deadline exists to answer one question: *did this
1047 /// message go out in time?* Once it has gone out the question is settled,
1048 /// and leaving the deadline registered only guarantees a false alarm later.
1049 /// [`fristen::discharges_delivery_window`] decides which labels a given
1050 /// message type answers for; deadlines that merely share the stream (a
1051 /// process-response window, say) are left alone.
1052 ///
1053 /// Best-effort: a failure here costs a spurious alert at the window's close,
1054 /// never a lost or duplicated message, so it is logged rather than
1055 /// propagated — the delivery itself has already been acknowledged.
1056 ///
1057 /// [`fristen::discharges_delivery_window`]: mako_fristen::discharges_delivery_window
1058 async fn discharge_delivery_window(&self, msg: &crate::outbox::OutboxMessage) {
1059 let open = match self.deadline_store.for_stream(&msg.stream_id).await {
1060 Ok(deadlines) => deadlines,
1061 Err(e) => {
1062 tracing::warn!(
1063 message_id = %msg.message_id,
1064 message_type = %msg.message_type,
1065 error = %e,
1066 "outbox worker: could not read deadlines to discharge the delivery \
1067 window; it may fire a spurious regulatory alert",
1068 );
1069 return;
1070 }
1071 };
1072
1073 for deadline in open
1074 .iter()
1075 .filter(|d| mako_fristen::discharges_delivery_window(&msg.message_type, d.label()))
1076 {
1077 if let Err(e) = self.deadline_store.cancel(deadline.deadline_id()).await {
1078 tracing::warn!(
1079 message_id = %msg.message_id,
1080 deadline_id = %deadline.deadline_id(),
1081 label = %deadline.label(),
1082 error = %e,
1083 "outbox worker: could not discharge the delivery window; \
1084 it may fire a spurious regulatory alert",
1085 );
1086 } else {
1087 tracing::debug!(
1088 message_id = %msg.message_id,
1089 message_type = %msg.message_type,
1090 deadline_id = %deadline.deadline_id(),
1091 label = %deadline.label(),
1092 "outbox worker: message delivered — delivery window discharged",
1093 );
1094 }
1095 }
1096 }
1097}
1098
1099impl<ES, SS, OS, DS, PR> std::fmt::Debug for EngineContext<ES, SS, OS, DS, PR>
1100where
1101 ES: std::fmt::Debug,
1102 SS: std::fmt::Debug,
1103 OS: std::fmt::Debug,
1104 DS: std::fmt::Debug,
1105 PR: std::fmt::Debug,
1106{
1107 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1108 f.debug_struct("EngineContext")
1109 .field("registered_modules", &self.registered_modules)
1110 .field("registered_workflows", &self.registered_workflows)
1111 .field("pid_router_len", &self.pid_router.len())
1112 .finish_non_exhaustive()
1113 }
1114}
1115
1116// ── NoopAs4Sender / LogAs4Sender ──────────────────────────────────────────────
1117
1118/// An [`As4Sender`] that succeeds immediately without sending anything.
1119///
1120/// Use in tests and environments where outbound AS4 delivery is not yet
1121/// wired. All outbox messages are acknowledged (removed from the queue)
1122/// without being transmitted.
1123///
1124/// # ⚠️ Data loss warning
1125///
1126/// Every outbox message is **silently discarded** — no EDIFACT message is
1127/// sent to any counterparty. Do not use in production.
1128#[derive(Debug, Clone, Copy, Default)]
1129#[must_use = "NoopAs4Sender discards all outbound messages silently — use a real AS4 gateway in production"]
1130#[cfg_attr(
1131 not(any(test, feature = "testing")),
1132 deprecated = "NoopAs4Sender must not be wired in production builds; every \
1133 outbound EDIFACT message would be silently discarded. Use \
1134 a real As4Sender implementation instead."
1135)]
1136pub struct NoopAs4Sender;
1137
1138// The trait impl is test/testing-only: a release build without the `testing`
1139// feature cannot wire NoopAs4Sender into an outbox worker at all.
1140#[cfg(any(test, feature = "testing"))]
1141impl As4Sender for NoopAs4Sender {
1142 async fn send(&self, _msg: &OutboxMessage) -> Result<(), EngineError> {
1143 Ok(())
1144 }
1145}
1146
1147/// An [`As4Sender`] that logs every outbound message at `warn` level and
1148/// succeeds without transmitting.
1149///
1150/// Useful for development and integration-testing environments where the
1151/// full AS4 stack is not yet available but message visibility is desired.
1152/// All outbox messages are acknowledged (removed from the queue) after logging.
1153///
1154/// # ⚠️ Data loss warning
1155///
1156/// No EDIFACT message is sent to any counterparty. Do not use in production.
1157#[derive(Debug, Clone, Copy, Default)]
1158#[must_use = "LogAs4Sender discards all outbound messages — use a real AS4 gateway in production"]
1159pub struct LogAs4Sender;
1160
1161impl As4Sender for LogAs4Sender {
1162 async fn send(&self, msg: &OutboxMessage) -> Result<(), EngineError> {
1163 tracing::warn!(
1164 message_id = %msg.message_id,
1165 message_type = %msg.message_type,
1166 recipient = %msg.recipient,
1167 "LogAs4Sender: outbox message dropped — configure a real AS4 gateway for production",
1168 );
1169 Ok(())
1170 }
1171}
1172
1173// ── DeadlineScheduler ─────────────────────────────────────────────────────────
1174
1175/// A background task that polls [`DeadlineStore::due_now`] and dispatches
1176/// deadline commands to the owning processes via a caller-supplied function.
1177///
1178/// Obtain via [`EngineContext::run_deadline_scheduler`] and drive by spawning
1179/// [`DeadlineScheduler::run`] in a Tokio task.
1180///
1181/// # Dispatch function
1182///
1183/// The `dispatch` function receives a fired [`Deadline`] and returns a future
1184/// that dispatches the appropriate timeout command to the process. The function
1185/// is responsible for resuming the correct workflow and calling `execute`.
1186/// After the future completes, the scheduler cancels the deadline from the
1187/// store regardless of the dispatch outcome (to prevent re-firing).
1188///
1189/// ```rust,ignore
1190/// use std::time::Duration;
1191///
1192/// let scheduler = ctx.run_deadline_scheduler(
1193/// |deadline| async move {
1194/// tracing::warn!(
1195/// deadline_id = %deadline.deadline_id(),
1196/// label = %deadline.label(),
1197/// "deadline fired",
1198/// );
1199/// Ok(())
1200/// },
1201/// 100,
1202/// Duration::from_secs(30),
1203/// );
1204/// tokio::spawn(async move { scheduler.run().await });
1205/// ```
1206pub struct DeadlineScheduler<DS: DeadlineStore> {
1207 store: DS,
1208 dispatch: Box<
1209 dyn Fn(
1210 Deadline,
1211 ) -> std::pin::Pin<
1212 Box<dyn std::future::Future<Output = Result<(), EngineError>> + Send>,
1213 > + Send
1214 + Sync,
1215 >,
1216 batch_size: usize,
1217 poll_interval: std::time::Duration,
1218 /// Optional liveness heartbeat — stores the current UTC Unix timestamp
1219 /// (seconds) after each poll cycle.
1220 heartbeat: Option<std::sync::Arc<std::sync::atomic::AtomicI64>>,
1221 /// Graceful-shutdown signal — see [`DeadlineScheduler::with_shutdown`].
1222 shutdown: Option<tokio_util::sync::CancellationToken>,
1223}
1224
1225impl<DS: DeadlineStore> DeadlineScheduler<DS> {
1226 /// Run the deadline poll loop until the shutdown token is cancelled.
1227 ///
1228 /// Cancellation is observed between deadlines and during the idle sleep, so
1229 /// a deadline already being dispatched runs to completion. A deadline left
1230 /// undispatched stays registered and fires on the next start — it is due, so
1231 /// the next `due_now` returns it again.
1232 pub async fn run(self) {
1233 loop {
1234 if self
1235 .shutdown
1236 .as_ref()
1237 .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
1238 {
1239 tracing::info!("deadline scheduler: shutdown signalled; stopping");
1240 return;
1241 }
1242 // Tick liveness at the *start* of every poll cycle, ahead of the
1243 // early-`continue` paths below. An idle scheduler (no due
1244 // deadlines) is alive and must keep ticking; only one genuinely
1245 // hung inside an `.await` stops.
1246 if let Some(ref hb) = self.heartbeat {
1247 hb.store(
1248 time::OffsetDateTime::now_utc().unix_timestamp(),
1249 std::sync::atomic::Ordering::Relaxed,
1250 );
1251 }
1252
1253 let result = match self.store.due_now(self.batch_size).await {
1254 Ok(r) => r,
1255 Err(e) => {
1256 tracing::warn!(
1257 error = %e,
1258 "deadline scheduler: store error polling due deadlines (will retry)",
1259 );
1260 if !sleep_or_cancel(self.poll_interval, self.shutdown.as_ref()).await {
1261 return;
1262 }
1263 continue;
1264 }
1265 };
1266
1267 if result.deadlines.is_empty() {
1268 if !sleep_or_cancel(self.poll_interval, self.shutdown.as_ref()).await {
1269 return;
1270 }
1271 continue;
1272 }
1273
1274 for deadline in result.deadlines {
1275 // Between deadlines, not inside one: a dispatch already running
1276 // must finish so its events and outbox entries commit together.
1277 if self
1278 .shutdown
1279 .as_ref()
1280 .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
1281 {
1282 tracing::info!(
1283 "deadline scheduler: shutdown signalled mid-batch; \
1284 undispatched deadlines remain due and fire on the next start"
1285 );
1286 return;
1287 }
1288 let id = deadline.deadline_id();
1289 let label = deadline.label().to_owned();
1290
1291 // An APERAK delivery window that reaches this point is a
1292 // regulatory violation under APERAK AHB 1.0 §2.4.1 (Strom
1293 // 45 min) / §2.3.1 (Gas 1 Werktag): the outbox worker discharges
1294 // the window the moment the APERAK goes out, so one that is
1295 // still registered when it comes due was never answered.
1296 //
1297 // Do NOT re-test `now > due_at` here. `due_now` selects on
1298 // `due_at <= now`, so that comparison is true by construction and
1299 // says nothing about compliance — it was the reason this counter
1300 // once tracked "Strom processes started" rather than "APERAKs
1301 // missed". The discharge is what carries the meaning.
1302 if label.starts_with(mako_fristen::APERAK_WINDOW_LABEL_PREFIX) {
1303 let now = time::OffsetDateTime::now_utc();
1304 crate::metrics::EngineMetrics::global().aperak_missed(&label);
1305 tracing::error!(
1306 deadline_id = %id,
1307 label = %label,
1308 due_at = %deadline.due_at(),
1309 fired_at = %now,
1310 overdue_secs = (now - deadline.due_at()).whole_seconds(),
1311 "APERAK delivery window closed with no delivery — regulatory \
1312 violation (APERAK AHB 1.0 §2.4.1 Strom / §2.3.1 Gas). \
1313 Counter: makod_aperak_missed_total",
1314 );
1315 }
1316
1317 let should_cancel = match (self.dispatch)(deadline).await {
1318 Ok(()) => true,
1319 Err(ref e) if e.is_version_conflict() => {
1320 // The process was modified concurrently; the timeout
1321 // command will be retried on the next poll cycle.
1322 // Do NOT cancel — let the deadline remain due so it
1323 // fires again until a non-conflict dispatch succeeds.
1324 tracing::warn!(
1325 deadline_id = %id,
1326 label = %label,
1327 "deadline scheduler: VersionConflict; will retry on next poll",
1328 );
1329 false
1330 }
1331 Err(e) => {
1332 tracing::warn!(
1333 deadline_id = %id,
1334 label = %label,
1335 error = %e,
1336 "deadline scheduler: dispatch failed (permanent); cancelling",
1337 );
1338 true
1339 }
1340 };
1341 if should_cancel && let Err(e) = self.store.cancel(id).await {
1342 tracing::error!(
1343 deadline_id = %id,
1344 error = %e,
1345 "deadline scheduler: cancel failed; deadline may fire again",
1346 );
1347 }
1348 }
1349
1350 // If has_more, loop immediately to drain the batch.
1351 }
1352 }
1353}
1354
1355impl<DS: DeadlineStore> DeadlineScheduler<DS> {
1356 /// Attach a liveness heartbeat to this scheduler.
1357 ///
1358 /// The scheduler will store the current UTC Unix timestamp (seconds) into
1359 /// `heartbeat` at the end of every poll cycle.
1360 #[must_use]
1361 pub fn with_heartbeat(
1362 mut self,
1363 heartbeat: std::sync::Arc<std::sync::atomic::AtomicI64>,
1364 ) -> Self {
1365 self.heartbeat = Some(heartbeat);
1366 self
1367 }
1368
1369 /// Attach a graceful-shutdown token.
1370 ///
1371 /// Cancelling it makes [`DeadlineScheduler::run`] return at the next
1372 /// deadline boundary or immediately out of its idle sleep, so the caller can
1373 /// close the event store once the scheduler has stopped writing to it.
1374 #[must_use]
1375 pub fn with_shutdown(mut self, shutdown: tokio_util::sync::CancellationToken) -> Self {
1376 self.shutdown = Some(shutdown);
1377 self
1378 }
1379}
1380
1381impl<ES, SS, OS, DS, PR> EngineContext<ES, SS, OS, DS, PR>
1382where
1383 ES: EventStore,
1384 DS: DeadlineStore + Clone,
1385{
1386 /// Construct a [`DeadlineScheduler`] that polls the deadline store and
1387 /// dispatches fired deadlines via `dispatch`.
1388 ///
1389 /// The `dispatch` function is called for every fired deadline. It should
1390 /// resume the owning process and execute the appropriate timeout command.
1391 ///
1392 /// `batch_size` — deadlines fetched per poll cycle.
1393 /// `poll_interval` — sleep duration when no deadlines are due.
1394 ///
1395 /// ```rust,ignore
1396 /// use std::time::Duration;
1397 ///
1398 /// let scheduler = ctx.run_deadline_scheduler(
1399 /// |d| async move {
1400 /// tracing::info!(label = %d.label(), "firing deadline");
1401 /// Ok(())
1402 /// },
1403 /// 100,
1404 /// Duration::from_secs(30),
1405 /// );
1406 /// tokio::spawn(async move { scheduler.run().await });
1407 /// ```
1408 #[must_use]
1409 pub fn run_deadline_scheduler<F, Fut>(
1410 &self,
1411 dispatch: F,
1412 batch_size: usize,
1413 poll_interval: std::time::Duration,
1414 ) -> DeadlineScheduler<DS>
1415 where
1416 F: Fn(Deadline) -> Fut + Send + Sync + 'static,
1417 Fut: std::future::Future<Output = Result<(), EngineError>> + Send + 'static,
1418 {
1419 DeadlineScheduler {
1420 store: self.deadline_store.clone(),
1421 dispatch: Box::new(move |d| Box::pin(dispatch(d))),
1422 batch_size,
1423 poll_interval,
1424 heartbeat: None,
1425 shutdown: None,
1426 }
1427 }
1428}
1429
1430// ── EngineBuilder ─────────────────────────────────────────────────────────────
1431
1432/// Assembles engine infrastructure and produces an [`EngineContext`].
1433///
1434/// Uses type-state to enforce that an event store is provided before
1435/// [`build`] can be called. All other stores default to `Noop`
1436/// implementations.
1437///
1438/// ## Quick start
1439///
1440/// ```rust,ignore
1441/// // Minimal — event store only, all others are Noop:
1442/// let ctx = EngineBuilder::new()
1443/// .with_event_store(InMemoryEventStore::new())
1444/// .build();
1445///
1446/// // Full infrastructure:
1447/// let ctx = EngineBuilder::new()
1448/// .with_event_store(InMemoryEventStore::new())
1449/// .with_snapshot_store(InMemorySnapshotStore::new())
1450/// .with_outbox_store(InMemoryOutboxStore::new())
1451/// .with_deadline_store(InMemoryDeadlineStore::new())
1452/// .with_registry(InMemoryProcessRegistry::new())
1453/// .register(Box::new(GpkeModule))
1454/// .build();
1455/// ```
1456///
1457/// [`build`]: EngineBuilder::build
1458pub struct EngineBuilder<
1459 ES = (),
1460 SS = NoopSnapshotStore,
1461 OS = NoopOutboxStore,
1462 DS = NoopDeadlineStore,
1463 PR = NoopProcessRegistry,
1464> {
1465 event_store: ES,
1466 snapshot_store: SS,
1467 outbox_store: OS,
1468 deadline_store: DS,
1469 registry: PR,
1470 dead_letter_sink: Arc<dyn DeadLetterSink>,
1471 modules: Vec<Box<dyn EngineModule>>,
1472 /// Active [`DeploymentRoles`] for this engine instance.
1473 ///
1474 /// Controls role-conditional PID registration via
1475 /// [`EngineModule::register_pids_with_roles`]. Defaults to
1476 /// [`DeploymentRoles::all()`]: an engine that names no roles registers
1477 /// every PID its modules declare, which is what a test harness and a
1478 /// combined-role deployment both want.
1479 deployment_roles: DeploymentRoles,
1480 /// Optional profile validator injected by `makod` or callers that have
1481 /// access to `edi-energy`. When `Some`, called for each
1482 /// [`ProfileRequirement`] declared by registered modules. When `None`,
1483 /// profile requirements are not validated (safe in unit tests).
1484 ///
1485 /// Signature: `fn(message_type: &str) -> bool`
1486 ///
1487 /// [`ProfileRequirement`]: crate::profile::ProfileRequirement
1488 profile_validator: Option<Box<dyn Fn(&str) -> bool + Send + Sync>>,
1489}
1490#[cfg(any(test, feature = "testing"))]
1491impl Default
1492 for EngineBuilder<
1493 (),
1494 NoopSnapshotStore,
1495 NoopOutboxStore,
1496 NoopDeadlineStore,
1497 NoopProcessRegistry,
1498 >
1499{
1500 fn default() -> Self {
1501 Self {
1502 event_store: (),
1503 snapshot_store: NoopSnapshotStore,
1504 outbox_store: NoopOutboxStore,
1505 deadline_store: NoopDeadlineStore,
1506 registry: NoopProcessRegistry,
1507 dead_letter_sink: Arc::new(LogDeadLetterSink),
1508 modules: Vec::new(),
1509 deployment_roles: DeploymentRoles::all(),
1510 profile_validator: None,
1511 }
1512 }
1513}
1514
1515#[cfg(any(test, feature = "testing"))]
1516impl EngineBuilder {
1517 /// Create a new builder with all `Noop` defaults.
1518 ///
1519 /// Only available in `#[cfg(test)]` or with the `testing` feature enabled,
1520 /// because the Noop defaults silently discard outbox messages, deadlines,
1521 /// and process registry entries. Production binaries must wire real stores
1522 /// via the `with_*` builder methods.
1523 ///
1524 /// Call [`with_event_store`] before [`build`] — the event store is
1525 /// **required**.
1526 ///
1527 /// [`with_event_store`]: EngineBuilder::with_event_store
1528 /// [`build`]: EngineBuilder::build
1529 #[must_use]
1530 pub fn new() -> Self {
1531 Self::default()
1532 }
1533}
1534
1535impl<OS, DS, PR> EngineBuilder<(), NoopSnapshotStore, OS, DS, PR>
1536where
1537 OS: OutboxStore,
1538 DS: DeadlineStore,
1539 PR: ProcessRegistry,
1540{
1541 /// Create a production-ready builder with explicit stores for outbox,
1542 /// deadline, and process registry.
1543 ///
1544 /// This constructor is available in all build configurations including
1545 /// production binaries. It enforces that the three stores that can cause
1546 /// silent data loss (`OutboxStore`, `DeadlineStore`, `ProcessRegistry`)
1547 /// are provided explicitly — there is no Noop fallback.
1548 ///
1549 /// `NoopSnapshotStore` is used as the snapshot default because it is safe
1550 /// for production: skipping snapshots means full replay, but no data loss.
1551 /// Override with [`with_snapshot_store`] to enable snapshot-accelerated
1552 /// replay.
1553 ///
1554 /// Call [`with_event_store`] before [`build`] — the event store is
1555 /// **required**.
1556 ///
1557 /// ```rust,ignore
1558 /// let ctx = EngineBuilder::with_stores(outbox, deadline, registry)
1559 /// .with_event_store(store.clone())
1560 /// .with_snapshot_store(InMemorySnapshotStore::new())
1561 /// .build();
1562 /// ```
1563 ///
1564 /// [`with_snapshot_store`]: EngineBuilder::with_snapshot_store
1565 /// [`with_event_store`]: EngineBuilder::with_event_store
1566 /// [`build`]: EngineBuilder::build
1567 #[must_use]
1568 pub fn with_stores(outbox_store: OS, deadline_store: DS, registry: PR) -> Self {
1569 Self {
1570 event_store: (),
1571 snapshot_store: NoopSnapshotStore,
1572 outbox_store,
1573 deadline_store,
1574 registry,
1575 dead_letter_sink: Arc::new(LogDeadLetterSink),
1576 modules: Vec::new(),
1577 deployment_roles: DeploymentRoles::all(),
1578 profile_validator: None,
1579 }
1580 }
1581}
1582
1583impl<ES, SS, OS, DS, PR> EngineBuilder<ES, SS, OS, DS, PR> {
1584 /// Set the event store. **Required** — `build()` is only available once
1585 /// this has been called with a type that implements [`EventStore`].
1586 ///
1587 /// Replaces any previously set event store (type-state transition).
1588 #[must_use]
1589 pub fn with_event_store<ES2: EventStore>(
1590 self,
1591 store: ES2,
1592 ) -> EngineBuilder<ES2, SS, OS, DS, PR> {
1593 EngineBuilder {
1594 event_store: store,
1595 snapshot_store: self.snapshot_store,
1596 outbox_store: self.outbox_store,
1597 deadline_store: self.deadline_store,
1598 registry: self.registry,
1599 dead_letter_sink: self.dead_letter_sink,
1600 modules: self.modules,
1601 deployment_roles: self.deployment_roles,
1602 profile_validator: self.profile_validator,
1603 }
1604 }
1605
1606 /// Set the snapshot store (default: [`NoopSnapshotStore`]).
1607 ///
1608 /// ## Default: `NoopSnapshotStore`
1609 ///
1610 /// Without calling this method the builder uses [`NoopSnapshotStore`],
1611 /// which silently discards all snapshot writes and returns `None` for
1612 /// every snapshot read. The engine still functions correctly — every
1613 /// command handling call replays the full event log from the beginning
1614 /// instead of starting from a stored snapshot. For low-volume processes
1615 /// this is fine; for long-lived processes with many events the replay cost
1616 /// can become significant.
1617 ///
1618 /// Enable snapshotting in production by providing a real [`SnapshotStore`]
1619 /// implementation (e.g. the SlateDB-backed store in `makod`). In tests,
1620 /// `InMemorySnapshotStore` is available behind the `testing` feature flag.
1621 ///
1622 /// Note: [`Process::state_with_snapshot`][crate::process::Process::state_with_snapshot]
1623 /// is a compile-time no-op when the snapshot store is `NoopSnapshotStore`
1624 /// — it never calls the store and always returns `None`, so no snapshot is
1625 /// ever saved or loaded.
1626 #[must_use]
1627 pub fn with_snapshot_store<SS2: SnapshotStore>(
1628 self,
1629 store: SS2,
1630 ) -> EngineBuilder<ES, SS2, OS, DS, PR> {
1631 EngineBuilder {
1632 event_store: self.event_store,
1633 snapshot_store: store,
1634 outbox_store: self.outbox_store,
1635 deadline_store: self.deadline_store,
1636 registry: self.registry,
1637 dead_letter_sink: self.dead_letter_sink,
1638 modules: self.modules,
1639 deployment_roles: self.deployment_roles,
1640 profile_validator: self.profile_validator,
1641 }
1642 }
1643
1644 /// Set the outbox store (default: [`NoopOutboxStore`]).
1645 #[must_use]
1646 pub fn with_outbox_store<OS2: OutboxStore>(
1647 self,
1648 store: OS2,
1649 ) -> EngineBuilder<ES, SS, OS2, DS, PR> {
1650 EngineBuilder {
1651 event_store: self.event_store,
1652 snapshot_store: self.snapshot_store,
1653 outbox_store: store,
1654 deadline_store: self.deadline_store,
1655 registry: self.registry,
1656 dead_letter_sink: self.dead_letter_sink,
1657 modules: self.modules,
1658 deployment_roles: self.deployment_roles,
1659 profile_validator: self.profile_validator,
1660 }
1661 }
1662
1663 /// Set the deadline store (default: [`NoopDeadlineStore`]).
1664 #[must_use]
1665 pub fn with_deadline_store<DS2: DeadlineStore>(
1666 self,
1667 store: DS2,
1668 ) -> EngineBuilder<ES, SS, OS, DS2, PR> {
1669 EngineBuilder {
1670 event_store: self.event_store,
1671 snapshot_store: self.snapshot_store,
1672 outbox_store: self.outbox_store,
1673 deadline_store: store,
1674 registry: self.registry,
1675 dead_letter_sink: self.dead_letter_sink,
1676 modules: self.modules,
1677 deployment_roles: self.deployment_roles,
1678 profile_validator: self.profile_validator,
1679 }
1680 }
1681
1682 /// Set the process registry (default: [`NoopProcessRegistry`]).
1683 #[must_use]
1684 pub fn with_registry<PR2: ProcessRegistry>(
1685 self,
1686 registry: PR2,
1687 ) -> EngineBuilder<ES, SS, OS, DS, PR2> {
1688 EngineBuilder {
1689 event_store: self.event_store,
1690 snapshot_store: self.snapshot_store,
1691 outbox_store: self.outbox_store,
1692 deadline_store: self.deadline_store,
1693 registry,
1694 dead_letter_sink: self.dead_letter_sink,
1695 modules: self.modules,
1696 deployment_roles: self.deployment_roles,
1697 profile_validator: self.profile_validator,
1698 }
1699 }
1700
1701 /// Set the dead-letter sink (default: [`LogDeadLetterSink`]).
1702 ///
1703 /// The dead-letter sink receives every message that cannot be routed to a
1704 /// workflow. The default [`LogDeadLetterSink`] emits `tracing::warn!`
1705 /// events, making rejections visible in log output without configuration.
1706 ///
1707 /// Override with a persistent DLQ implementation in production:
1708 ///
1709 /// ```rust,ignore
1710 /// use mako_engine::dead_letter::LogDeadLetterSink;
1711 ///
1712 /// let ctx = EngineBuilder::new()
1713 /// .with_event_store(my_store)
1714 /// .with_dead_letter_sink(MyPersistentDlq::new())
1715 /// .build();
1716 /// ```
1717 ///
1718 /// [`LogDeadLetterSink`]: crate::dead_letter::LogDeadLetterSink
1719 #[must_use]
1720 pub fn with_dead_letter_sink(mut self, sink: impl DeadLetterSink) -> Self {
1721 self.dead_letter_sink = Arc::new(sink);
1722 self
1723 }
1724
1725 /// Register an `edi-energy` profile validator for startup profile checks.
1726 ///
1727 /// The closure receives a message-type string (e.g. `"UTILMD"`) and must
1728 /// return `true` if at least one active profile for that message type is
1729 /// registered for today's date.
1730 ///
1731 /// Wire this in `makod` using the `edi-energy` global registry:
1732 ///
1733 /// ```rust,ignore
1734 /// use edi_energy::registry::ReleaseRegistry;
1735 ///
1736 /// let today = mako_fristen::heute();
1737 /// builder.with_profile_validator(move |msg_type| {
1738 /// ReleaseRegistry::global()
1739 /// .profiles_for_str(msg_type)
1740 /// .any(|p| match (p.valid_from(), p.valid_until()) {
1741 /// (Some(f), Some(u)) => f <= today && today <= u,
1742 /// (Some(f), None) => f <= today,
1743 /// (None, _) => true,
1744 /// })
1745 /// })
1746 /// ```
1747 ///
1748 /// Domain crates do **not** need to call this — they only declare
1749 /// [`profile_requirements`].
1750 ///
1751 /// [`profile_requirements`]: EngineModule::profile_requirements
1752 #[must_use]
1753 pub fn with_profile_validator(
1754 mut self,
1755 validator: impl Fn(&str) -> bool + Send + Sync + 'static,
1756 ) -> Self {
1757 self.profile_validator = Some(Box::new(validator));
1758 self
1759 }
1760
1761 /// Register a domain module.
1762 ///
1763 /// The module name becomes visible in
1764 /// [`EngineContext::registered_modules`] after [`build`] is called.
1765 ///
1766 /// [`build`]: EngineBuilder::build
1767 #[must_use]
1768 pub fn register(mut self, module: Box<dyn EngineModule>) -> Self {
1769 self.modules.push(module);
1770 self
1771 }
1772
1773 /// Register multiple [`EngineModule`]s at once from a pre-built `Vec`.
1774 ///
1775 /// Equivalent to calling [`register`] in a loop. Useful when the set of
1776 /// modules is assembled conditionally (e.g. via `#[cfg]`-gated pushes to a
1777 /// `Vec<Box<dyn EngineModule>>`) before the builder chain starts.
1778 ///
1779 /// [`register`]: EngineBuilder::register
1780 #[must_use]
1781 pub fn register_many(mut self, modules: Vec<Box<dyn EngineModule>>) -> Self {
1782 self.modules.extend(modules);
1783 self
1784 }
1785
1786 /// Set the active [`DeploymentRoles`] for this engine instance.
1787 ///
1788 /// Controls role-conditional PID registration in [`EngineModule::register_pids_with_roles`].
1789 ///
1790 /// The default is [`DeploymentRoles::all()`], which registers every PID unconditionally
1791 /// — identical to the pre-role-aware behavior. Providing an explicit role set
1792 /// restricts role-conditional blocks to only the declared roles:
1793 ///
1794 /// - **NB-only** (`DeploymentRoles::nb()`): 19001/19002 route to `gpke-konfiguration`;
1795 /// WiM nMSB blocks are skipped.
1796 /// - **nMSB-only** (`DeploymentRoles::nmsb()`): 19001/19002 route to `wim-geraeteubernahme`;
1797 /// GPKE NB blocks are skipped.
1798 /// - **NB + gMSB** (`DeploymentRoles::nb_msb()`): most common Stadtwerke combination.
1799 ///
1800 /// # Conflict guard
1801 ///
1802 /// When two modules would register the same PID to **different** workflows, the
1803 /// engine panics during [`build`]. Set explicit roles to prevent both modules from
1804 /// activating the same PID simultaneously:
1805 ///
1806 /// ```rust,ignore
1807 /// use mako_engine::marktrolle::DeploymentRoles;
1808 ///
1809 /// let ctx = EngineBuilder::with_stores(outbox, deadline, registry)
1810 /// .with_event_store(store)
1811 /// .with_deployment_roles(DeploymentRoles::nb()) // only NB: GPKE gets 19001/19002
1812 /// .register(Box::new(GpkeModule))
1813 /// .register(Box::new(WimModule)) // nMSB block skipped — no conflict
1814 /// .build();
1815 /// ```
1816 ///
1817 /// [`build`]: EngineBuilder::build
1818 #[must_use]
1819 pub fn with_deployment_roles(mut self, roles: DeploymentRoles) -> Self {
1820 self.deployment_roles = roles;
1821 self
1822 }
1823}
1824
1825impl<ES, SS, OS, DS, PR> EngineBuilder<ES, SS, OS, DS, PR>
1826where
1827 ES: EventStore,
1828 SS: SnapshotStore,
1829 OS: OutboxStore,
1830 DS: DeadlineStore,
1831 PR: ProcessRegistry,
1832{
1833 /// Build the [`EngineContext`].
1834 ///
1835 /// Consumes the builder. All registered modules and configured stores are
1836 /// moved into the returned [`EngineContext`].
1837 ///
1838 /// This method is only available when `ES` implements [`EventStore`].
1839 /// If you have not called [`with_event_store`], this will not compile.
1840 ///
1841 /// # Panics
1842 ///
1843 /// Panics when any registered module returns `Err` from
1844 /// [`EngineModule::configure`]. The panic message includes the module
1845 /// name and the error string so the deployment failure is actionable.
1846 ///
1847 /// [`with_event_store`]: EngineBuilder::with_event_store
1848 #[must_use]
1849 #[allow(clippy::too_many_lines)]
1850 pub fn build(self) -> EngineContext<ES, SS, OS, DS, PR> {
1851 // ── Noop store safety checks ──────────────────────────────────────────
1852 //
1853 // Noop stores lose data silently: NoopDeadlineStore drops every APERAK
1854 // deadline (BNetzA violation), NoopOutboxStore discards all outbound
1855 // messages, NoopProcessRegistry loses conversation routing on restart.
1856 //
1857 // In production builds (no `testing` feature, not running under
1858 // `#[test]`), the Noop constructors are cfg-gated out so this branch
1859 // is dead code and compiles away. In test/testing/tracing builds we
1860 // emit warnings so test harnesses see the configuration in log output.
1861 //
1862 // IMPORTANT: if you are reading this because a panic fired in production,
1863 // it means the `testing` feature was accidentally enabled in the binary.
1864 // Remove it from the production Cargo.toml feature list immediately.
1865 {
1866 let os_name = std::any::type_name::<OS>();
1867 let ds_name = std::any::type_name::<DS>();
1868 let pr_name = std::any::type_name::<PR>();
1869
1870 // Regulatory-critical stores: panic in any build context if these
1871 // are noop. OutboxStore and DeadlineStore must be durable in
1872 // production; ProcessRegistry must survive restarts.
1873 #[cfg(not(any(test, feature = "testing")))]
1874 {
1875 assert!(
1876 !ds_name.contains("NoopDeadlineStore"),
1877 "EngineBuilder::build: NoopDeadlineStore is active in a \
1878 non-testing build. This silently discards all APERAK deadlines, \
1879 which is an immediately reportable BNetzA violation \
1880 (BK6-22-024 §5, BK7-24-01-009). \
1881 Call .with_deadline_store(SlateDbStore::as_deadline_store()) \
1882 in your production engine assembly. \
1883 If this is a test, enable the 'testing' feature."
1884 );
1885 assert!(
1886 !os_name.contains("NoopOutboxStore"),
1887 "EngineBuilder::build: NoopOutboxStore is active in a \
1888 non-testing build. This silently discards all outbound \
1889 APERAK, CONTRL, and UTILMD messages. \
1890 Call .with_outbox_store(SlateDbStore::as_outbox_store()) \
1891 in your production engine assembly. \
1892 If this is a test, enable the 'testing' feature."
1893 );
1894 assert!(
1895 !pr_name.contains("NoopProcessRegistry"),
1896 "EngineBuilder::build: NoopProcessRegistry is active in a \
1897 non-testing build. This means conversation routing \
1898 (PID → stream_id lookup) is lost on every restart, \
1899 breaking all WiM, GeLi Gas, and GPKE in-flight processes. \
1900 Call .with_registry(SlateDbStore::as_process_registry()) \
1901 in your production engine assembly. \
1902 If this is a test, enable the 'testing' feature."
1903 );
1904 }
1905
1906 // In test/testing/tracing builds: emit warnings instead of panicking.
1907 #[cfg(any(test, feature = "testing", feature = "tracing"))]
1908 {
1909 let ss_name = std::any::type_name::<SS>();
1910 if ss_name.contains("NoopSnapshotStore") {
1911 tracing::warn!(
1912 store = ss_name,
1913 "EngineBuilder: NoopSnapshotStore is active — snapshots will not be \
1914 persisted. Use SlateDbStore::as_snapshot_store() in production."
1915 );
1916 }
1917 if os_name.contains("NoopOutboxStore") {
1918 tracing::warn!(
1919 store = os_name,
1920 "EngineBuilder: NoopOutboxStore is active — outbound messages will be \
1921 silently discarded. Use SlateDbStore::as_outbox_store() in production."
1922 );
1923 }
1924 if ds_name.contains("NoopDeadlineStore") {
1925 tracing::warn!(
1926 store = ds_name,
1927 "EngineBuilder: NoopDeadlineStore is active — scheduled deadlines will \
1928 not fire after restart. Use SlateDbStore::as_deadline_store() in production."
1929 );
1930 }
1931 if pr_name.contains("NoopProcessRegistry") {
1932 tracing::warn!(
1933 store = pr_name,
1934 "EngineBuilder: NoopProcessRegistry is active — process routing will be \
1935 lost on restart. Use SlateDbStore::as_process_registry() in production."
1936 );
1937 }
1938 }
1939 }
1940 // Validate every module before assembling the context.
1941 // A missing adapter or misconfigured module fails at startup (not at
1942 // first inbound message), making deployment failures observable immediately.
1943 for module in &self.modules {
1944 if let Err(msg) = module.configure() {
1945 panic!(
1946 "EngineBuilder::build: module '{}' failed configuration validation: {}",
1947 module.name(),
1948 msg
1949 );
1950 }
1951 // Validate profile requirements via the injected validator.
1952 // Domain crates declare requirements; only the binary crate (makod)
1953 // injects the edi-energy registry — domain crates need no edi-energy
1954 // import for this check.
1955 if let Some(ref validator) = self.profile_validator {
1956 for req in module.profile_requirements() {
1957 assert!(
1958 validator(req.message_type),
1959 "EngineBuilder::build: module '{}' requires an active edi-energy \
1960 profile for '{}' ({}) but none is registered for today's date. \
1961 Run `cargo xtask codegen` to add the missing profile.",
1962 module.name(),
1963 req.message_type,
1964 req.label,
1965 );
1966 }
1967 }
1968 }
1969 // Build the PID router from all registered modules.
1970 // Also assert that no two modules claim the same PID — a PID overlap
1971 // is always a configuration error: one module's messages would be
1972 // silently swallowed by another's workflow, producing missing-process
1973 // errors or incorrect audit trails.
1974 let mut pid_router = PidRouter::new();
1975 let mut pid_owners: std::collections::HashMap<u32, &str> = std::collections::HashMap::new();
1976 // Keep each module's scratch router so we can build `pid_router` from
1977 // them in a second pass with the resolved ownership table.
1978 let mut module_scratches: Vec<PidRouter> = Vec::with_capacity(self.modules.len());
1979
1980 // Pass 1 — detect conflicts, determine PID ownership (first-wins for
1981 // explicit roles, last-wins for DeploymentRoles::all()).
1982 for module in &self.modules {
1983 // Temporarily build a scratch router to read this module's PIDs
1984 // for cross-module overlap detection (module-ownership level).
1985 let mut scratch = PidRouter::new();
1986 module.register_pids_with_roles(&mut scratch, &self.deployment_roles);
1987 for pid in scratch.registered_pids() {
1988 if let Some(prev) = pid_owners.insert(pid, module.name()) {
1989 if self.deployment_roles.is_all() {
1990 // With DeploymentRoles::all() (the default), role-conditional PIDs
1991 // are registered by all modules that claim them, producing last-wins
1992 // semantics. This is acceptable for single-role and dev/test deployments.
1993 //
1994 // In production multi-role deployments where both an NB and nMSB role
1995 // are served by the same instance, set explicit roles via
1996 // `EngineBuilder::with_deployment_roles` to prevent silent misrouting.
1997 //
1998 // We emit a debug-level log here (not warn) because the vast majority
1999 // of deployments are single-role and this overlap is expected/harmless.
2000 #[cfg(feature = "tracing")]
2001 tracing::debug!(
2002 pid,
2003 previous_module = prev,
2004 current_module = module.name(),
2005 "PID registered by multiple modules with DeploymentRoles::all(); \
2006 last module wins (use with_deployment_roles for strict routing)",
2007 );
2008 let _ = prev; // suppress unused-variable warning when tracing is off
2009 } else {
2010 // Explicit roles: the FIRST module to register a PID retains ownership.
2011 // Restore the previous (first) owner and emit a warning so the operator
2012 // can investigate. A panic would be too strict: some shared PIDs
2013 // (e.g. REMADV 33001/33002) are legitimately claimed by both GPKE and
2014 // WiM billing; conversation-ID routing is the long-term solution, but
2015 // first-wins gives correct behaviour for all current deployments.
2016 pid_owners.insert(pid, prev); // restore first owner
2017 #[cfg(feature = "tracing")]
2018 tracing::warn!(
2019 pid,
2020 first_module = prev,
2021 second_module = module.name(),
2022 "PID {pid} claimed by both '{prev}' and '{}' with explicit \
2023 DeploymentRoles; first module ('{prev}') retains ownership. \
2024 Verify PID registration is correct for this deployment.",
2025 module.name(),
2026 );
2027 #[cfg(not(feature = "tracing"))]
2028 let _ = prev; // suppress unused-variable warning when tracing is off
2029 }
2030 }
2031 }
2032 module_scratches.push(scratch);
2033 }
2034
2035 // Pass 2 — build the real `pid_router` from the scratch pads, respecting
2036 // the ownership table built in pass 1.
2037 for (module, scratch) in self.modules.iter().zip(module_scratches.iter()) {
2038 // Unambiguous (Sparte-agnostic) entries: only register if this module
2039 // owns the PID in the resolved ownership table.
2040 for pid in scratch.registered_pids() {
2041 if pid_owners.get(&pid).copied() == Some(module.name())
2042 && let Some(wf) = scratch.route(pid)
2043 {
2044 pid_router.register(pid, wf);
2045 }
2046 }
2047 // Commodity (Sparte-qualified) entries use distinct (pid, Sparte) keys
2048 // and never conflict across modules; register them all unconditionally.
2049 for (pid, sparte, wf) in scratch.registered_commodity_entries() {
2050 pid_router.register_with_sparte(pid, sparte, wf);
2051 }
2052 }
2053 let registered_modules = self.modules.iter().map(|m| m.name()).collect();
2054 let registered_workflows = self
2055 .modules
2056 .iter()
2057 .flat_map(|m| m.workflow_names().iter().copied())
2058 .collect();
2059 EngineContext {
2060 event_store: Arc::new(self.event_store),
2061 snapshot_store: self.snapshot_store,
2062 outbox_store: self.outbox_store,
2063 deadline_store: self.deadline_store,
2064 registry: self.registry,
2065 dead_letter_sink: self.dead_letter_sink,
2066 pid_router,
2067 registered_modules,
2068 registered_workflows,
2069 }
2070 }
2071}
2072
2073#[cfg(test)]
2074mod tests {
2075 use super::*;
2076 use crate::{
2077 deadline::InMemoryDeadlineStore,
2078 error::WorkflowError,
2079 event_store::InMemoryEventStore,
2080 ids::TenantId,
2081 outbox::InMemoryOutboxStore,
2082 pid_router::PidRouter,
2083 registry::InMemoryProcessRegistry,
2084 snapshot::InMemorySnapshotStore,
2085 version::WorkflowId,
2086 workflow::{CommandPayload, EventPayload, Workflow},
2087 };
2088
2089 // ── Minimal workflow for spawn/resume tests ───────────────────────────────
2090
2091 #[derive(serde::Serialize, serde::Deserialize)]
2092 struct PingEvent;
2093
2094 impl EventPayload for PingEvent {
2095 fn event_type(&self) -> &'static str {
2096 "Ping"
2097 }
2098 }
2099
2100 struct PingCommand;
2101
2102 impl CommandPayload for PingCommand {}
2103
2104 #[derive(Default, Clone)]
2105 struct PingState;
2106
2107 struct PingWorkflow;
2108
2109 impl Workflow for PingWorkflow {
2110 type State = PingState;
2111 type Event = PingEvent;
2112 type Command = PingCommand;
2113
2114 fn apply(state: PingState, _: &PingEvent) -> PingState {
2115 state
2116 }
2117
2118 fn handle(
2119 _: &PingState,
2120 _: PingCommand,
2121 ) -> Result<crate::workflow::WorkflowOutput<PingEvent>, WorkflowError> {
2122 Ok(vec![PingEvent].into())
2123 }
2124 }
2125
2126 struct TestModule;
2127
2128 impl EngineModule for TestModule {
2129 fn name(&self) -> &'static str {
2130 "test-module"
2131 }
2132 }
2133
2134 // ── Tests ─────────────────────────────────────────────────────────────────
2135
2136 #[test]
2137 fn build_with_event_store_only() {
2138 let ctx = EngineBuilder::new()
2139 .with_event_store(InMemoryEventStore::new())
2140 .build();
2141 assert!(ctx.registered_modules().is_empty());
2142 }
2143
2144 #[test]
2145 fn build_with_all_stores_and_module() {
2146 let ctx = EngineBuilder::new()
2147 .with_event_store(InMemoryEventStore::new())
2148 .with_snapshot_store(InMemorySnapshotStore::new())
2149 .with_outbox_store(InMemoryOutboxStore::new())
2150 .with_deadline_store(InMemoryDeadlineStore::new())
2151 .with_registry(InMemoryProcessRegistry::new())
2152 .register(Box::new(TestModule))
2153 .build();
2154 assert_eq!(ctx.registered_modules(), &["test-module"]);
2155 }
2156
2157 #[test]
2158 fn multiple_modules_ordered() {
2159 struct ModA;
2160 impl EngineModule for ModA {
2161 fn name(&self) -> &'static str {
2162 "mod-a"
2163 }
2164 }
2165 struct ModB;
2166 impl EngineModule for ModB {
2167 fn name(&self) -> &'static str {
2168 "mod-b"
2169 }
2170 }
2171
2172 let ctx = EngineBuilder::new()
2173 .with_event_store(InMemoryEventStore::new())
2174 .register(Box::new(ModA))
2175 .register(Box::new(ModB))
2176 .build();
2177 assert_eq!(ctx.registered_modules(), &["mod-a", "mod-b"]);
2178 }
2179
2180 #[tokio::test]
2181 async fn spawn_creates_independent_processes() {
2182 let ctx = EngineBuilder::new()
2183 .with_event_store(InMemoryEventStore::new())
2184 .build();
2185 let wf_id = WorkflowId::new("ping", "FV2024-10-01");
2186
2187 let p1 = ctx.spawn::<PingWorkflow>(TenantId::new(), wf_id.clone());
2188 let p2 = ctx.spawn::<PingWorkflow>(TenantId::new(), wf_id);
2189
2190 assert_ne!(p1.process_id(), p2.process_id());
2191 }
2192
2193 #[tokio::test]
2194 async fn resume_sees_previously_appended_events() {
2195 let store = InMemoryEventStore::new();
2196 let ctx = EngineBuilder::new().with_event_store(store).build();
2197
2198 let p = ctx.spawn::<PingWorkflow>(TenantId::new(), WorkflowId::new("ping", "FV2024-10-01"));
2199 p.execute(PingCommand).await.unwrap();
2200
2201 let identity = p.identity();
2202 let resumed = ctx.resume::<PingWorkflow>(identity);
2203 assert_eq!(resumed.event_count().await.unwrap(), 1);
2204 }
2205
2206 #[tokio::test]
2207 async fn registry_routes_process_via_conversation_key() {
2208 use crate::registry::RegistryKey;
2209 let ctx = EngineBuilder::new()
2210 .with_event_store(InMemoryEventStore::new())
2211 .with_registry(InMemoryProcessRegistry::new())
2212 .build();
2213
2214 let p = ctx.spawn::<PingWorkflow>(TenantId::new(), WorkflowId::new("ping", "FV2024-10-01"));
2215 let tenant = p.tenant_id();
2216 let conv_key = RegistryKey::parse("conv:test-conversation-123").expect("valid key");
2217 ctx.registry()
2218 .register(tenant, &conv_key, p.identity())
2219 .await
2220 .unwrap();
2221
2222 let found = ctx
2223 .registry()
2224 .lookup(tenant, &conv_key)
2225 .await
2226 .unwrap()
2227 .expect("must be registered");
2228 let resumed = ctx.resume::<PingWorkflow>(found);
2229 assert_eq!(resumed.process_id(), p.process_id());
2230 }
2231
2232 #[test]
2233 fn pid_router_populated_by_module_register_pids() {
2234 struct PidModule;
2235 impl EngineModule for PidModule {
2236 fn name(&self) -> &'static str {
2237 "pid-module"
2238 }
2239 fn register_pids(&self, router: &mut PidRouter) {
2240 router.register(55001, "gpke-supplier-change");
2241 router.register(55002, "gpke-supplier-change");
2242 }
2243 }
2244
2245 let ctx = EngineBuilder::new()
2246 .with_event_store(InMemoryEventStore::new())
2247 .register(Box::new(PidModule))
2248 .build();
2249
2250 assert_eq!(ctx.pid_router().route(55001), Some("gpke-supplier-change"));
2251 assert_eq!(ctx.pid_router().route(55002), Some("gpke-supplier-change"));
2252 assert!(ctx.pid_router().route(99999).is_none());
2253 assert_eq!(ctx.pid_router().len(), 2);
2254 }
2255
2256 /// Verify that `register_pids_with_roles` gates PIDs behind role checks.
2257 ///
2258 /// Scenario: two modules share PID 19001.
2259 /// - ModuleA registers 19001 → "workflow-a" when role `Nb` is present.
2260 /// - ModuleB registers 19001 → "workflow-b" when role `Nmsb` is explicitly set
2261 /// (not on `all()`).
2262 ///
2263 /// - `all()`: ModuleA fires (Nb ∈ all), ModuleB does NOT (is_all → skip).
2264 /// → 19001 routes to "workflow-a".
2265 /// - `from_roles([Nb])`: ModuleA fires, ModuleB skips.
2266 /// → 19001 routes to "workflow-a".
2267 /// - `from_roles([Nmsb])`: ModuleA skips, ModuleB fires.
2268 /// → 19001 routes to "workflow-b".
2269 #[test]
2270 fn register_pids_with_roles_gates_pids_correctly() {
2271 use crate::marktrolle::{DeploymentRoles, Marktrolle};
2272
2273 struct ModuleA;
2274 impl EngineModule for ModuleA {
2275 fn name(&self) -> &'static str {
2276 "module-a"
2277 }
2278 fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
2279 if roles.contains(Marktrolle::Nb) {
2280 router.register(19_001, "workflow-a");
2281 }
2282 }
2283 }
2284
2285 struct ModuleB;
2286 impl EngineModule for ModuleB {
2287 fn name(&self) -> &'static str {
2288 "module-b"
2289 }
2290 fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
2291 // Only fires on explicit Nmsb, not on all() (backward-compat sentinel).
2292 if !roles.is_all() && roles.contains(Marktrolle::Nmsb) {
2293 router.register(19_001, "workflow-b");
2294 router.register(19_015, "workflow-b");
2295 }
2296 }
2297 }
2298
2299 let build = |roles: DeploymentRoles| {
2300 EngineBuilder::new()
2301 .with_event_store(InMemoryEventStore::new())
2302 .with_deployment_roles(roles)
2303 .register(Box::new(ModuleA))
2304 .register(Box::new(ModuleB))
2305 .build()
2306 };
2307
2308 // all() → backward compat: ModuleA registers 19001 (Nb ∈ all), ModuleB skips.
2309 let ctx = build(DeploymentRoles::all());
2310 assert_eq!(ctx.pid_router().route(19_001), Some("workflow-a"));
2311 assert!(ctx.pid_router().route(19_015).is_none());
2312
2313 // Explicit Nb → same result: ModuleA registers, ModuleB (nMSB) skips.
2314 let ctx = build(DeploymentRoles::nb());
2315 assert_eq!(ctx.pid_router().route(19_001), Some("workflow-a"));
2316 assert!(ctx.pid_router().route(19_015).is_none());
2317
2318 // Explicit Nmsb → ModuleA skips (Nb ∉ roles), ModuleB registers.
2319 let ctx = build(DeploymentRoles::nmsb());
2320 assert_eq!(ctx.pid_router().route(19_001), Some("workflow-b"));
2321 assert_eq!(ctx.pid_router().route(19_015), Some("workflow-b"));
2322 }
2323
2324 /// Verify that explicit roles with two conflicting modules use first-wins semantics
2325 /// (the first module to register a PID retains ownership; the second is silently skipped).
2326 #[test]
2327 fn register_pids_with_roles_conflict_uses_first_wins_with_explicit_roles() {
2328 use crate::marktrolle::{DeploymentRoles, Marktrolle};
2329
2330 struct ConflictA;
2331 impl EngineModule for ConflictA {
2332 fn name(&self) -> &'static str {
2333 "conflict-a"
2334 }
2335 fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
2336 if roles.contains(Marktrolle::Nb) {
2337 router.register(19_001, "workflow-a");
2338 }
2339 }
2340 }
2341
2342 struct ConflictB;
2343 impl EngineModule for ConflictB {
2344 fn name(&self) -> &'static str {
2345 "conflict-b"
2346 }
2347 fn register_pids_with_roles(&self, router: &mut PidRouter, roles: &DeploymentRoles) {
2348 if !roles.is_all() && roles.contains(Marktrolle::Nmsb) {
2349 router.register(19_001, "workflow-b"); // same PID, different workflow
2350 }
2351 }
2352 }
2353
2354 // from_roles([Nb, Nmsb]): both modules fire for PID 19_001.
2355 // First-wins: ConflictA (registered first) retains ownership → "workflow-a".
2356 let ctx = EngineBuilder::new()
2357 .with_event_store(InMemoryEventStore::new())
2358 .with_deployment_roles(DeploymentRoles::from_roles([
2359 Marktrolle::Nb,
2360 Marktrolle::Nmsb,
2361 ]))
2362 .register(Box::new(ConflictA))
2363 .register(Box::new(ConflictB))
2364 .build();
2365 assert_eq!(
2366 ctx.pid_router().route(19_001),
2367 Some("workflow-a"),
2368 "first module should win on PID conflict with explicit roles"
2369 );
2370 }
2371
2372 // ── Graceful shutdown ─────────────────────────────────────────────────────
2373
2374 /// Cancelling the token must make `run` return.
2375 ///
2376 /// A worker that does not read the token loops until the process exits,
2377 /// and dropping its `JoinHandle` does not abort a Tokio task — so the event
2378 /// store would close underneath a worker still running. An outbox
2379 /// `acknowledge` losing that race leaves the counterparty holding a message
2380 /// the outbox still shows as pending, and the next start delivers it
2381 /// again.
2382 #[tokio::test]
2383 async fn a_cancelled_outbox_worker_returns() {
2384 let worker = OutboxWorker {
2385 store: InMemoryOutboxStore::new(),
2386 sender: AlwaysDelivers,
2387 deadline_store: InMemoryDeadlineStore::new(),
2388 batch_size: 10,
2389 // Far longer than the timeout below: the point is that cancellation
2390 // interrupts the idle sleep rather than being noticed after it.
2391 poll_interval: std::time::Duration::from_secs(300),
2392 max_attempts: 48,
2393 max_retry_window: std::time::Duration::from_secs(72 * 3600),
2394 dead_letter_sink: std::sync::Arc::new(crate::dead_letter::LogDeadLetterSink),
2395 heartbeat: None,
2396 shutdown: None,
2397 };
2398 let token = tokio_util::sync::CancellationToken::new();
2399 let worker = worker.with_shutdown(token.clone());
2400
2401 let handle = tokio::spawn(worker.run());
2402 // Let it reach the sleep, then signal.
2403 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2404 token.cancel();
2405
2406 tokio::time::timeout(std::time::Duration::from_secs(5), handle)
2407 .await
2408 .expect("outbox worker must return promptly after cancellation")
2409 .expect("outbox worker must not panic");
2410 }
2411
2412 #[tokio::test]
2413 async fn a_cancelled_deadline_scheduler_returns() {
2414 let scheduler = DeadlineScheduler {
2415 store: InMemoryDeadlineStore::new(),
2416 dispatch: Box::new(|_| Box::pin(async { Ok(()) })),
2417 batch_size: 100,
2418 poll_interval: std::time::Duration::from_secs(300),
2419 heartbeat: None,
2420 shutdown: None,
2421 };
2422 let token = tokio_util::sync::CancellationToken::new();
2423 let scheduler = scheduler.with_shutdown(token.clone());
2424
2425 let handle = tokio::spawn(scheduler.run());
2426 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2427 token.cancel();
2428
2429 tokio::time::timeout(std::time::Duration::from_secs(5), handle)
2430 .await
2431 .expect("deadline scheduler must return promptly after cancellation")
2432 .expect("deadline scheduler must not panic");
2433 }
2434
2435 /// A token cancelled before the first poll must stop the worker without it
2436 /// touching the store at all — the case where shutdown arrives during boot.
2437 #[tokio::test]
2438 async fn a_worker_cancelled_before_it_starts_does_no_work() {
2439 let outbox = InMemoryOutboxStore::new();
2440 let stream_id = crate::ids::StreamId::new("gpke/shutdown-test");
2441 let msg = outbox_message(&stream_id, "UTILMD");
2442 outbox.enqueue(std::slice::from_ref(&msg)).await.unwrap();
2443
2444 let token = tokio_util::sync::CancellationToken::new();
2445 token.cancel();
2446
2447 let worker = OutboxWorker {
2448 store: outbox.clone(),
2449 sender: AlwaysDelivers,
2450 deadline_store: InMemoryDeadlineStore::new(),
2451 batch_size: 10,
2452 poll_interval: std::time::Duration::from_millis(5),
2453 max_attempts: 48,
2454 max_retry_window: std::time::Duration::from_secs(72 * 3600),
2455 dead_letter_sink: std::sync::Arc::new(crate::dead_letter::LogDeadLetterSink),
2456 heartbeat: None,
2457 shutdown: None,
2458 }
2459 .with_shutdown(token);
2460
2461 tokio::time::timeout(std::time::Duration::from_secs(5), worker.run())
2462 .await
2463 .expect("an already-cancelled worker must return immediately");
2464
2465 assert_eq!(
2466 outbox.pending_now(10).await.unwrap().len(),
2467 1,
2468 "the message must stay queued for the next start, not be delivered \
2469 by a worker that was told to stop",
2470 );
2471 }
2472
2473 // ── APERAK delivery-window discharge ──────────────────────────────────────
2474
2475 /// A sender that always succeeds, so the worker takes the delivery path.
2476 struct AlwaysDelivers;
2477 impl As4Sender for AlwaysDelivers {
2478 async fn send(&self, _msg: &crate::outbox::OutboxMessage) -> Result<(), EngineError> {
2479 Ok(())
2480 }
2481 }
2482
2483 fn outbox_message(
2484 stream_id: &crate::ids::StreamId,
2485 message_type: &str,
2486 ) -> crate::outbox::OutboxMessage {
2487 crate::outbox::OutboxMessage::new(
2488 stream_id.clone(),
2489 crate::ids::ProcessId::new(),
2490 TenantId::new(),
2491 crate::ids::CorrelationId::new(),
2492 crate::ids::ConversationId::new(),
2493 crate::ids::EventId::new(),
2494 message_type,
2495 "9900357000004",
2496 serde_json::json!({}),
2497 )
2498 }
2499
2500 fn deadline_on(
2501 stream_id: &crate::ids::StreamId,
2502 msg: &crate::outbox::OutboxMessage,
2503 label: &str,
2504 ) -> Deadline {
2505 Deadline::new(
2506 stream_id.clone(),
2507 msg.process_id,
2508 msg.tenant_id,
2509 WorkflowId::new("gpke-supplier-change", "FV2025-10-01"),
2510 label,
2511 time::OffsetDateTime::now_utc() + time::Duration::hours(6),
2512 )
2513 }
2514
2515 /// Deliver `msg` through the worker's real loop and return the labels that
2516 /// survive on its stream.
2517 ///
2518 /// Drives `run` rather than calling the discharge directly — the wiring is
2519 /// the thing under test, and calling the method straight passes even when
2520 /// `run` never invokes it.
2521 async fn labels_surviving_delivery(
2522 msg: &crate::outbox::OutboxMessage,
2523 stream_id: &crate::ids::StreamId,
2524 registered: &[&str],
2525 ) -> Vec<String> {
2526 let deadlines = InMemoryDeadlineStore::new();
2527 for label in registered {
2528 deadlines
2529 .register(&deadline_on(stream_id, msg, label))
2530 .await
2531 .unwrap();
2532 }
2533 let outbox = InMemoryOutboxStore::new();
2534 outbox.enqueue(std::slice::from_ref(msg)).await.unwrap();
2535
2536 let worker = OutboxWorker {
2537 store: outbox.clone(),
2538 sender: AlwaysDelivers,
2539 deadline_store: deadlines.clone(),
2540 batch_size: 10,
2541 poll_interval: std::time::Duration::from_millis(5),
2542 max_attempts: 48,
2543 max_retry_window: std::time::Duration::from_secs(72 * 3600),
2544 dead_letter_sink: std::sync::Arc::new(crate::dead_letter::LogDeadLetterSink),
2545 heartbeat: None,
2546 shutdown: None,
2547 };
2548 // `run` never returns; give it enough cycles to drain the one message.
2549 let _ = tokio::time::timeout(std::time::Duration::from_millis(300), worker.run()).await;
2550 assert!(
2551 outbox.pending_now(10).await.unwrap().is_empty(),
2552 "the message must have been delivered and acknowledged",
2553 );
2554
2555 let mut left: Vec<String> = deadlines
2556 .for_stream(stream_id)
2557 .await
2558 .unwrap()
2559 .iter()
2560 .map(|d| d.label().to_owned())
2561 .collect();
2562 left.sort();
2563 left
2564 }
2565
2566 /// Delivering a message must retire the window that was watching for it.
2567 ///
2568 /// These windows are registered when the message is enqueued and nothing
2569 /// else ever cancels them, so without the discharge they fire for **every**
2570 /// process — including every one that answered on time. The scheduler cannot
2571 /// tell those apart (a deadline reaching `due_now` is late by construction),
2572 /// so the miss counters would track processes started, not obligations
2573 /// missed.
2574 #[tokio::test]
2575 async fn delivering_a_message_discharges_its_delivery_window() {
2576 // (message type, the window it answers for)
2577 for (message_type, window) in [
2578 ("APERAK", mako_fristen::APERAK_STROM_WINDOW_LABEL),
2579 ("APERAK", mako_fristen::APERAK_GAS_FOLGEPROZESS_LABEL),
2580 ("APERAK", mako_fristen::APERAK_GAS_INITIALPROZESS_LABEL),
2581 ("CONTRL", mako_fristen::CONTRL_FRIST_LABEL),
2582 ] {
2583 let stream_id = crate::ids::StreamId::new("gpke-supplier-change-1");
2584 let msg = outbox_message(&stream_id, message_type);
2585 // A process-response deadline shares the stream and must survive:
2586 // it is waiting on the counterparty, not on our delivery.
2587 let left =
2588 labels_surviving_delivery(&msg, &stream_id, &[window, "gpke-response-window"])
2589 .await;
2590 assert_eq!(
2591 left,
2592 vec!["gpke-response-window"],
2593 "delivering {message_type} must discharge `{window}` and leave \
2594 every other deadline alone",
2595 );
2596 }
2597 }
2598
2599 /// A delivery must not discharge a *different* message's window.
2600 ///
2601 /// The CONTRL and APERAK obligations run concurrently on the same
2602 /// interchange. Acknowledging syntax (CONTRL) says nothing about whether the
2603 /// application-level APERAK went out, so discharging both on one delivery
2604 /// would silence a real violation.
2605 #[tokio::test]
2606 async fn a_delivery_does_not_discharge_another_messages_window() {
2607 let stream_id = crate::ids::StreamId::new("gpke-supplier-change-1");
2608 let contrl = outbox_message(&stream_id, "CONTRL");
2609
2610 let left = labels_surviving_delivery(
2611 &contrl,
2612 &stream_id,
2613 &[
2614 mako_fristen::CONTRL_FRIST_LABEL,
2615 mako_fristen::APERAK_STROM_WINDOW_LABEL,
2616 ],
2617 )
2618 .await;
2619
2620 assert_eq!(
2621 left,
2622 vec![mako_fristen::APERAK_STROM_WINDOW_LABEL.to_owned()],
2623 "a delivered CONTRL discharges only the CONTRL window; the APERAK \
2624 obligation is still outstanding",
2625 );
2626 }
2627
2628 /// Every delivery-window label must be discharged by the message it watches.
2629 ///
2630 /// This is the invariant the miss counters rest on. A window label that
2631 /// `discharges_delivery_window` does not recognise is never retired, so it
2632 /// fires on every process and is counted as a regulatory violation each
2633 /// time — which is precisely how `makod_aperak_missed_total` once came to
2634 /// count Strom processes rather than missed APERAKs.
2635 ///
2636 /// Adding a delivery window means adding a row here.
2637 #[test]
2638 fn every_delivery_window_label_is_discharged_by_its_message() {
2639 for (message_type, label) in [
2640 ("APERAK", mako_fristen::APERAK_STROM_WINDOW_LABEL),
2641 ("APERAK", mako_fristen::APERAK_GAS_FOLGEPROZESS_LABEL),
2642 ("APERAK", mako_fristen::APERAK_GAS_INITIALPROZESS_LABEL),
2643 ("CONTRL", mako_fristen::CONTRL_FRIST_LABEL),
2644 ] {
2645 assert!(
2646 mako_fristen::discharges_delivery_window(message_type, label),
2647 "delivering {message_type} must discharge `{label}`, or the window \
2648 outlives its obligation and alerts on every process",
2649 );
2650 }
2651 }
2652
2653 // ── Retry-budget classification ───────────────────────────────────────────
2654
2655 /// Sink double that records every rejection's attempt count.
2656 #[derive(Default)]
2657 struct RecordingSink(std::sync::Mutex<Vec<u32>>);
2658 impl crate::dead_letter::DeadLetterSink for std::sync::Arc<RecordingSink> {
2659 fn reject(&self, reason: &crate::dead_letter::DeadLetterReason) {
2660 if let crate::dead_letter::DeadLetterReason::OutboxExhausted { attempts, .. } = reason {
2661 self.0
2662 .lock()
2663 .unwrap_or_else(std::sync::PoisonError::into_inner)
2664 .push(*attempts);
2665 }
2666 }
2667 }
2668
2669 struct NoRenderer;
2670 impl As4Sender for NoRenderer {
2671 async fn send(&self, msg: &crate::outbox::OutboxMessage) -> Result<(), EngineError> {
2672 Err(EngineError::RendererNotImplemented {
2673 message_type: msg.message_type.as_ref().into(),
2674 message_id: msg.message_id.to_string().into(),
2675 })
2676 }
2677 }
2678
2679 async fn drive_worker<S: As4Sender>(
2680 sender: S,
2681 msg: crate::outbox::OutboxMessage,
2682 ) -> (InMemoryOutboxStore, std::sync::Arc<RecordingSink>) {
2683 let outbox = InMemoryOutboxStore::new();
2684 outbox.enqueue(std::slice::from_ref(&msg)).await.unwrap();
2685 let sink = std::sync::Arc::new(RecordingSink::default());
2686 let worker = OutboxWorker {
2687 store: outbox.clone(),
2688 sender,
2689 deadline_store: InMemoryDeadlineStore::new(),
2690 batch_size: 10,
2691 poll_interval: std::time::Duration::from_millis(5),
2692 max_attempts: 48,
2693 max_retry_window: std::time::Duration::from_secs(72 * 3600),
2694 dead_letter_sink: std::sync::Arc::new(std::sync::Arc::clone(&sink)),
2695 heartbeat: None,
2696 shutdown: None,
2697 };
2698 let _ = tokio::time::timeout(std::time::Duration::from_millis(300), worker.run()).await;
2699 (outbox, sink)
2700 }
2701
2702 /// `RendererNotImplemented` is documented as permanent — the worker must
2703 /// dead-letter it on the *first* attempt, not burn the retry budget on a
2704 /// failure that cannot heal between attempts. Until the permanent arm
2705 /// matched it, this promise was broken.
2706 #[tokio::test(start_paused = true)]
2707 async fn a_missing_renderer_dead_letters_without_retrying() {
2708 let stream_id = crate::ids::StreamId::new("test-renderer-missing");
2709 let msg = outbox_message(&stream_id, "MSCONS");
2710 let (outbox, sink) = drive_worker(NoRenderer, msg).await;
2711
2712 assert!(
2713 outbox.pending_now(10).await.unwrap().is_empty(),
2714 "the message must be acknowledged, not left for another attempt",
2715 );
2716 let rejections = sink
2717 .0
2718 .lock()
2719 .unwrap_or_else(std::sync::PoisonError::into_inner)
2720 .clone();
2721 assert_eq!(
2722 rejections,
2723 vec![0],
2724 "exactly one dead-letter, on the first attempt (attempt_count 0)",
2725 );
2726 }
2727
2728 /// The retry budget is a *window*, not a count: a message whose age has
2729 /// exceeded it after at least one attempt is dead-lettered even though the
2730 /// attempt belt is nowhere near exhausted — full-jitter backoff makes a
2731 /// count no proxy for the 72 h duty.
2732 #[tokio::test(start_paused = true)]
2733 async fn an_aged_message_with_a_prior_attempt_is_dead_lettered() {
2734 let stream_id = crate::ids::StreamId::new("test-window-exhausted");
2735 let mut msg = outbox_message(&stream_id, "UTILMD");
2736 msg.created_at = time::OffsetDateTime::now_utc() - time::Duration::hours(73);
2737 msg.attempt_count = 1;
2738 let (outbox, sink) = drive_worker(AlwaysDelivers, msg).await;
2739
2740 assert!(
2741 outbox.pending_now(10).await.unwrap().is_empty(),
2742 "the exhausted message must leave the outbox",
2743 );
2744 let rejections = sink
2745 .0
2746 .lock()
2747 .unwrap_or_else(std::sync::PoisonError::into_inner)
2748 .clone();
2749 assert_eq!(
2750 rejections,
2751 vec![1],
2752 "the window, not the attempt belt, must have dead-lettered it",
2753 );
2754 }
2755
2756 /// A message that aged past the window while the worker was down still
2757 /// gets its first try — the window is only consulted after an attempt, so
2758 /// downtime never buries a message unsent.
2759 #[tokio::test(start_paused = true)]
2760 async fn an_aged_message_with_no_attempts_is_still_tried_once() {
2761 let stream_id = crate::ids::StreamId::new("test-aged-first-try");
2762 let mut msg = outbox_message(&stream_id, "UTILMD");
2763 msg.created_at = time::OffsetDateTime::now_utc() - time::Duration::hours(200);
2764 let (outbox, sink) = drive_worker(AlwaysDelivers, msg).await;
2765
2766 assert!(
2767 outbox.pending_now(10).await.unwrap().is_empty(),
2768 "the message must have been delivered and acknowledged",
2769 );
2770 assert!(
2771 sink.0
2772 .lock()
2773 .unwrap_or_else(std::sync::PoisonError::into_inner)
2774 .is_empty(),
2775 "delivery, not dead-lettering: age alone must never bury a message",
2776 );
2777 }
2778}