1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
//! Lifecycle and restart types for actor supervision.
//!
//! This module defines the types that control how actors are supervised:
//!
//! - [`TerminationReason`] — why an actor stopped (clean shutdown, panic, restart limit)
//! - [`RestartPolicy`] — whether and when to restart (Temporary, Transient, Permanent)
//! - [`RestartConfig`] — full restart specification with limits, rolling window, and backoff
//! - [`ActorFactory`] — creates new actor instances on restart
//!
//! # Restart policies (OTP-inspired)
//!
//! | Policy | Restart on panic? | Restart on clean stop? |
//! |--------|-------------------|------------------------|
//! | `Temporary` | No | No |
//! | `Transient` | Yes | No |
//! | `Permanent` | Yes | Yes |
//!
//! # Backoff
//!
//! When an actor crashes repeatedly, [`BackoffConfig`] applies exponential
//! backoff between restarts to avoid tight crash loops. The delay starts at
//! `initial` and doubles (by `multiplier`) up to `max`.
use Duration;
use mpsc;
use crate;
// =============================================================================
// LIFECYCLE & RESTART TYPES
// =============================================================================
/// Why an actor terminated — used by watches and restart policies.
///
/// # Examples
///
/// ```rust,ignore
/// impl Actor for Supervisor {
/// type State = SupervisorState;
///
/// fn on_actor_terminated(&mut self, state: &mut SupervisorState, event: &ActorTerminated) {
/// match &event.reason {
/// TerminationReason::Stopped => println!("{} stopped cleanly", event.label),
/// TerminationReason::Panicked(msg) => println!("{} panicked: {msg}", event.label),
/// TerminationReason::RestartLimitExceeded => println!("{} gave up", event.label),
/// }
/// }
/// }
/// ```
/// Notification delivered to watchers when a watched actor terminates.
///
/// # Examples
///
/// ```rust,ignore
/// fn on_actor_terminated(&mut self, state: &mut MyState, event: &ActorTerminated) {
/// match event.tag.as_deref() {
/// Some("writer") => state.writer_down = true,
/// Some("reader") => state.reader_down = true,
/// _ => {}
/// }
/// tracing::warn!("Watched actor {} terminated: {:?}", event.label, event.reason);
/// }
/// ```
/// Restart strategy for an actor — mirrors Erlang/OTP child spec strategies.
///
/// # Examples
///
/// ```rust,ignore
/// use murmer::{RestartConfig, RestartPolicy};
///
/// // Restart on crash only (most common for workers)
/// let config = RestartConfig { policy: RestartPolicy::Transient, ..Default::default() };
///
/// // Always restart (for essential services)
/// let config = RestartConfig { policy: RestartPolicy::Permanent, ..Default::default() };
///
/// // Never restart (for one-shot tasks)
/// let config = RestartConfig { policy: RestartPolicy::Temporary, ..Default::default() };
/// ```
/// Configuration for restart limits and backoff behavior.
///
/// # Examples
///
/// ```rust,ignore
/// use murmer::{RestartConfig, RestartPolicy, BackoffConfig};
/// use std::time::Duration;
///
/// let config = RestartConfig {
/// policy: RestartPolicy::Transient, // restart on panic only
/// max_restarts: 3, // max 3 restarts...
/// window: Duration::from_secs(30), // ...within 30 seconds
/// backoff: BackoffConfig {
/// initial: Duration::from_millis(200),
/// max: Duration::from_secs(10),
/// multiplier: 2.0,
/// },
/// };
///
/// let endpoint = system.start_with_config("worker/0", MyFactory, config);
/// ```
/// Exponential backoff configuration for actor restarts.
///
/// # Examples
///
/// ```rust,ignore
/// use murmer::BackoffConfig;
/// use std::time::Duration;
///
/// let backoff = BackoffConfig {
/// initial: Duration::from_millis(500), // first retry after 500ms
/// max: Duration::from_secs(30), // cap at 30s
/// multiplier: 2.0, // 500ms → 1s → 2s → 4s → ...
/// };
/// ```
/// Factory for creating actor instances on restart.
///
/// `&mut self` allows stateful factories (e.g. incrementing restart counters,
/// loading config from disk).
///
/// # Examples
///
/// ```rust,ignore
/// struct CounterFactory { initial_count: i64 }
///
/// impl ActorFactory for CounterFactory {
/// type Actor = Counter;
/// fn create(&mut self) -> (Counter, CounterState) {
/// (Counter, CounterState { count: self.initial_count })
/// }
/// }
///
/// let ep = system.start_with_policy(
/// "counter/0",
/// CounterFactory { initial_count: 0 },
/// RestartPolicy::Permanent,
/// );
/// ```
// =============================================================================
// TERMINATE HOOK (fault-injection seam)
// =============================================================================
/// Fault-injection seam fired between an actor's supervisor loop exiting and
/// the actor's removal from the receptionist.
///
/// # Why this exists
///
/// Deregistration is normally instantaneous: the supervisor loop breaks and the
/// `DeregisterGuard` fires in `Drop`, which is synchronous. Real systems are not
/// so tidy — an actor can accept its stop, then take time to drain a mailbox or
/// tear down external resources, staying *registered but not serving* for a
/// while. Callers must tolerate that window (murmer's own eviction paths wait on
/// deregistration with a deadline), but without a seam there is no way to *test*
/// that tolerance, because the window is zero-width by construction.
///
/// Install a hook and the returned future is awaited inside that window. While
/// it is pending the actor is still in the receptionist — `lookup` finds it,
/// listings include it, watches have not fired — and its supervisor is no longer
/// processing messages. That is precisely the "slow to die" state.
///
/// # Determinism
///
/// The hook returns a future, so a simulation test builds its delay from the
/// [`Runtime`](crate::runtime::Runtime) seam (`runtime.sleep(d)`) and the wait
/// runs on virtual time — reproducible from the world seed like everything else.
/// Do not sleep on wall-clock time inside a hook used under simulation.
///
/// # Scope
///
/// This is a test/fault-injection seam. It is not a place to do real work:
/// it runs on the supervisor's task, it blocks the actor's deregistration for as
/// long as it is pending, and a hook that never completes leaks a registry entry.
/// It fires on every terminating actor on the node, including murmer's internal
/// ones, so a hook that only wants one actor must filter on `label`.
///
/// It does *not* fire on the restart-limit-exceeded path
/// (`TerminationReason::RestartLimitExceeded`), where the receptionist
/// deregisters directly rather than through a supervisor exit.
///
/// # Examples
///
/// ```rust,ignore
/// use murmer::lifecycle::{TerminateHook, TerminationReason};
/// use murmer::runtime::{BoxFuture, Runtime};
/// use std::sync::Arc;
/// use std::time::Duration;
///
/// /// Makes one labelled actor take `delay` of virtual time to de-register.
/// struct SlowToDie {
/// label: String,
/// delay: Duration,
/// runtime: Arc<dyn Runtime>,
/// }
///
/// impl TerminateHook for SlowToDie {
/// fn before_deregister(
/// &self,
/// label: &str,
/// _reason: &TerminationReason,
/// ) -> BoxFuture<'static, ()> {
/// if label == self.label {
/// self.runtime.sleep(self.delay)
/// } else {
/// Box::pin(std::future::ready(()))
/// }
/// }
/// }
///
/// world.system().receptionist().set_terminate_hook(Some(Arc::new(SlowToDie {
/// label: "cache/user".into(),
/// delay: Duration::from_secs(5),
/// runtime: Arc::new(world.runtime().clone()),
/// })));
/// ```
/// Internal: signals delivered to actors from the system.
pub
/// Internal: a watch entry stored in the receptionist.
pub