1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
use crate::enums::{
BackoffStrategy, CircuitBreakerConfig, HealthStatus, RestartPolicy, SupervisorEvent, TaskState,
};
use crate::error::SupervisorResult;
use std::time::Duration;
/// Core trait for any long-running task that needs supervision
#[async_trait::async_trait]
pub trait SupervisedTask: Send + Sync {
// REQUIRED METHODS
/// Unique identifier for this task (used in logs, monitoring, and dependency resolution)
fn id(&self) -> &'static str;
/// Human-readable name for this task (used in logs and monitoring)
fn name(&self) -> String {
self.id().to_string()
}
/// Main task execution - this should run until completion or error
async fn run(&self) -> SupervisorResult<()>;
// DEPENDENCY MANAGEMENT
/// Declare task IDs that must complete setup before this task starts
fn dependencies(&self) -> &'static [&'static str] {
&[]
}
// OPTIONAL CONFIGURATION
/// Restart policy when task fails or panics
fn restart_policy(&self) -> RestartPolicy {
RestartPolicy::Always
}
/// Backoff strategy between restart attempts
fn backoff_strategy(&self) -> BackoffStrategy {
BackoffStrategy::default()
}
/// Priority of the task (higher values = higher priority)
/// Used when concurrency limits are reached to decide which tasks start first
fn priority(&self) -> i32 {
0
}
/// Optional concurrency limit for this specific task
/// This is useful if the task is part of a pool or if multiple instances exist
fn concurrency_limit(&self) -> Option<usize> {
None
}
/// Configuration for the task's circuit breaker
/// If `None`, the circuit breaker is disabled for this task
fn circuit_breaker(&self) -> Option<CircuitBreakerConfig> {
None
}
/// Maximum time to wait for the task to finish its `run()` and `on_shutdown()`
/// before forced termination during supervisor shutdown.
fn shutdown_timeout(&self) -> Duration {
Duration::from_secs(30)
}
/// Optional cron expression for scheduled execution
#[cfg(feature = "cron")]
fn cron_schedule(&self) -> Option<&'static str> {
None
}
/// Optional initial delay before the first execution
///
/// This is useful for staggering task startup to prevent thundering herd problems
/// or for giving dependencies time to initialize.
fn initial_delay(&self) -> Option<Duration> {
None
}
/// Optional jitter to add randomness to the initial delay
///
/// Returns a tuple of (min_jitter, max_jitter) that will be randomly added
/// to the initial delay. This helps prevent thundering herd problems in
/// distributed deployments where multiple instances start simultaneously.
///
/// Example: If initial_delay is 5s and jitter is (0, 2s), the actual delay
/// will be between 5s and 7s.
fn jitter(&self) -> Option<(Duration, Duration)> {
None
}
/// Minimum time between restart attempts (rate limiting)
///
/// This prevents tasks from restarting too frequently, which can overwhelm
/// external services or resources. If a task fails and wants to restart,
/// it will wait at least this duration before attempting again.
///
/// This is applied in addition to the backoff strategy - the actual delay
/// will be the maximum of the backoff delay and this minimum restart interval.
fn min_restart_interval(&self) -> Option<Duration> {
None
}
/// Time window restrictions for task execution
///
/// Returns optional start and end times (as hours in 24-hour format) during
/// which the task is allowed to run. Outside this window, the task will wait
/// until the next allowed time.
///
/// Example: Some((9, 17)) means the task can only run between 9 AM and 5 PM.
/// Use None for either value to indicate no restriction on that boundary.
///
/// Note: Times are evaluated in UTC by default.
fn execution_time_window(&self) -> Option<(Option<u8>, Option<u8>)> {
None
}
/// Task group identifier for atomic operations
///
/// Tasks with the same group ID can be started, stopped, or managed together.
/// This is useful for related tasks that should be treated as a unit.
///
/// Example: A database connection pool and its query processor might be in
/// the same group so they start and stop together.
fn group_id(&self) -> Option<&'static str> {
None
}
/// Conditional dependencies based on environment or runtime conditions
///
/// Returns a list of dependencies that are only active when certain conditions are met.
/// Each tuple contains (dependency_id, condition_function).
/// The condition function receives the current environment context and returns true
/// if the dependency should be enforced.
///
/// Example: Only depend on "cache-service" if USE_CACHE env var is set
/// ```ignore
/// vec![("cache-service", Box::new(|| std::env::var("USE_CACHE").is_ok()))]
/// ```
#[allow(clippy::type_complexity)]
fn conditional_dependencies(&self) -> Vec<(&'static str, Box<dyn Fn() -> bool + Send + Sync>)> {
Vec::new()
}
/// Get all active dependencies (regular + conditional that evaluate to true)
///
/// This combines regular dependencies with conditional dependencies whose
/// conditions currently evaluate to true.
fn active_dependencies(&self) -> Vec<&'static str> {
let mut deps = self.dependencies().to_vec();
// Add conditional dependencies whose conditions are met
for (dep_id, condition) in self.conditional_dependencies() {
if condition() {
deps.push(dep_id);
}
}
deps
}
// LIFECYCLE HOOKS
/// Called once before the first run() attempt
async fn setup(&self) -> SupervisorResult<()> {
Ok(())
}
/// Called after task stops (success, failure, or panic)
///
/// **Purpose:** Internal resource cleanup and teardown.
///
/// This hook is called automatically by the supervision loop whenever a task's
/// `run()` method completes (successfully or with error) or panics. It's meant for:
/// - Closing database connections
/// - Releasing file handles
/// - Dropping temporary resources
/// - Any internal cleanup needed before restart or shutdown
///
/// **When it's called:**
/// - After every `run()` completion (success or failure)
/// - After a panic is caught
/// - When task receives Stop command
/// - During supervisor shutdown
///
/// **Important:** This is NOT for graceful shutdown logic. Use `on_shutdown()` instead
/// if you need to perform actions specifically during supervisor-initiated shutdown.
async fn cleanup(&self) {
// Default: no cleanup
}
// RESTART CONTROL
/// Dynamic restart control - called before each restart
async fn should_restart(&self, _attempt: usize, _last_error: &str) -> bool {
true
}
// MONITORING & OBSERVABILITY
/// Health check for monitoring endpoints
async fn health_check(&self) -> HealthStatus {
HealthStatus::Healthy
}
/// Return task-specific metrics (optional)
async fn metrics(&self) -> Option<TaskMetrics> {
None
}
/// Called before each restart (not called on first attempt)
async fn on_restart(&self, _attempt: usize) {
// Default: no action
}
// ERROR HANDLING HOOKS
/// Called when run() returns an error
async fn on_error(&self, _error: &str, _attempt: usize) {
// Default: no action (supervisor logs by default)
}
/// Called when run() panics
async fn on_panic(&self, _panic_info: &str, _attempt: usize) {
// Default: no action (supervisor logs by default)
}
/// Called when the service is shutting down
///
/// **Purpose:** Graceful shutdown hook for external/user-defined cleanup.
///
/// This hook is called ONLY during supervisor-initiated graceful shutdown,
/// AFTER the supervision loop has terminated. It's meant for:
/// - Flushing buffers to disk/network
/// - Sending final notifications
/// - Performing graceful disconnections
/// - Any user-defined shutdown logic that should run once at the end
///
/// **When it's called:**
/// - ONLY during `TaskRuntime::shutdown()`
/// - AFTER the supervision loop ends
/// - AFTER `cleanup()` has been called
/// - Once per task, in dependency-aware order
///
/// **Key difference from `cleanup()`:**
/// - `cleanup()` = Internal teardown, called after EVERY run() completion
/// - `on_shutdown()` = Graceful shutdown, called ONCE during supervisor shutdown
///
/// **Example:**
/// ```ignore
/// async fn on_shutdown(&self) {
/// // Flush pending messages
/// self.message_queue.flush().await.ok();
/// // Notify monitoring service
/// self.notify_shutdown().await.ok();
/// }
/// ```
async fn on_shutdown(&self) {
// Default implementation does nothing
}
}
/// Extension trait for tasks that need state tracking
#[async_trait::async_trait]
pub trait StatefulTask: SupervisedTask {
/// Get the current state of the task
async fn state(&self) -> TaskState;
}
/// Trait for components that listen to supervisor events
#[async_trait::async_trait]
pub trait SupervisorEventListener: Send + Sync {
/// Called when a supervisor event occurs
async fn on_event(&self, event: SupervisorEvent);
}
/// Optional metrics that tasks can expose
#[derive(Debug, Clone)]
pub struct TaskMetrics {
/// Number of successful operations
pub success_count: u64,
/// Number of failed operations
pub error_count: u64,
/// Number of messages/items processed
pub processed_count: u64,
/// Current processing rate (per second)
pub processing_rate: f64,
/// Average processing time (milliseconds)
pub avg_processing_time_ms: f64,
/// Custom metrics (key-value pairs)
pub custom: std::collections::HashMap<String, f64>,
}
impl Default for TaskMetrics {
fn default() -> Self {
Self {
success_count: 0,
error_count: 0,
processed_count: 0,
processing_rate: 0.0,
avg_processing_time_ms: 0.0,
custom: std::collections::HashMap::new(),
}
}
}
impl TaskMetrics {
pub fn new() -> Self {
Self::default()
}
/// Add a custom metric
pub fn add_custom(&mut self, key: impl Into<String>, value: f64) {
self.custom.insert(key.into(), value);
}
}