1use crate::error::types::SupervisorError;
7use crate::id::types::{ChildId, SupervisorPath};
8use crate::policy::budget::RestartBudgetConfig;
9use crate::policy::failure_window::FailureWindowConfig;
10use crate::policy::group::GroupDependencyEdge;
11use crate::policy::meltdown::MeltdownPolicy;
12use crate::policy::task_role_defaults::{SeverityClass, TaskRole, semantic_conflicts_for_child};
13use crate::spec::child::{BackoffPolicy, ChildSpec, HealthPolicy, RestartPolicy};
14use crate::spec::shutdown::{ShutdownBudget, TreeShutdownPolicy};
15use confique::Config;
16use schemars::JsonSchema;
17use serde::{Deserialize, Serialize};
18use std::collections::{HashMap, HashSet};
19use std::time::Duration;
20
21pub const RECOMMENDED_CHANNEL_CAPACITY: usize = 256;
23
24#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
26pub enum SupervisionStrategy {
27 OneForOne,
29 OneForAll,
31 RestForOne,
33}
34
35#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
37#[serde(rename_all = "snake_case")]
38pub enum EscalationPolicy {
39 EscalateToParent,
41 ShutdownTree,
43 QuarantineScope,
45}
46
47#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
49pub struct RestartLimit {
50 pub max_restarts: u32,
52 pub window: Duration,
54}
55
56impl RestartLimit {
57 pub fn new(max_restarts: u32, window: Duration) -> Self {
68 Self {
69 max_restarts,
70 window,
71 }
72 }
73}
74
75#[derive(Debug, Clone, PartialEq, Eq)]
77pub struct GroupStrategy {
78 pub group: String,
80 pub strategy: SupervisionStrategy,
82 pub restart_limit: Option<RestartLimit>,
84 pub escalation_policy: Option<EscalationPolicy>,
86}
87
88impl GroupStrategy {
89 pub fn new(group: impl Into<String>, strategy: SupervisionStrategy) -> Self {
100 Self {
101 group: group.into(),
102 strategy,
103 restart_limit: None,
104 escalation_policy: None,
105 }
106 }
107}
108
109#[derive(Debug, Clone, PartialEq)]
112pub struct GroupConfig {
113 pub name: String,
115 pub children: Vec<ChildId>,
117 pub budget: Option<RestartBudgetConfig>,
123}
124
125impl GroupConfig {
126 pub fn new(
138 name: impl Into<String>,
139 children: Vec<ChildId>,
140 budget: Option<RestartBudgetConfig>,
141 ) -> Self {
142 Self {
143 name: name.into(),
144 children,
145 budget,
146 }
147 }
148}
149
150#[derive(Debug, Clone, PartialEq, Eq)]
152pub struct ChildStrategyOverride {
153 pub child_id: ChildId,
155 pub strategy: SupervisionStrategy,
157 pub restart_limit: Option<RestartLimit>,
159 pub escalation_policy: Option<EscalationPolicy>,
161}
162
163impl ChildStrategyOverride {
164 pub fn new(child_id: ChildId, strategy: SupervisionStrategy) -> Self {
175 Self {
176 child_id,
177 strategy,
178 restart_limit: None,
179 escalation_policy: None,
180 }
181 }
182}
183
184#[derive(Debug, Clone, Copy, PartialEq, Eq)]
186pub struct DynamicSupervisorPolicy {
187 pub enabled: bool,
189 pub child_limit: Option<usize>,
191}
192
193impl DynamicSupervisorPolicy {
194 pub fn unbounded() -> Self {
204 Self {
205 enabled: true,
206 child_limit: None,
207 }
208 }
209
210 pub fn limited(child_limit: usize) -> Self {
220 Self {
221 enabled: true,
222 child_limit: Some(child_limit),
223 }
224 }
225
226 pub fn allows_addition(&self, current_child_count: usize) -> bool {
236 self.enabled
237 && self
238 .child_limit
239 .is_none_or(|limit| current_child_count < limit)
240 }
241}
242
243#[derive(Debug, Clone, PartialEq, Eq)]
245pub struct StrategyExecutionPlan {
246 pub failed_child: ChildId,
248 pub strategy: SupervisionStrategy,
250 pub scope: Vec<ChildId>,
252 pub group: Option<String>,
254 pub restart_limit: Option<RestartLimit>,
256 pub escalation_policy: Option<EscalationPolicy>,
258 pub dynamic_supervisor_enabled: bool,
260}
261
262#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
264#[serde(rename_all = "snake_case")]
265pub enum BackpressureStrategy {
266 AlertAndBlock,
268 SampleAndAudit,
270}
271
272impl Default for BackpressureStrategy {
273 fn default() -> Self {
275 Self::AlertAndBlock
276 }
277}
278
279#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Config, JsonSchema)]
281pub struct BackpressureConfig {
282 #[config(default = "alert_and_block")]
284 #[serde(default)]
285 pub strategy: BackpressureStrategy,
286 #[config(default = 80)]
288 #[serde(default = "default_warn_threshold")]
289 pub warn_threshold_pct: u8,
290 #[config(default = 95)]
292 #[serde(default = "default_critical_threshold")]
293 pub critical_threshold_pct: u8,
294 #[config(default = 30)]
296 #[serde(default = "default_window_secs")]
297 pub window_secs: u64,
298 #[config(default = 1024)]
300 #[serde(default = "default_audit_capacity")]
301 pub audit_channel_capacity: usize,
302}
303
304fn default_warn_threshold() -> u8 {
306 80
307}
308
309fn default_critical_threshold() -> u8 {
311 95
312}
313
314fn default_window_secs() -> u64 {
316 30
317}
318
319fn default_audit_capacity() -> usize {
321 1024
322}
323
324impl Default for BackpressureConfig {
325 fn default() -> Self {
327 Self {
328 strategy: BackpressureStrategy::AlertAndBlock,
329 warn_threshold_pct: default_warn_threshold(),
330 critical_threshold_pct: default_critical_threshold(),
331 window_secs: default_window_secs(),
332 audit_channel_capacity: default_audit_capacity(),
333 }
334 }
335}
336
337#[derive(Debug, Clone)]
339pub struct SupervisorSpec {
340 pub path: SupervisorPath,
342 pub strategy: SupervisionStrategy,
344 pub children: Vec<ChildSpec>,
346 pub config_version: String,
348 pub default_restart_policy: RestartPolicy,
350 pub default_backoff_policy: BackoffPolicy,
352 pub default_health_policy: HealthPolicy,
354 pub tree_shutdown: TreeShutdownPolicy,
356 pub supervisor_failure_limit: u32,
358 pub restart_limit: Option<RestartLimit>,
360 pub escalation_policy: Option<EscalationPolicy>,
368 pub group_strategies: Vec<GroupStrategy>,
370 pub group_configs: Vec<GroupConfig>,
372 pub group_dependencies: Vec<GroupDependencyEdge>,
374 pub severity_defaults: HashMap<TaskRole, SeverityClass>,
376 pub child_strategy_overrides: Vec<ChildStrategyOverride>,
378 pub dynamic_supervisor_policy: DynamicSupervisorPolicy,
380 pub control_channel_capacity: usize,
385 pub event_channel_capacity: usize,
390 pub backpressure_config: BackpressureConfig,
392 pub meltdown_policy: MeltdownPolicy,
394 pub failure_window_config: FailureWindowConfig,
396 pub restart_budget_config: RestartBudgetConfig,
398 pub pipeline_journal_capacity: usize,
400 pub pipeline_subscriber_capacity: usize,
402 pub concurrent_restart_limit: u32,
404 pub metrics_enabled: bool,
406 pub audit_enabled: bool,
408}
409
410impl SupervisorSpec {
411 pub fn root(children: Vec<ChildSpec>) -> Self {
428 let tree_shutdown = TreeShutdownPolicy::default();
429 let mut spec = Self {
430 path: SupervisorPath::root(),
431 strategy: SupervisionStrategy::OneForOne,
432 children,
433 config_version: String::from("unversioned"),
434 default_restart_policy: RestartPolicy::Transient,
435 default_backoff_policy: BackoffPolicy::new(
436 Duration::from_millis(10),
437 Duration::from_secs(1),
438 0.0,
439 ),
440 default_health_policy: HealthPolicy::new(
441 Duration::from_secs(1),
442 Duration::from_secs(3),
443 ),
444 tree_shutdown,
445 supervisor_failure_limit: 1,
446 restart_limit: None,
447 escalation_policy: None,
448 group_strategies: Vec::new(),
449 group_configs: Vec::new(),
450 group_dependencies: Vec::new(),
451 severity_defaults: HashMap::new(),
452 child_strategy_overrides: Vec::new(),
453 dynamic_supervisor_policy: DynamicSupervisorPolicy::unbounded(),
454 control_channel_capacity: RECOMMENDED_CHANNEL_CAPACITY,
455 event_channel_capacity: RECOMMENDED_CHANNEL_CAPACITY,
456 backpressure_config: BackpressureConfig::default(),
457 meltdown_policy: MeltdownPolicy::new(
458 3,
459 Duration::from_secs(10),
460 5,
461 Duration::from_secs(30),
462 10,
463 Duration::from_secs(60),
464 Duration::from_secs(120),
465 ),
466 failure_window_config: FailureWindowConfig::time_sliding(60, 5),
467 restart_budget_config: RestartBudgetConfig::safe_default(),
468 pipeline_journal_capacity: 100,
469 pipeline_subscriber_capacity: 10,
470 concurrent_restart_limit: 5,
471 metrics_enabled: true,
472 audit_enabled: true,
473 };
474 spec.propagate_tree_shutdown_budget();
475 spec
476 }
477
478 pub fn propagate_tree_shutdown_budget(&mut self) {
491 let budget = self.tree_shutdown.budget;
492 for child in &mut self.children {
493 child.shutdown_budget = budget;
494 }
495 }
496
497 pub fn shutdown_budget_for(&self, child_id: &ChildId) -> ShutdownBudget {
507 self.children
508 .iter()
509 .find(|child| &child.id == child_id)
510 .map(|child| child.shutdown_budget)
511 .unwrap_or(self.tree_shutdown.budget)
512 }
513
514 pub fn validate(&self) -> Result<(), SupervisorError> {
524 if self.config_version.trim().is_empty() {
525 return Err(SupervisorError::fatal_config(
526 "config version must not be empty",
527 ));
528 }
529 if self.supervisor_failure_limit == 0 {
530 return Err(SupervisorError::fatal_config(
531 "supervisor failure limit must be greater than zero",
532 ));
533 }
534 if self.control_channel_capacity == 0 {
535 return Err(SupervisorError::fatal_config(
536 "control channel capacity must be greater than zero",
537 ));
538 }
539 if self.event_channel_capacity == 0 {
540 return Err(SupervisorError::fatal_config(
541 "event channel capacity must be greater than zero",
542 ));
543 }
544 validate_backpressure_config(&self.backpressure_config)?;
545 for child in &self.children {
546 child.validate()?;
547 }
548 validate_restart_limit(self.restart_limit)?;
549 validate_group_strategies(&self.group_strategies, &self.children)?;
550 validate_child_strategy_overrides(self)?;
551 validate_task_roles(&self.children)?;
552 validate_dynamic_policy(self.dynamic_supervisor_policy)?;
553 validate_child_group_names(&self.children, &self.group_configs)?;
554 validate_pipeline_policy(self)?;
555 Ok(())
556 }
557}
558
559fn validate_child_group_names(
563 children: &[ChildSpec],
564 group_configs: &[GroupConfig],
565) -> Result<(), SupervisorError> {
566 let group_names: std::collections::HashSet<&str> =
567 group_configs.iter().map(|g| g.name.as_str()).collect();
568
569 for child in children {
570 if let Some(ref group_name) = child.group
571 && !group_names.contains(group_name.as_str())
572 {
573 return Err(SupervisorError::fatal_config(format!(
574 "child '{}' references unknown group '{}'; available groups: {:?}",
575 child.id,
576 group_name,
577 group_names.iter().copied().collect::<Vec<_>>(),
578 )));
579 }
580 }
581 Ok(())
582}
583
584fn validate_restart_limit(limit: Option<RestartLimit>) -> Result<(), SupervisorError> {
594 let Some(limit) = limit else {
595 return Ok(());
596 };
597 if limit.max_restarts == 0 {
598 return Err(SupervisorError::fatal_config(
599 "restart limit max_restarts must be greater than zero",
600 ));
601 }
602 if limit.window.is_zero() {
603 return Err(SupervisorError::fatal_config(
604 "restart limit window must be greater than zero",
605 ));
606 }
607 Ok(())
608}
609
610fn validate_group_strategies(
620 strategies: &[GroupStrategy],
621 children: &[ChildSpec],
622) -> Result<(), SupervisorError> {
623 let mut groups = HashSet::new();
624 for strategy in strategies {
625 if strategy.group.trim().is_empty() {
626 return Err(SupervisorError::fatal_config(
627 "group strategy group must not be empty",
628 ));
629 }
630 if !groups.insert(strategy.group.clone()) {
631 return Err(SupervisorError::fatal_config(format!(
632 "duplicate group strategy: {}",
633 strategy.group
634 )));
635 }
636 validate_restart_limit(strategy.restart_limit)?;
637 }
638 validate_group_membership(strategies, children)?;
639 Ok(())
640}
641
642fn validate_group_membership(
653 strategies: &[GroupStrategy],
654 children: &[ChildSpec],
655) -> Result<(), SupervisorError> {
656 for strategy in strategies {
657 if !children
658 .iter()
659 .any(|child| child.group.as_deref() == Some(strategy.group.as_str()))
660 {
661 return Err(SupervisorError::fatal_config(format!(
662 "group strategy references unused group: {}",
663 strategy.group
664 )));
665 }
666 }
667 Ok(())
668}
669
670fn validate_child_strategy_overrides(spec: &SupervisorSpec) -> Result<(), SupervisorError> {
680 let child_ids = spec
681 .children
682 .iter()
683 .map(|child| child.id.clone())
684 .collect::<HashSet<_>>();
685 let mut overrides = HashSet::new();
686 for strategy in &spec.child_strategy_overrides {
687 if !child_ids.contains(&strategy.child_id) {
688 return Err(SupervisorError::fatal_config(format!(
689 "child strategy override references unknown child: {}",
690 strategy.child_id
691 )));
692 }
693 if !overrides.insert(strategy.child_id.clone()) {
694 return Err(SupervisorError::fatal_config(format!(
695 "duplicate child strategy override: {}",
696 strategy.child_id
697 )));
698 }
699 validate_restart_limit(strategy.restart_limit)?;
700 }
701 Ok(())
702}
703
704fn validate_task_roles(children: &[ChildSpec]) -> Result<(), SupervisorError> {
714 let child_ids = children
715 .iter()
716 .map(|child| child.id.clone())
717 .collect::<HashSet<_>>();
718
719 for child in children {
720 emit_role_conflict_warnings(child);
721 if child.task_role != Some(TaskRole::Sidecar) {
722 continue;
723 }
724
725 let sidecar_config = child.sidecar_config.as_ref().ok_or_else(|| {
726 SupervisorError::fatal_config(format!(
727 "sidecar child {} requires sidecar_config",
728 child.id
729 ))
730 })?;
731
732 if !child_ids.contains(&sidecar_config.primary_child_id) {
733 return Err(SupervisorError::fatal_config(format!(
734 "sidecar child {} references unknown primary_child_id {}",
735 child.id, sidecar_config.primary_child_id
736 )));
737 }
738
739 let primary_child = children
740 .iter()
741 .find(|candidate| candidate.id == sidecar_config.primary_child_id)
742 .ok_or_else(|| {
743 SupervisorError::fatal_config(format!(
744 "sidecar child {} references unknown primary_child_id {}",
745 child.id, sidecar_config.primary_child_id
746 ))
747 })?;
748
749 if primary_child.task_role == Some(TaskRole::Sidecar) {
750 return Err(SupervisorError::fatal_config(format!(
751 "sidecar child {} must not use another sidecar {} as primary_child_id",
752 child.id, sidecar_config.primary_child_id
753 )));
754 }
755 }
756
757 Ok(())
758}
759
760fn emit_role_conflict_warnings(child: &ChildSpec) {
770 for conflict in semantic_conflicts_for_child(child) {
771 tracing::warn!(
772 child_id = %conflict.child_id,
773 task_role = %conflict.task_role,
774 conflicting_field = %conflict.conflicting_field,
775 user_value = %conflict.user_value,
776 expected_semantic = %conflict.expected_semantic,
777 reason = %conflict.reason,
778 "task role semantic conflict"
779 );
780 }
781}
782
783fn validate_dynamic_policy(policy: DynamicSupervisorPolicy) -> Result<(), SupervisorError> {
793 if policy.child_limit == Some(0) {
794 return Err(SupervisorError::fatal_config(
795 "dynamic supervisor child_limit must be greater than zero",
796 ));
797 }
798 Ok(())
799}
800
801fn validate_pipeline_policy(spec: &SupervisorSpec) -> Result<(), SupervisorError> {
811 if spec.pipeline_journal_capacity == 0 {
812 return Err(SupervisorError::fatal_config(
813 "pipeline journal capacity must be greater than zero",
814 ));
815 }
816 if spec.pipeline_subscriber_capacity == 0 {
817 return Err(SupervisorError::fatal_config(
818 "pipeline subscriber capacity must be greater than zero",
819 ));
820 }
821 if spec.concurrent_restart_limit == 0 {
822 return Err(SupervisorError::fatal_config(
823 "concurrent restart limit must be greater than zero",
824 ));
825 }
826 if spec.restart_budget_config.window.is_zero() {
827 return Err(SupervisorError::fatal_config(
828 "restart budget window must be greater than zero",
829 ));
830 }
831 if spec.restart_budget_config.max_burst == 0 {
832 return Err(SupervisorError::fatal_config(
833 "restart budget max_burst must be greater than zero",
834 ));
835 }
836 if spec.restart_budget_config.recovery_rate_per_sec <= 0.0 {
837 return Err(SupervisorError::fatal_config(
838 "restart budget recovery_rate_per_sec must be greater than zero",
839 ));
840 }
841 Ok(())
842}
843
844fn validate_backpressure_config(config: &BackpressureConfig) -> Result<(), SupervisorError> {
854 if config.warn_threshold_pct == 0 || config.warn_threshold_pct > 100 {
855 return Err(SupervisorError::fatal_config(
856 "backpressure warn_threshold_pct must be between 1 and 100",
857 ));
858 }
859 if config.critical_threshold_pct == 0 || config.critical_threshold_pct > 100 {
860 return Err(SupervisorError::fatal_config(
861 "backpressure critical_threshold_pct must be between 1 and 100",
862 ));
863 }
864 if config.warn_threshold_pct >= config.critical_threshold_pct {
865 return Err(SupervisorError::fatal_config(
866 "backpressure warn_threshold_pct must be less than critical_threshold_pct",
867 ));
868 }
869 if config.window_secs == 0 {
870 return Err(SupervisorError::fatal_config(
871 "backpressure window_secs must be greater than zero",
872 ));
873 }
874 if config.audit_channel_capacity == 0 {
875 return Err(SupervisorError::fatal_config(
876 "backpressure audit_channel_capacity must be greater than zero",
877 ));
878 }
879 Ok(())
880}