1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
// Vigil
//
// Microservices Status Page
// Copyright: 2018, Valerian Saliou <valerian@valeriansaliou.name>
// License: Mozilla Public License v2.0 (MPL v2.0)
use std::thread;
use std::time::Duration;
use crate::config::config::ConfigNotify;
use crate::prober::status::Status;
const DISPATCH_TRY_WAIT_SECONDS: u64 = 2;
const DISPATCH_TRY_ATTEMPT_TIMES: u8 = 3;
pub const DISPATCH_TIMEOUT_SECONDS: u64 = 10;
pub struct Notification<'a> {
pub status: &'a Status,
pub time: String,
pub replicas: Vec<&'a str>,
pub changed: bool,
pub escalated: Option<u16>,
pub startup: bool,
}
pub trait GenericNotifier {
fn attempt(notify: &ConfigNotify, notification: &Notification) -> Result<(), bool>;
fn can_notify(notify: &ConfigNotify, notification: &Notification) -> bool;
fn name() -> &'static str;
}
impl<'a> Notification<'a> {
pub fn dispatch<N: GenericNotifier>(
notify: &ConfigNotify,
notification: &Notification,
) -> Result<(), bool> {
if N::can_notify(notify, notification) == true {
info!(
"dispatch {} notification for status: {:?} and replicas: {:?}",
N::name(),
notification.status,
notification.replicas
);
for try_index in 1..(DISPATCH_TRY_ATTEMPT_TIMES + 1) {
debug!(
"dispatch {} notification attempt: #{}",
N::name(),
try_index
);
// Hold on for next try
if try_index > 1 {
thread::sleep(Duration::from_secs(DISPATCH_TRY_WAIT_SECONDS))
}
// Attempt notification dispatch
if N::attempt(notify, notification).is_ok() == true {
debug!("dispatched notification to provider: {}", N::name());
return Ok(());
}
}
error!("failed dispatching notification to provider: {}", N::name());
return Err(true);
}
debug!("did not dispatch notification to provider: {}", N::name());
Err(false)
}
pub fn expected(&self, reminders_only: bool) -> bool {
// Notification may not be expected if status has changed, but we only want to receive \
// reminders on this specific notifier channel.
if reminders_only == false || (reminders_only == true && self.changed == false) {
true
} else {
false
}
}
pub fn escalated_for(&self, target_index: usize) -> bool {
// In case reminder escalation is enabled and counter is currently active, check if \
// the passed target index is within range of the current escalation count. As the \
// outage persists and the escalation count increments, then the notifier will tend to \
// notify all targets in range at each reminder tick, while on first reminder only the \
// first target will get the high-priority alert (others will still get reminded but \
// under a normal priority alert). This can be used to page (ie. wake-up at night) \
// the defined always-on-call person first, and then if that person does not respond or \
// does not fix the issue in due time, then other targets will get progressively paged \
// one after the other as the outage persists.
if self.changed == false {
if let Some(escalated) = self.escalated {
// Escalation is enabled, only escalate to target if in range
target_index < escalated as usize
} else {
// Escalation is disabled, consider all targets as escalated
true
}
} else {
// Status did change, nothing to escalate
false
}
}
}