second_brain_api/
alert.rs1use std::time::Duration;
5
6use reqwest::StatusCode;
7
8use second_brain_core::config::AlertConfig;
9use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel};
10
11use crate::health::Transition;
12
13const FIRST_RETRY: Duration = Duration::from_secs(1);
14const LAST_RETRY: Duration = Duration::from_secs(5 * 60);
15
16fn payload(transition: &Transition, machine: &str) -> serde_json::Value {
17 serde_json::json!({
18 "state": transition.state,
19 "reason": transition.reason,
20 "machine": machine,
21 "at": transition.at,
22 })
23}
24
25pub fn start(config: &AlertConfig, machine: String) -> Option<UnboundedSender<Transition>> {
28 let url = config.webhook_url.clone()?;
29 let (tx, rx) = unbounded_channel();
30 tokio::spawn(deliver(url, machine, rx));
31 Some(tx)
32}
33
34async fn deliver(url: String, machine: String, mut transitions: UnboundedReceiver<Transition>) {
35 let client = match reqwest::Client::builder()
36 .timeout(Duration::from_secs(10))
37 .build()
38 {
39 Ok(c) => c,
40 Err(e) => {
41 tracing::error!("alert webhook disabled, cannot build an HTTP client: {e}");
42 return;
43 }
44 };
45 while let Some(mut transition) = transitions.recv().await {
46 let mut wait = FIRST_RETRY;
47 while post(&client, &url, &payload(&transition, &machine)).await == Posted::Retry {
48 tokio::select! {
49 newer = transitions.recv() => match newer {
50 Some(newer) => {
51 transition = newer;
52 while let Ok(newest) = transitions.try_recv() {
53 transition = newest;
54 }
55 }
56 None => return,
57 },
58 _ = tokio::time::sleep(wait) => wait = (wait * 2).min(LAST_RETRY),
59 }
60 }
61 }
62}
63
64#[derive(PartialEq)]
65enum Posted {
66 Done,
67 Retry,
68}
69
70async fn post(client: &reqwest::Client, url: &str, body: &serde_json::Value) -> Posted {
72 match client.post(url).json(body).send().await {
73 Ok(r) if r.status().is_success() => Posted::Done,
74 Ok(r) if r.status().is_client_error() && r.status() != StatusCode::TOO_MANY_REQUESTS => {
75 tracing::warn!("alert webhook refused the post with {}", r.status());
76 Posted::Done
77 }
78 Ok(r) => {
79 tracing::warn!("alert webhook answered {}; retrying", r.status());
80 Posted::Retry
81 }
82 Err(e) => {
83 tracing::warn!("alert webhook failed: {e}; retrying");
84 Posted::Retry
85 }
86 }
87}