Skip to main content

second_brain_api/
alert.rs

1//! Posts each health state change to `[alert] webhook_url`, retrying a
2//! failed post until it lands or a newer state replaces it.
3
4use std::time::Duration;
5
6use reqwest::StatusCode;
7
8use second_brain_core::config::AlertConfig;
9use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel};
10
11use crate::health::Transition;
12
13const FIRST_RETRY: Duration = Duration::from_secs(1);
14const LAST_RETRY: Duration = Duration::from_secs(5 * 60);
15
16fn payload(transition: &Transition, machine: &str) -> serde_json::Value {
17    serde_json::json!({
18        "state": transition.state,
19        "reason": transition.reason,
20        "machine": machine,
21        "at": transition.at,
22    })
23}
24
25/// The sender to hand `Health::new`, or None when no webhook is set, so
26/// nothing is ever queued or sent.
27pub fn start(config: &AlertConfig, machine: String) -> Option<UnboundedSender<Transition>> {
28    let url = config.webhook_url.clone()?;
29    let (tx, rx) = unbounded_channel();
30    tokio::spawn(deliver(url, machine, rx));
31    Some(tx)
32}
33
34async fn deliver(url: String, machine: String, mut transitions: UnboundedReceiver<Transition>) {
35    let client = match reqwest::Client::builder()
36        .timeout(Duration::from_secs(10))
37        .build()
38    {
39        Ok(c) => c,
40        Err(e) => {
41            tracing::error!("alert webhook disabled, cannot build an HTTP client: {e}");
42            return;
43        }
44    };
45    while let Some(mut transition) = transitions.recv().await {
46        let mut wait = FIRST_RETRY;
47        while post(&client, &url, &payload(&transition, &machine)).await == Posted::Retry {
48            tokio::select! {
49                newer = transitions.recv() => match newer {
50                    Some(newer) => {
51                        transition = newer;
52                        while let Ok(newest) = transitions.try_recv() {
53                            transition = newest;
54                        }
55                    }
56                    None => return,
57                },
58                _ = tokio::time::sleep(wait) => wait = (wait * 2).min(LAST_RETRY),
59            }
60        }
61    }
62}
63
64#[derive(PartialEq)]
65enum Posted {
66    Done,
67    Retry,
68}
69
70/// A 4xx other than 429 will not change on retry, so it is logged once.
71async fn post(client: &reqwest::Client, url: &str, body: &serde_json::Value) -> Posted {
72    match client.post(url).json(body).send().await {
73        Ok(r) if r.status().is_success() => Posted::Done,
74        Ok(r) if r.status().is_client_error() && r.status() != StatusCode::TOO_MANY_REQUESTS => {
75            tracing::warn!("alert webhook refused the post with {}", r.status());
76            Posted::Done
77        }
78        Ok(r) => {
79            tracing::warn!("alert webhook answered {}; retrying", r.status());
80            Posted::Retry
81        }
82        Err(e) => {
83            tracing::warn!("alert webhook failed: {e}; retrying");
84            Posted::Retry
85        }
86    }
87}