Skip to main content

rustango_cms/notify/
worker.rs

1//! The outbox: enqueue on the request path, deliver off it.
2
3use chrono::Utc;
4use rustango::core::Column as _;
5use rustango::query::Q;
6use rustango::sql::{ExecError, FetcherPool as _, Pool};
7
8use super::model::{
9    backoff_secs, NotificationDelivery, NotificationTarget, MAX_ATTEMPTS, STATUS_FAILED,
10    STATUS_PENDING, STATUS_SENT,
11};
12use super::{channel_for, Delivery, Notification};
13
14/// Queue `msg` for every enabled target subscribed to its event.
15///
16/// Returns how many rows were written. Cheap and synchronous — one SELECT
17/// and N INSERTs — so it is safe on a request path; the network calls
18/// happen in [`drain`].
19///
20/// # Errors
21/// Driver / query failures. Callers on a visitor-facing path should log
22/// and carry on rather than fail the submission: the content is already
23/// saved, and losing the notification is better than losing the lead.
24pub async fn enqueue(
25    pool: &Pool,
26    msg: &Notification,
27    source_id: i64,
28) -> Result<usize, ExecError> {
29    let targets: Vec<NotificationTarget> = NotificationTarget::objects()
30        .where_(NotificationTarget::enabled.eq(true))
31        .fetch(pool)
32        .await?;
33    let payload = serde_json::to_string(msg).unwrap_or_else(|_| "{}".to_owned());
34    let mut n = 0;
35    for t in targets.iter().filter(|t| t.wants(&msg.event, source_id)) {
36        let mut row = NotificationDelivery {
37            id: rustango::sql::Auto::Unset,
38            target_id: t.id.get().copied().unwrap_or_default(),
39            event: msg.event.clone(),
40            payload: payload.clone(),
41            status: STATUS_PENDING.to_owned(),
42            attempts: 0,
43            last_error: String::new(),
44            next_attempt_at: Some(Utc::now()),
45            created_at: rustango::sql::Auto::Unset,
46            sent_at: None,
47        };
48        row.insert(pool).await?;
49        n += 1;
50    }
51    Ok(n)
52}
53
54/// Context the worker needs that is not in the database.
55pub struct DrainCtx<'a> {
56    pub mailer: Option<&'a dyn rustango::email::Mailer>,
57    pub mailer_from: &'a str,
58}
59
60/// What one drain pass did.
61#[derive(Debug, Default, PartialEq, Eq)]
62pub struct DrainReport {
63    pub sent: usize,
64    pub retried: usize,
65    pub failed: usize,
66}
67
68/// Send every delivery that is due, up to `limit`.
69///
70/// Call it from a scheduled task, a background loop, or a management verb.
71/// It takes no lock: two concurrent drains may both pick up a row and
72/// deliver twice. That is the deliberate trade — for notifications a rare
73/// duplicate is a much smaller harm than a lock that strands the queue
74/// when a worker dies holding it, and the alternative (`SELECT … FOR
75/// UPDATE SKIP LOCKED`) is unavailable on SQLite, which this CMS supports
76/// first-class.
77///
78/// # Errors
79/// Driver / query failures while reading the queue.
80pub async fn drain(pool: &Pool, ctx: &DrainCtx<'_>, limit: usize) -> Result<DrainReport, ExecError> {
81    let due: Vec<NotificationDelivery> = NotificationDelivery::objects()
82        .where_(due_at(Utc::now()))
83        .order_by(&[("created_at", false)])
84        .limit(limit as i64)
85        .fetch(pool)
86        .await?;
87
88    let mut report = DrainReport::default();
89    for mut row in due {
90        let target: Option<NotificationTarget> = NotificationTarget::objects()
91            .where_(NotificationTarget::id.eq(row.target_id))
92            .first(pool)
93            .await?;
94        let Some(target) = target.filter(|t| t.enabled) else {
95            // The destination was deleted or switched off after this was
96            // queued. Sending is meaningless; say so instead of retrying.
97            row.status = STATUS_FAILED.to_owned();
98            row.last_error = "target missing or disabled".to_owned();
99            row.save(pool).await?;
100            report.failed += 1;
101            continue;
102        };
103        let Some(channel) = channel_for(&target.kind) else {
104            row.status = STATUS_FAILED.to_owned();
105            row.last_error = format!("no channel registered for kind `{}`", target.kind);
106            row.save(pool).await?;
107            report.failed += 1;
108            continue;
109        };
110        let msg: Notification = match serde_json::from_str(&row.payload) {
111            Ok(m) => m,
112            Err(e) => {
113                row.status = STATUS_FAILED.to_owned();
114                row.last_error = format!("unreadable payload: {e}");
115                row.save(pool).await?;
116                report.failed += 1;
117                continue;
118            }
119        };
120
121        let config = target.config_map();
122        let secret = target.secret.clone().into_inner().to_string();
123        let d = Delivery {
124            config: &config,
125            secret: &secret,
126            mailer: ctx.mailer,
127            mailer_from: ctx.mailer_from,
128        };
129        row.attempts += 1;
130        match channel.deliver(&d, &msg).await {
131            Ok(()) => {
132                row.status = STATUS_SENT.to_owned();
133                row.sent_at = Some(Utc::now());
134                row.last_error = String::new();
135                row.next_attempt_at = None;
136                report.sent += 1;
137            }
138            Err(e) => {
139                row.last_error = e.message().chars().take(500).collect();
140                if e.is_permanent() || row.attempts >= MAX_ATTEMPTS {
141                    row.status = STATUS_FAILED.to_owned();
142                    row.next_attempt_at = None;
143                    report.failed += 1;
144                } else {
145                    row.next_attempt_at =
146                        Some(Utc::now() + chrono::Duration::seconds(backoff_secs(row.attempts)));
147                    report.retried += 1;
148                }
149            }
150        }
151        row.save(pool).await?;
152    }
153    Ok(report)
154}
155
156/// Put a failed delivery back in the queue — the admin's "try again".
157///
158/// # Errors
159/// Driver / query failures.
160pub async fn retry(pool: &Pool, delivery_id: i64) -> Result<bool, ExecError> {
161    let Some(mut row): Option<NotificationDelivery> = NotificationDelivery::objects()
162        .where_(NotificationDelivery::id.eq(delivery_id))
163        .first(pool)
164        .await?
165    else {
166        return Ok(false);
167    };
168    row.status = STATUS_PENDING.to_owned();
169    // Reset the counter: a human retrying after fixing the webhook wants a
170    // fresh budget, not the one attempt the old run had left.
171    row.attempts = 0;
172    row.last_error = String::new();
173    row.next_attempt_at = Some(Utc::now());
174    row.save(pool).await?;
175    Ok(true)
176}
177
178/// A tenant's drain in progress. `dirty` is set when a drain was asked for
179/// while this one was running; `wake` cuts short its wait for a retry.
180struct Slot {
181    dirty: bool,
182    wake: std::sync::Arc<rustango::__private_runtime::tokio::sync::Notify>,
183}
184
185/// Drains in progress, so a burst of submissions does not start a drain per
186/// request. Keyed by tenant, since each has its own pool.
187static IN_FLIGHT: std::sync::Mutex<Option<std::collections::HashMap<String, Slot>>> =
188    std::sync::Mutex::new(None);
189
190/// Start draining `key` and return its wake handle — or, when a drain is
191/// already running, mark it dirty, wake it, and return `None`.
192fn claim(key: &str) -> Option<std::sync::Arc<rustango::__private_runtime::tokio::sync::Notify>> {
193    let mut g = IN_FLIGHT.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
194    let map = g.get_or_insert_with(Default::default);
195    if let Some(slot) = map.get_mut(key) {
196        slot.dirty = true;
197        slot.wake.notify_one();
198        return None;
199    }
200    let wake = std::sync::Arc::default();
201    map.insert(
202        key.to_owned(),
203        Slot {
204            dirty: false,
205            wake: std::sync::Arc::clone(&wake),
206        },
207    );
208    Some(wake)
209}
210
211/// Finish a pass: `true` keeps the claim for another one. It is kept when a
212/// drain was requested during the pass — a row enqueued after the pass read
213/// the queue would otherwise wait for the next submission — or when
214/// the caller still has a retry to wait for.
215fn finish_pass(key: &str, waiting: bool) -> bool {
216    let mut g = IN_FLIGHT.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
217    let Some(map) = g.as_mut() else { return false };
218    let Some(slot) = map.get_mut(key) else { return false };
219    if std::mem::take(&mut slot.dirty) || waiting {
220        return true;
221    }
222    map.remove(key);
223    false
224}
225
226/// Pending and due at `now`. Filtered in SQL: selecting the oldest
227/// pending rows and skipping backed-off ones in Rust let a window full of
228/// retries (a webhook that is down) starve newer rows that were due.
229fn due_at(now: chrono::DateTime<Utc>) -> Q {
230    Q::eq("status", STATUS_PENDING)
231        .and(Q::is_null("next_attempt_at").or(Q::lte("next_attempt_at", now)))
232}
233
234/// When the earliest pending delivery is due, if any is. Two lookups, not
235/// one `ORDER BY next_attempt_at`: where NULLs sort differs per dialect.
236async fn next_due(pool: &Pool) -> Result<Option<chrono::DateTime<Utc>>, ExecError> {
237    let now = Utc::now();
238    if NotificationDelivery::objects().where_(due_at(now)).first(pool).await?.is_some() {
239        return Ok(Some(now));
240    }
241    let later: Option<NotificationDelivery> = NotificationDelivery::objects()
242        .where_(NotificationDelivery::status.eq(STATUS_PENDING.to_owned()))
243        .where_(Q::gt("next_attempt_at", now))
244        .order_by(&[("next_attempt_at", false)])
245        .first(pool)
246        .await?;
247    Ok(later.and_then(|r| r.next_attempt_at))
248}
249
250/// Drain in the background, off the request path.
251///
252/// Called right after [`enqueue`] so a notification normally goes out in the
253/// same second it was queued, without the visitor waiting on Slack. It
254/// drains everything *due*, not just what this request added, so a retry
255/// queued earlier rides along.
256///
257/// The task stays up while retries are pending, sleeping until the next
258/// one is due, so a backed-off delivery goes out without new traffic. It
259/// ends once the queue has nothing pending: a retry never waits longer than
260/// the last backoff step, and a delivery gives up after [`MAX_ATTEMPTS`].
261pub fn spawn_drain(
262    pool: rustango::sql::Pool,
263    key: String,
264    mailer: Option<std::sync::Arc<dyn rustango::email::Mailer>>,
265    mailer_from: String,
266) {
267    let Some(wake) = claim(&key) else {
268        return; // one already running for this tenant; it goes round again
269    };
270    rustango::__private_runtime::tokio::spawn(async move {
271        let ctx = DrainCtx {
272            mailer: mailer.as_deref(),
273            mailer_from: &mailer_from,
274        };
275        loop {
276            // A failed pass or lookup does not wait: retrying a broken
277            // database on a timer would only spin.
278            let due = match drain(&pool, &ctx, 50).await {
279                Ok(_) => next_due(&pool).await.unwrap_or_else(|e| {
280                    tracing::warn!(target: "rustango_cms::notify", error = %e, "notification queue lookup failed");
281                    None
282                }),
283                Err(e) => {
284                    tracing::warn!(target: "rustango_cms::notify", error = %e, "notification drain failed");
285                    None
286                }
287            };
288            if !finish_pass(&key, due.is_some()) {
289                break;
290            }
291            if let Some(at) = due {
292                let wait = (at - Utc::now()).to_std().unwrap_or_default();
293                // A new submission wakes the wait early; timing out is the
294                // normal path.
295                let _ = rustango::__private_runtime::tokio::time::timeout(wait, wake.notified()).await;
296            }
297        }
298    });
299}
300
301#[cfg(test)]
302mod tests {
303    use super::{claim, finish_pass};
304
305    #[test]
306    fn a_request_during_a_pass_runs_another() {
307        let key = "test-dirty";
308        let wake = claim(key).expect("first claim starts a drain");
309        assert!(claim(key).is_none(), "second claim joins the running one");
310        assert!(finish_pass(key, false), "the joined request gets its pass");
311        assert!(!finish_pass(key, false), "nothing more asked: released");
312        assert!(claim(key).is_some(), "released keys can be claimed again");
313        assert!(!finish_pass(key, false));
314        drop(wake);
315    }
316
317    #[test]
318    fn a_pending_retry_keeps_the_claim() {
319        let key = "test-waiting";
320        let _wake = claim(key).expect("claim");
321        assert!(finish_pass(key, true), "a retry to wait for keeps the claim");
322        assert!(claim(key).is_none(), "still held while waiting");
323        assert!(finish_pass(key, false), "the request made while waiting runs");
324        assert!(!finish_pass(key, false));
325    }
326
327    #[tokio::test]
328    async fn a_new_request_wakes_the_wait() {
329        let key = "test-wake";
330        let wake = claim(key).expect("claim");
331        assert!(finish_pass(key, true));
332        assert!(claim(key).is_none());
333        // notify_one stored a permit, so this returns at once rather than
334        // sitting out the retry's backoff.
335        tokio::time::timeout(std::time::Duration::from_secs(5), wake.notified())
336            .await
337            .expect("woken, not timed out");
338        assert!(finish_pass(key, false));
339        assert!(!finish_pass(key, false));
340    }
341}