Skip to main content

rushls_common/hooks/
dispatch.rs

1//! Draining one hook's queue, one request per subject at a time.
2
3use std::{
4    sync::{Arc, atomic::Ordering},
5    time::Duration,
6};
7
8use http::StatusCode;
9use tokio::{
10    sync::{mpsc, watch},
11    task::JoinSet,
12};
13
14use crate::outbound::{HttpClient, OutboundError, Response};
15
16use crate::hooks::{
17    CONTENT_TYPE, Dispatcher, Dispatchers, Envelope, HookObserver, Loss, Occurrence, Queue, Shared,
18};
19
20/// First wait between attempts, doubling from there.
21///
22/// Not configurable: the shape of a backoff curve is an implementation detail,
23/// while how long an operator is willing to keep trying is `maximum_attempts`.
24const INITIAL_BACKOFF: Duration = Duration::from_millis(250);
25const MAXIMUM_BACKOFF: Duration = Duration::from_secs(30);
26
27/// One hook's envelope type, which two type parameters would otherwise repeat.
28type Item<E> = Envelope<<E as Occurrence>::Subject, <E as Occurrence>::Kind>;
29type Pending<E> = Queue<<E as Occurrence>::Subject, <E as Occurrence>::Kind>;
30
31impl<E: Occurrence> Dispatchers<E> {
32    /// Runs every configured hook until `stop`, then drains what it can.
33    pub async fn run(self, mut stop: watch::Receiver<bool>) {
34        let mut hooks = JoinSet::new();
35        for dispatcher in self.hooks {
36            // A destination that configured its own identity or trust roots
37            // brought its own client; everything else shares the process pool,
38            // which is what keeps one TLS setup and one connection cache for
39            // the ordinary case.
40            let client = dispatcher
41                .shared
42                .config
43                .client
44                .clone()
45                .unwrap_or_else(|| self.client.clone());
46            hooks.spawn(run_hook(
47                dispatcher,
48                client,
49                self.drain_timeout,
50                Arc::clone(&self.observer),
51                stop.clone(),
52            ));
53        }
54        if hooks.is_empty() {
55            // Nothing configured: wait for shutdown rather than returning, so a
56            // caller that joins this task is not woken immediately.
57            let _ = stop.wait_for(|stopped| *stopped).await;
58            return;
59        }
60        while hooks.join_next().await.is_some() {}
61    }
62}
63
64async fn run_hook<E: Occurrence>(
65    dispatcher: Dispatcher<E>,
66    client: HttpClient,
67    drain_timeout: Duration,
68    observer: Arc<dyn HookObserver<E::Kind>>,
69    mut stop: watch::Receiver<bool>,
70) {
71    let Dispatcher {
72        shared,
73        mut ingress,
74    } = dispatcher;
75    let mut queue = Pending::<E>::new(shared.config.queue_capacity);
76    let mut sending = JoinSet::new();
77
78    loop {
79        drain_ingress(&shared, &mut ingress, &mut queue);
80        start_available(&shared, &client, &observer, &mut queue, &mut sending);
81
82        tokio::select! {
83          biased;
84
85          changed = stop.changed() => {
86            if changed.is_err() || *stop.borrow() {
87              break;
88            }
89          }
90
91          Some(finished) = sending.join_next(), if !sending.is_empty() => {
92            release(&shared, &mut queue, finished.ok());
93          }
94
95          Some(envelope) = ingress.recv() => {
96            accept(&shared, &mut queue, envelope);
97          }
98        }
99    }
100
101    drain(
102        shared,
103        ingress,
104        queue,
105        client,
106        drain_timeout,
107        observer,
108        sending,
109    )
110    .await;
111}
112
113/// Moves all immediately available producer work into the dispatcher-owned
114/// ordering queue. Network progress is deliberately irrelevant here.
115fn drain_ingress<E: Occurrence>(
116    shared: &Shared<E>,
117    ingress: &mut mpsc::Receiver<Item<E>>,
118    queue: &mut Pending<E>,
119) {
120    while let Ok(envelope) = ingress.try_recv() {
121        accept(shared, queue, envelope);
122    }
123}
124
125fn accept<E: Occurrence>(shared: &Shared<E>, queue: &mut Pending<E>, envelope: Item<E>) {
126    shared.meters.ingress_depth.fetch_sub(1, Ordering::Relaxed);
127    if queue.push(envelope) {
128        shared.meters.record_loss(Loss::Overflow);
129    }
130    shared
131        .meters
132        .queue_depth
133        .store(queue.queued(), Ordering::Relaxed);
134}
135
136/// Starts requests for as many idle subjects as the concurrency limit allows.
137fn start_available<E: Occurrence>(
138    shared: &Arc<Shared<E>>,
139    client: &HttpClient,
140    observer: &Arc<dyn HookObserver<E::Kind>>,
141    queue: &mut Pending<E>,
142    sending: &mut JoinSet<E::Subject>,
143) {
144    while sending.len() < shared.config.maximum_in_flight {
145        let Some(envelope) = queue.take_ready() else {
146            break;
147        };
148        sending.spawn(send(
149            Arc::clone(shared),
150            client.clone(),
151            Arc::clone(observer),
152            envelope,
153        ));
154    }
155    shared
156        .meters
157        .queue_depth
158        .store(queue.queued(), Ordering::Relaxed);
159    shared
160        .meters
161        .in_flight
162        .store(sending.len(), Ordering::Relaxed);
163}
164
165/// Lets a subject's next event through once its predecessor has finished.
166fn release<E: Occurrence>(
167    shared: &Arc<Shared<E>>,
168    queue: &mut Pending<E>,
169    subject: Option<E::Subject>,
170) {
171    if let Some(subject) = subject {
172        queue.finish(&subject);
173    }
174    shared.meters.in_flight.fetch_sub(1, Ordering::Relaxed);
175}
176
177/// Delivers one event, retrying until it lands or the attempts run out.
178///
179/// Retries happen here rather than by re-queuing, so the subject's slot stays
180/// held for the whole thing. That is deliberate: releasing it between attempts
181/// would let the next event overtake the one being retried, which is exactly
182/// the reordering the per-subject rule exists to prevent. The cost is that one
183/// unreachable endpoint stalls that subject for up to
184/// `maximum_attempts` × backoff.
185async fn send<E: Occurrence>(
186    shared: Arc<Shared<E>>,
187    client: HttpClient,
188    observer: Arc<dyn HookObserver<E::Kind>>,
189    envelope: Item<E>,
190) -> E::Subject {
191    let mut backoff = INITIAL_BACKOFF;
192
193    for attempt in 1..=shared.config.maximum_attempts {
194        // Timestamp the attempt, not the occurrence: queued events can be old,
195        // while receivers must still be able to enforce a short replay window.
196        let headers =
197            shared
198                .config
199                .signing_secret
200                .as_ref()
201                .map_or_else(http::HeaderMap::new, |secret| {
202                    secret.headers(
203                        &envelope.id,
204                        time::OffsetDateTime::now_utc().unix_timestamp(),
205                        &envelope.body,
206                    )
207                });
208        let result = client
209            .post_with_headers(
210                &shared.config.endpoint,
211                CONTENT_TYPE,
212                shared.config.bearer.as_ref(),
213                envelope.body.clone(),
214                &headers,
215            )
216            .await;
217
218        match verdict(&result) {
219            Verdict::Delivered => {
220                shared.meters.delivered.fetch_add(1, Ordering::Relaxed);
221                return envelope.subject;
222            }
223            Verdict::Rejected => {
224                report(&shared, &observer, &envelope, Loss::Rejected, &result);
225                return envelope.subject;
226            }
227            Verdict::Retry if attempt == shared.config.maximum_attempts => {
228                report(&shared, &observer, &envelope, Loss::Exhausted, &result);
229                return envelope.subject;
230            }
231            Verdict::Retry => {
232                shared.meters.retried.fetch_add(1, Ordering::Relaxed);
233                // An endpoint asking to be left alone knows better than the backoff
234                // curve does, so its answer wins — but only up to the cap, since it is
235                // also the party currently misbehaving.
236                let wait = result
237                    .as_ref()
238                    .ok()
239                    .and_then(|response| response.retry_after(MAXIMUM_BACKOFF))
240                    .unwrap_or(backoff);
241                tokio::time::sleep(wait).await;
242                backoff = (backoff * 2).min(MAXIMUM_BACKOFF);
243            }
244        }
245    }
246
247    envelope.subject
248}
249
250enum Verdict {
251    Delivered,
252    /// Worth another attempt: the endpoint or the path to it is unwell.
253    Retry,
254    /// Retrying cannot help.
255    Rejected,
256}
257
258fn verdict(result: &Result<Response, OutboundError>) -> Verdict {
259    match result {
260        Ok(response) if response.status.is_success() => Verdict::Delivered,
261        Ok(response) => match response.status {
262            // Transient by definition, plus the two the specifications reserve for
263            // "ask again": a request timeout and too-early.
264            StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_EARLY | StatusCode::TOO_MANY_REQUESTS => {
265                Verdict::Retry
266            }
267            status if status.is_server_error() => Verdict::Retry,
268            // Any other 4xx is the consumer saying the request itself is wrong.
269            // Retrying would move a poison event to the front of this subject's queue
270            // for the whole backoff, delaying everything behind it.
271            _ => Verdict::Rejected,
272        },
273        Err(
274            OutboundError::Timeout | OutboundError::Unreachable(_) | OutboundError::Response(_),
275        ) => Verdict::Retry,
276        // An endpoint answering an event POST with a body this large is
277        // misconfigured, and every retry pays to download it again.
278        Err(OutboundError::ResponseTooLarge { .. }) => Verdict::Rejected,
279    }
280}
281
282fn report<E: Occurrence>(
283    shared: &Arc<Shared<E>>,
284    observer: &Arc<dyn HookObserver<E::Kind>>,
285    envelope: &Item<E>,
286    loss: Loss,
287    result: &Result<Response, OutboundError>,
288) {
289    shared.meters.record_loss(loss);
290    observer.event_dropped(
291        &shared.config.name,
292        &envelope.id,
293        envelope.kind,
294        loss,
295        &match result {
296            Ok(response) => format!("HTTP {}", response.status),
297            Err(error) => error.to_string(),
298        },
299    );
300}
301
302/// Finishes what is in flight, then what is queued, until the deadline.
303async fn drain<E: Occurrence>(
304    shared: Arc<Shared<E>>,
305    mut ingress: mpsc::Receiver<Item<E>>,
306    mut queue: Pending<E>,
307    client: HttpClient,
308    timeout: Duration,
309    observer: Arc<dyn HookObserver<E::Kind>>,
310    mut sending: JoinSet<E::Subject>,
311) {
312    let deadline = tokio::time::Instant::now() + timeout;
313
314    let drained = tokio::time::timeout_at(deadline, async {
315        loop {
316            drain_ingress(&shared, &mut ingress, &mut queue);
317            start_available(&shared, &client, &observer, &mut queue, &mut sending);
318            let Some(finished) = sending.join_next().await else {
319                break;
320            };
321            release(&shared, &mut queue, finished.ok());
322        }
323    })
324    .await;
325
326    if drained.is_err() {
327        sending.abort_all();
328        let mut unknown = 0;
329        // Reap after cancellation rather than counting `JoinSet::len`: a task may
330        // have completed at the deadline without yet being joined, in which case
331        // its delivered or loss counter is already definitive.
332        while let Some(finished) = sending.join_next().await {
333            if finished.is_err() {
334                unknown += 1;
335            }
336        }
337        shared
338            .meters
339            .outcome_unknown_shutdown
340            .fetch_add(unknown as u64, Ordering::Relaxed);
341        if unknown > 0 {
342            observer.delivery_outcomes_unknown(&shared.config.name, unknown);
343        }
344    }
345    // Whatever is still waiting in either bounded stage is gone: it lives in
346    // memory only, and the process is on its way out.
347    let mut lost = queue.clear();
348    while ingress.try_recv().is_ok() {
349        shared.meters.ingress_depth.fetch_sub(1, Ordering::Relaxed);
350        lost += 1;
351    }
352    shared.meters.queue_depth.store(0, Ordering::Relaxed);
353    shared.meters.in_flight.store(0, Ordering::Relaxed);
354    for _ in 0..lost {
355        shared.meters.record_loss(Loss::Shutdown);
356    }
357    if lost > 0 {
358        observer.events_abandoned(&shared.config.name, lost);
359    }
360}