1use std::{
4 sync::{Arc, atomic::Ordering},
5 time::Duration,
6};
7
8use http::StatusCode;
9use tokio::{
10 sync::{mpsc, watch},
11 task::JoinSet,
12};
13
14use crate::outbound::{HttpClient, OutboundError, Response};
15
16use crate::hooks::{
17 CONTENT_TYPE, Dispatcher, Dispatchers, Envelope, HookObserver, Loss, Occurrence, Queue, Shared,
18};
19
20const INITIAL_BACKOFF: Duration = Duration::from_millis(250);
25const MAXIMUM_BACKOFF: Duration = Duration::from_secs(30);
26
27type Item<E> = Envelope<<E as Occurrence>::Subject, <E as Occurrence>::Kind>;
29type Pending<E> = Queue<<E as Occurrence>::Subject, <E as Occurrence>::Kind>;
30
31impl<E: Occurrence> Dispatchers<E> {
32 pub async fn run(self, mut stop: watch::Receiver<bool>) {
34 let mut hooks = JoinSet::new();
35 for dispatcher in self.hooks {
36 let client = dispatcher
41 .shared
42 .config
43 .client
44 .clone()
45 .unwrap_or_else(|| self.client.clone());
46 hooks.spawn(run_hook(
47 dispatcher,
48 client,
49 self.drain_timeout,
50 Arc::clone(&self.observer),
51 stop.clone(),
52 ));
53 }
54 if hooks.is_empty() {
55 let _ = stop.wait_for(|stopped| *stopped).await;
58 return;
59 }
60 while hooks.join_next().await.is_some() {}
61 }
62}
63
64async fn run_hook<E: Occurrence>(
65 dispatcher: Dispatcher<E>,
66 client: HttpClient,
67 drain_timeout: Duration,
68 observer: Arc<dyn HookObserver<E::Kind>>,
69 mut stop: watch::Receiver<bool>,
70) {
71 let Dispatcher {
72 shared,
73 mut ingress,
74 } = dispatcher;
75 let mut queue = Pending::<E>::new(shared.config.queue_capacity);
76 let mut sending = JoinSet::new();
77
78 loop {
79 drain_ingress(&shared, &mut ingress, &mut queue);
80 start_available(&shared, &client, &observer, &mut queue, &mut sending);
81
82 tokio::select! {
83 biased;
84
85 changed = stop.changed() => {
86 if changed.is_err() || *stop.borrow() {
87 break;
88 }
89 }
90
91 Some(finished) = sending.join_next(), if !sending.is_empty() => {
92 release(&shared, &mut queue, finished.ok());
93 }
94
95 Some(envelope) = ingress.recv() => {
96 accept(&shared, &mut queue, envelope);
97 }
98 }
99 }
100
101 drain(
102 shared,
103 ingress,
104 queue,
105 client,
106 drain_timeout,
107 observer,
108 sending,
109 )
110 .await;
111}
112
113fn drain_ingress<E: Occurrence>(
116 shared: &Shared<E>,
117 ingress: &mut mpsc::Receiver<Item<E>>,
118 queue: &mut Pending<E>,
119) {
120 while let Ok(envelope) = ingress.try_recv() {
121 accept(shared, queue, envelope);
122 }
123}
124
125fn accept<E: Occurrence>(shared: &Shared<E>, queue: &mut Pending<E>, envelope: Item<E>) {
126 shared.meters.ingress_depth.fetch_sub(1, Ordering::Relaxed);
127 if queue.push(envelope) {
128 shared.meters.record_loss(Loss::Overflow);
129 }
130 shared
131 .meters
132 .queue_depth
133 .store(queue.queued(), Ordering::Relaxed);
134}
135
136fn start_available<E: Occurrence>(
138 shared: &Arc<Shared<E>>,
139 client: &HttpClient,
140 observer: &Arc<dyn HookObserver<E::Kind>>,
141 queue: &mut Pending<E>,
142 sending: &mut JoinSet<E::Subject>,
143) {
144 while sending.len() < shared.config.maximum_in_flight {
145 let Some(envelope) = queue.take_ready() else {
146 break;
147 };
148 sending.spawn(send(
149 Arc::clone(shared),
150 client.clone(),
151 Arc::clone(observer),
152 envelope,
153 ));
154 }
155 shared
156 .meters
157 .queue_depth
158 .store(queue.queued(), Ordering::Relaxed);
159 shared
160 .meters
161 .in_flight
162 .store(sending.len(), Ordering::Relaxed);
163}
164
165fn release<E: Occurrence>(
167 shared: &Arc<Shared<E>>,
168 queue: &mut Pending<E>,
169 subject: Option<E::Subject>,
170) {
171 if let Some(subject) = subject {
172 queue.finish(&subject);
173 }
174 shared.meters.in_flight.fetch_sub(1, Ordering::Relaxed);
175}
176
177async fn send<E: Occurrence>(
186 shared: Arc<Shared<E>>,
187 client: HttpClient,
188 observer: Arc<dyn HookObserver<E::Kind>>,
189 envelope: Item<E>,
190) -> E::Subject {
191 let mut backoff = INITIAL_BACKOFF;
192
193 for attempt in 1..=shared.config.maximum_attempts {
194 let headers =
197 shared
198 .config
199 .signing_secret
200 .as_ref()
201 .map_or_else(http::HeaderMap::new, |secret| {
202 secret.headers(
203 &envelope.id,
204 time::OffsetDateTime::now_utc().unix_timestamp(),
205 &envelope.body,
206 )
207 });
208 let result = client
209 .post_with_headers(
210 &shared.config.endpoint,
211 CONTENT_TYPE,
212 shared.config.bearer.as_ref(),
213 envelope.body.clone(),
214 &headers,
215 )
216 .await;
217
218 match verdict(&result) {
219 Verdict::Delivered => {
220 shared.meters.delivered.fetch_add(1, Ordering::Relaxed);
221 return envelope.subject;
222 }
223 Verdict::Rejected => {
224 report(&shared, &observer, &envelope, Loss::Rejected, &result);
225 return envelope.subject;
226 }
227 Verdict::Retry if attempt == shared.config.maximum_attempts => {
228 report(&shared, &observer, &envelope, Loss::Exhausted, &result);
229 return envelope.subject;
230 }
231 Verdict::Retry => {
232 shared.meters.retried.fetch_add(1, Ordering::Relaxed);
233 let wait = result
237 .as_ref()
238 .ok()
239 .and_then(|response| response.retry_after(MAXIMUM_BACKOFF))
240 .unwrap_or(backoff);
241 tokio::time::sleep(wait).await;
242 backoff = (backoff * 2).min(MAXIMUM_BACKOFF);
243 }
244 }
245 }
246
247 envelope.subject
248}
249
250enum Verdict {
251 Delivered,
252 Retry,
254 Rejected,
256}
257
258fn verdict(result: &Result<Response, OutboundError>) -> Verdict {
259 match result {
260 Ok(response) if response.status.is_success() => Verdict::Delivered,
261 Ok(response) => match response.status {
262 StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_EARLY | StatusCode::TOO_MANY_REQUESTS => {
265 Verdict::Retry
266 }
267 status if status.is_server_error() => Verdict::Retry,
268 _ => Verdict::Rejected,
272 },
273 Err(
274 OutboundError::Timeout | OutboundError::Unreachable(_) | OutboundError::Response(_),
275 ) => Verdict::Retry,
276 Err(OutboundError::ResponseTooLarge { .. }) => Verdict::Rejected,
279 }
280}
281
282fn report<E: Occurrence>(
283 shared: &Arc<Shared<E>>,
284 observer: &Arc<dyn HookObserver<E::Kind>>,
285 envelope: &Item<E>,
286 loss: Loss,
287 result: &Result<Response, OutboundError>,
288) {
289 shared.meters.record_loss(loss);
290 observer.event_dropped(
291 &shared.config.name,
292 &envelope.id,
293 envelope.kind,
294 loss,
295 &match result {
296 Ok(response) => format!("HTTP {}", response.status),
297 Err(error) => error.to_string(),
298 },
299 );
300}
301
302async fn drain<E: Occurrence>(
304 shared: Arc<Shared<E>>,
305 mut ingress: mpsc::Receiver<Item<E>>,
306 mut queue: Pending<E>,
307 client: HttpClient,
308 timeout: Duration,
309 observer: Arc<dyn HookObserver<E::Kind>>,
310 mut sending: JoinSet<E::Subject>,
311) {
312 let deadline = tokio::time::Instant::now() + timeout;
313
314 let drained = tokio::time::timeout_at(deadline, async {
315 loop {
316 drain_ingress(&shared, &mut ingress, &mut queue);
317 start_available(&shared, &client, &observer, &mut queue, &mut sending);
318 let Some(finished) = sending.join_next().await else {
319 break;
320 };
321 release(&shared, &mut queue, finished.ok());
322 }
323 })
324 .await;
325
326 if drained.is_err() {
327 sending.abort_all();
328 let mut unknown = 0;
329 while let Some(finished) = sending.join_next().await {
333 if finished.is_err() {
334 unknown += 1;
335 }
336 }
337 shared
338 .meters
339 .outcome_unknown_shutdown
340 .fetch_add(unknown as u64, Ordering::Relaxed);
341 if unknown > 0 {
342 observer.delivery_outcomes_unknown(&shared.config.name, unknown);
343 }
344 }
345 let mut lost = queue.clear();
348 while ingress.try_recv().is_ok() {
349 shared.meters.ingress_depth.fetch_sub(1, Ordering::Relaxed);
350 lost += 1;
351 }
352 shared.meters.queue_depth.store(0, Ordering::Relaxed);
353 shared.meters.in_flight.store(0, Ordering::Relaxed);
354 for _ in 0..lost {
355 shared.meters.record_loss(Loss::Shutdown);
356 }
357 if lost > 0 {
358 observer.events_abandoned(&shared.config.name, lost);
359 }
360}