rushls_common/hooks/mod.rs
1//! Telling other systems what happened, without ever slowing this one down.
2//!
3//! # What is promised
4//!
5//! Best-effort, at-least-once while the process lives. Every event carries a
6//! stable id that retries reuse, so a consumer deduplicates on `source` + `id`.
7//! Events for one subject arrive in the order they occurred, though a prefix
8//! may be missing; nothing is promised across subjects.
9//!
10//! Events are lost by exceptional ingress saturation, queue overflow, permanent
11//! rejection, exhausted retries, or the drain deadline at shutdown, each counted
12//! separately because they mean different things. Nothing survives a crash.
13//!
14//! **Hooks are not a substitute for reading state.** Anything correctness-
15//! critical has to reconcile. If that ever stops being acceptable the answer is
16//! a durable outbox, not more retry settings.
17//!
18//! # Why it cannot stall the caller
19//!
20//! [`Hooks::deliver`] renders once, performs only bounded `try_send` operations,
21//! and returns — no waiting, no queue scans, no network. Callers invoke it from
22//! whatever task produced the event, often a media session's own, so anything
23//! slower would pause that work for whatever a remote endpoint felt like taking.
24//!
25//! # What an application supplies
26//!
27//! This crate owns the *transport*: CloudEvents rendering, per-subject
28//! ordering, drop-oldest overflow, retry with backoff, and a bounded drain at
29//! shutdown. It deliberately owns none of the *vocabulary*.
30//!
31//! An application implements [`Occurrence`] for its own event type, naming the
32//! [`Subject`] that ordering follows and the [`Kind`] that subscriptions
33//! filter on. Two applications therefore share the machinery while promising
34//! their consumers entirely different facts.
35
36mod dispatch;
37mod envelope;
38mod queue;
39mod signing;
40pub use signing::{InvalidSigningSecret, SigningSecret};
41
42#[cfg(test)]
43mod tests;
44
45use std::{
46 collections::BTreeSet,
47 hash::Hash,
48 sync::{
49 Arc,
50 atomic::{AtomicU64, AtomicUsize, Ordering},
51 },
52 time::Duration,
53};
54
55use tokio::sync::mpsc;
56
57use crate::outbound::{BearerToken, Endpoint, HttpClient};
58
59pub use envelope::{CONTENT_TYPE, Envelope, RenderError, Renderer};
60
61use queue::Queue;
62
63/// What delivery orders by: one subject's events keep their relative order.
64///
65/// Typically a stream or session identity. It is a hash key and a metrics-free
66/// value, so the bounds are only what the queue needs, plus a borrowed string
67/// for the CloudEvents `subject` attribute.
68///
69/// `Sync` because a hook's configuration and its in-flight envelopes are shared
70/// behind an `Arc` across the dispatcher's tasks.
71pub trait Subject: Clone + Eq + Hash + Send + Sync + 'static {
72 /// How this subject is spelled in the envelope.
73 fn as_str(&self) -> &str;
74}
75
76/// Which fact an event reports, and what a subscription list is written in.
77///
78/// `Display` is what appears in the CloudEvents `type`, so it is part of an
79/// application's public contract with its consumers.
80pub trait Kind: Copy + Eq + Ord + Hash + std::fmt::Display + Send + Sync + 'static {}
81
82impl<T> Kind for T where T: Copy + Eq + Ord + Hash + std::fmt::Display + Send + Sync + 'static {}
83
84/// One thing an application promises to report.
85///
86/// The vocabulary lives with the application that owns it. This crate never
87/// interprets `data`; it only stamps it with an id, a time, and a type.
88pub trait Occurrence: Send + 'static {
89 type Subject: Subject;
90 type Kind: Kind;
91
92 /// Prefix for every `type` an application emits, such as `rushls`.
93 ///
94 /// Deliberately not reverse-DNS, and deliberately not configurable. The
95 /// convention is a SHOULD in the specification and exists so producers
96 /// sharing a bus can be told apart — a job `source` already does, since that
97 /// is what an operator sets per deployment and what consumers deduplicate on.
98 /// `type` is also what a consumer routes on, so letting an operator change it
99 /// would break the very contract the name exists to keep.
100 const TYPE_PREFIX: &'static str;
101
102 fn kind(&self) -> Self::Kind;
103 fn subject(&self) -> Self::Subject;
104 /// The CloudEvents `data` member, rendered once per occurrence.
105 fn data(&self) -> serde_json::Value;
106}
107
108/// Where this crate reports its own failures.
109///
110/// A trait rather than a concrete sink because two applications count and log
111/// differently, and neither should have to adopt the other's observability
112/// model to reuse the delivery machinery. Every method has a default, so an
113/// application that wants none of it implements nothing.
114///
115/// Implementations are called from the dispatcher task and must not block.
116///
117/// Nothing reported here may be fed back into [`Hooks::deliver`]: a dead
118/// endpoint would then generate failure events addressed to the dead endpoint.
119///
120/// Generic over the application's [`Kind`] rather than taking a rendered
121/// string, so an application that already has a typed event vocabulary keeps it
122/// when reporting a failure against one.
123pub trait HookObserver<K: Kind>: Send + Sync + 'static {
124 /// An event will not be attempted again. `detail` is an HTTP status or a
125 /// transport error, and `event` is the CloudEvents id a consumer would quote.
126 fn event_dropped(&self, _hook: &str, _event: &str, _kind: K, _loss: Loss, _detail: &str) {}
127 /// Queued events were abandoned at the shutdown drain deadline.
128 fn events_abandoned(&self, _hook: &str, _dropped: usize) {}
129 /// Deliveries were in flight when the drain deadline passed, so they may have
130 /// reached the endpoint before being cancelled. Unknown rather than dropped.
131 fn delivery_outcomes_unknown(&self, _hook: &str, _count: usize) {}
132 /// An event could not be turned into bytes at all.
133 ///
134 /// A fault in this process rather than in delivery, so it names no hook:
135 /// nothing was addressed yet when it failed.
136 fn event_unrenderable(&self, _reason: &str) {}
137}
138
139/// Discards everything, for a caller that wants no reporting.
140#[derive(Clone, Copy, Debug, Default)]
141pub struct IgnoreHookEvents;
142
143impl<K: Kind> HookObserver<K> for IgnoreHookEvents {}
144
145/// One configured destination.
146#[derive(Clone, derive_more::Debug)]
147pub struct HookConfig<K> {
148 /// Identifies this hook in metrics and logs, so it must be unique.
149 pub name: Arc<str>,
150 pub endpoint: Endpoint,
151 /// Only these are delivered. Explicit rather than defaulting to everything,
152 /// so a consumer written today cannot be sent an event type added later.
153 pub events: BTreeSet<K>,
154 /// Capacity of both bounded stages. Ingress is normally empty; the
155 /// dispatcher-owned stage is the durable in-memory backlog that applies
156 /// drop-oldest while an endpoint is unhealthy.
157 pub queue_capacity: usize,
158 /// Distinct subjects that may be in flight at once. One request per subject
159 /// is the ordering rule, so this is also the concurrency.
160 pub maximum_in_flight: usize,
161 /// Attempts per event, the first included.
162 pub maximum_attempts: u32,
163 #[debug(skip)]
164 pub bearer: Option<BearerToken>,
165 /// Optional Standard Webhooks HMAC-SHA256 signing key.
166 pub signing_secret: Option<SigningSecret>,
167 /// A client of this destination's own, or `None` to use the shared one.
168 ///
169 /// Present only when a destination asked for something a shared pool
170 /// cannot give it: a client certificate, or an authority pinned in place
171 /// of the platform store. Both belong to *this* endpoint, and a pooled
172 /// connection reused across endpoints would present one service's identity
173 /// to another — so asking for either costs a pool, and only the endpoints
174 /// that ask pay for one.
175 #[debug(skip)]
176 pub client: Option<HttpClient>,
177}
178
179/// A destination that cannot deliver must fail before any dispatcher starts.
180#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)]
181#[error("hook `{hook}`: {reason}")]
182pub struct ConfigError {
183 pub hook: Arc<str>,
184 pub reason: &'static str,
185}
186
187impl<K> HookConfig<K> {
188 /// Checks transport invariants independently of the application's parser.
189 pub fn validate(&self) -> Result<(), ConfigError> {
190 let reason = if self.events.is_empty() {
191 Some("subscribes to no events")
192 } else if self.queue_capacity == 0 {
193 Some("queue capacity must be nonzero")
194 } else if self.queue_capacity > tokio::sync::Semaphore::MAX_PERMITS {
195 Some("queue capacity exceeds the channel limit")
196 } else if self.maximum_in_flight == 0 {
197 Some("maximum in-flight requests must be nonzero")
198 } else if self.maximum_attempts == 0 {
199 Some("maximum attempts must be nonzero")
200 } else {
201 None
202 };
203 match reason {
204 Some(reason) => Err(ConfigError {
205 hook: self.name.clone(),
206 reason,
207 }),
208 None => Ok(()),
209 }
210 }
211}
212
213/// Process-wide hook settings.
214#[derive(Clone, Debug)]
215pub struct HooksConfig<K> {
216 /// CloudEvents `source`; with the event id it identifies an occurrence.
217 ///
218 /// Must be stable across restarts. Nodes may share one, in which case
219 /// consumers see a single logical producer.
220 pub source: String,
221 /// Versions the whole event vocabulary as one family.
222 ///
223 /// Additive changes keep the version: consumers must ignore fields they
224 /// do not recognize. Removing or redefining a field bumps it, and a
225 /// subscription names the kind, so it receives every version of that kind.
226 pub schema_version: u32,
227 /// How long a shutdown waits for queued events before abandoning them.
228 pub drain_timeout: Duration,
229 pub hooks: Vec<HookConfig<K>>,
230}
231
232impl<K> HooksConfig<K> {
233 /// `source` has no sensible default: it identifies the deployment.
234 pub fn new(source: impl Into<String>) -> Self {
235 Self {
236 source: source.into(),
237 schema_version: 1,
238 drain_timeout: Duration::from_secs(5),
239 hooks: Vec::new(),
240 }
241 }
242}
243
244/// Why an event never reached its endpoint.
245///
246/// Spelled once, in [`Loss::as_str`], so an application reporting these does
247/// not restate the five names.
248#[derive(Clone, Copy, Debug, Eq, PartialEq, derive_more::Display)]
249#[display("{}", self.as_str())]
250pub enum Loss {
251 /// The dispatcher ingress channel was unavailable, so the new event could not
252 /// reach the queue that implements the normal drop-oldest policy.
253 Ingress,
254 /// The queue was full, so the oldest waiting event made room.
255 Overflow,
256 /// The endpoint refused it in a way retrying cannot fix.
257 Rejected,
258 /// Every attempt failed.
259 Exhausted,
260 /// Still queued when the drain deadline passed.
261 Shutdown,
262}
263
264impl Loss {
265 /// Stable enough to appear in a log or a metric label.
266 pub fn as_str(self) -> &'static str {
267 match self {
268 Self::Ingress => "ingress",
269 Self::Overflow => "overflow",
270 Self::Rejected => "rejected",
271 Self::Exhausted => "exhausted",
272 Self::Shutdown => "shutdown",
273 }
274 }
275}
276
277/// Delivery counters and current saturation for one hook.
278#[derive(Debug, Default)]
279pub struct HookMeters {
280 delivered: AtomicU64,
281 retried: AtomicU64,
282 ingress: AtomicU64,
283 overflow: AtomicU64,
284 rejected: AtomicU64,
285 exhausted: AtomicU64,
286 shutdown: AtomicU64,
287 outcome_unknown_shutdown: AtomicU64,
288 /// Events for a kind this hook did not subscribe to are not losses.
289 filtered: AtomicU64,
290 ingress_depth: AtomicUsize,
291 queue_depth: AtomicUsize,
292 in_flight: AtomicUsize,
293}
294
295impl HookMeters {
296 fn record_loss(&self, loss: Loss) {
297 match loss {
298 Loss::Ingress => &self.ingress,
299 Loss::Overflow => &self.overflow,
300 Loss::Rejected => &self.rejected,
301 Loss::Exhausted => &self.exhausted,
302 Loss::Shutdown => &self.shutdown,
303 }
304 .fetch_add(1, Ordering::Relaxed);
305 }
306
307 pub fn snapshot(&self) -> HookSnapshot {
308 HookSnapshot {
309 delivered: self.delivered.load(Ordering::Relaxed),
310 retried: self.retried.load(Ordering::Relaxed),
311 ingress: self.ingress.load(Ordering::Relaxed),
312 overflow: self.overflow.load(Ordering::Relaxed),
313 rejected: self.rejected.load(Ordering::Relaxed),
314 exhausted: self.exhausted.load(Ordering::Relaxed),
315 shutdown: self.shutdown.load(Ordering::Relaxed),
316 outcome_unknown_shutdown: self.outcome_unknown_shutdown.load(Ordering::Relaxed),
317 filtered: self.filtered.load(Ordering::Relaxed),
318 ingress_depth: self.ingress_depth.load(Ordering::Relaxed),
319 ingress_capacity: 0,
320 queue_depth: self.queue_depth.load(Ordering::Relaxed),
321 queue_capacity: 0,
322 in_flight: self.in_flight.load(Ordering::Relaxed),
323 }
324 }
325}
326
327/// One hook's operational numbers at an instant.
328///
329/// The losses are kept apart rather than summed into one "failed" counter: an
330/// endpoint refusing an event, a queue overflowing, and a shutdown cutting a
331/// drain short call for three different responses from whoever is looking.
332///
333/// How these are exported is the application's business — this crate has no
334/// opinion about Prometheus — so the field names are the whole contract.
335#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
336pub struct HookSnapshot {
337 pub delivered: u64,
338 pub retried: u64,
339 pub ingress: u64,
340 pub overflow: u64,
341 pub rejected: u64,
342 pub exhausted: u64,
343 pub shutdown: u64,
344 pub outcome_unknown_shutdown: u64,
345 pub filtered: u64,
346 pub ingress_depth: usize,
347 /// What an operator configured, rather than something measured.
348 pub ingress_capacity: usize,
349 pub queue_depth: usize,
350 /// What an operator configured, rather than something measured.
351 pub queue_capacity: usize,
352 pub in_flight: usize,
353}
354
355/// One hook's queue, settings, and counters, shared with its dispatcher.
356#[derive(Debug)]
357struct Shared<E: Occurrence> {
358 config: HookConfig<E::Kind>,
359 /// The producer only performs a bounded `try_send`; ordering and eviction are
360 /// owned exclusively by the dispatcher on the receiving side.
361 ingress: mpsc::Sender<Envelope<E::Subject, E::Kind>>,
362 meters: HookMeters,
363}
364
365struct Dispatcher<E: Occurrence> {
366 shared: Arc<Shared<E>>,
367 ingress: mpsc::Receiver<Envelope<E::Subject, E::Kind>>,
368}
369
370/// The enqueue side, held by whatever observes application events.
371#[derive(derive_more::Debug)]
372pub struct Hooks<E: Occurrence> {
373 renderer: Renderer<E>,
374 shared: Arc<Vec<Arc<Shared<E>>>>,
375 #[debug(skip)]
376 observer: Arc<dyn HookObserver<E::Kind>>,
377}
378
379// Derived `Clone` would demand `E: Clone`, which the event type has no reason
380// to be: everything here is behind an `Arc` or is itself cheap to clone.
381impl<E: Occurrence> Clone for Hooks<E> {
382 fn clone(&self) -> Self {
383 Self {
384 renderer: self.renderer.clone(),
385 shared: Arc::clone(&self.shared),
386 observer: Arc::clone(&self.observer),
387 }
388 }
389}
390
391impl<E: Occurrence> Hooks<E> {
392 /// Renders an event once and offers it to every subscribed hook.
393 ///
394 /// Never blocks and never fails: rendering happens once even with several
395 /// hooks, then each subscribed dispatcher receives the immutable envelope
396 /// through a bounded `try_send`. Its ordering queue drops the oldest event
397 /// when the endpoint cannot keep up.
398 pub fn deliver(&self, event: &E) {
399 if self.shared.is_empty() {
400 return;
401 }
402 let kind = event.kind();
403 // Rendered only if somebody wants it, so a node with a narrow subscription
404 // pays nothing for the events nobody asked for.
405 if !self
406 .shared
407 .iter()
408 .any(|hook| hook.config.events.contains(&kind))
409 {
410 for hook in self.shared.iter() {
411 hook.meters.filtered.fetch_add(1, Ordering::Relaxed);
412 }
413 return;
414 }
415 let envelope = match self.renderer.render(event) {
416 Ok(envelope) => envelope,
417 // Rendering fails only if the clock or the serializer does, which is a
418 // process-level fault rather than a delivery one.
419 Err(error) => {
420 self.observer.event_unrenderable(&error.to_string());
421 return;
422 }
423 };
424
425 for hook in self.shared.iter() {
426 if !hook.config.events.contains(&kind) {
427 hook.meters.filtered.fetch_add(1, Ordering::Relaxed);
428 continue;
429 }
430 // Incremented before publishing so a receiver cannot observe the item
431 // before the gauge does. A failed send rolls it back.
432 hook.meters.ingress_depth.fetch_add(1, Ordering::Relaxed);
433 if hook.ingress.try_send(envelope.clone()).is_err() {
434 hook.meters.ingress_depth.fetch_sub(1, Ordering::Relaxed);
435 hook.meters.record_loss(Loss::Ingress);
436 }
437 }
438 }
439
440 /// Operational snapshot for each configured hook, in configuration order.
441 pub fn snapshots(&self) -> Vec<(Arc<str>, HookSnapshot)> {
442 self.shared
443 .iter()
444 .map(|hook| {
445 let mut snapshot = hook.meters.snapshot();
446 snapshot.ingress_capacity = hook.config.queue_capacity;
447 snapshot.queue_capacity = hook.config.queue_capacity;
448 (Arc::clone(&hook.config.name), snapshot)
449 })
450 .collect()
451 }
452}
453
454/// Everything needed to run the configured hooks.
455pub struct Dispatchers<E: Occurrence> {
456 hooks: Vec<Dispatcher<E>>,
457 client: HttpClient,
458 drain_timeout: Duration,
459 observer: Arc<dyn HookObserver<E::Kind>>,
460}
461
462/// Builds the enqueue side and the dispatchers that drain it.
463///
464/// Split so the caller owns where the dispatchers run: they belong in the
465/// application's task set, alongside the listeners they outlive.
466/// Every destination is validated before any channel is allocated.
467pub fn build<E: Occurrence>(
468 config: HooksConfig<E::Kind>,
469 client: HttpClient,
470 observer: Arc<dyn HookObserver<E::Kind>>,
471) -> Result<(Hooks<E>, Dispatchers<E>), ConfigError> {
472 let mut names = BTreeSet::new();
473 for hook in &config.hooks {
474 hook.validate()?;
475 if !names.insert(hook.name.clone()) {
476 return Err(ConfigError {
477 hook: hook.name.clone(),
478 reason: "duplicate destination name",
479 });
480 }
481 }
482 let dispatchers: Vec<Dispatcher<E>> = config
483 .hooks
484 .into_iter()
485 .map(|hook| {
486 let (ingress, receiver) = mpsc::channel(hook.queue_capacity);
487 let shared = Arc::new(Shared {
488 ingress,
489 config: hook,
490 meters: HookMeters::default(),
491 });
492 Dispatcher {
493 shared,
494 ingress: receiver,
495 }
496 })
497 .collect();
498 let shared: Vec<Arc<Shared<E>>> = dispatchers
499 .iter()
500 .map(|dispatcher| Arc::clone(&dispatcher.shared))
501 .collect();
502
503 Ok((
504 Hooks {
505 renderer: Renderer::new(config.source, config.schema_version),
506 shared: Arc::new(shared),
507 observer: Arc::clone(&observer),
508 },
509 Dispatchers {
510 hooks: dispatchers,
511 client,
512 drain_timeout: config.drain_timeout,
513 observer,
514 },
515 ))
516}