Skip to main content

smeltery_anvil/
lib.rs

1#![doc = include_str!("../README.md")]
2#![cfg_attr(docsrs, feature(doc_cfg))]
3
4mod auth_route;
5mod channels;
6mod endpoint;
7mod event;
8mod hub;
9mod listeners;
10mod live;
11mod origin;
12mod presence;
13mod process;
14mod protocol;
15mod revocation;
16mod session;
17mod settings;
18mod signature;
19pub mod testing;
20#[cfg(test)]
21mod transcripts;
22
23use std::future::{Future, IntoFuture};
24use std::pin::Pin;
25use std::sync::{Arc, Mutex, OnceLock, PoisonError};
26
27use serde::Serialize;
28use smeltery_core::pubsub::{PubSub, RecvError, Subscription};
29use smeltery_core::{App, AppBuilder, Error, Result};
30use tungstenite::Utf8Bytes;
31
32pub use channels::{ChannelCtx, Channels, PrivateChannel};
33pub use event::{BroadcastEvent, Channel, SocketId};
34pub use presence::{Member, migrations as presence_migrations};
35pub use protocol::{CloseCode, valid_channel, valid_socket_id};
36pub use settings::Settings;
37/// `#[derive(BroadcastEvent)]`: implements [`BroadcastEvent`] from `#[broadcast(...)]` (see the trait).
38pub use smeltery_macros::BroadcastEvent;
39
40/// The PubSub topic events travel on between the app's processes.
41const TOPIC: &str = "anvil";
42
43/// The signature purpose of the derived channel secret (`ANVIL_APP_SECRET` unset).
44const SECRET_PURPOSE: &str = "anvil.secret";
45
46/// The items a `routes/channels.rs` and event files use: `use smeltery::anvil::prelude::*;`.
47pub mod prelude {
48    pub use crate::{
49        Anvil, AnvilExt as _, BroadcastEvent, Channel, ChannelCtx, Channels, SocketId,
50    };
51}
52
53/// The shared state of an app's Anvil.
54pub(crate) struct Inner {
55    pub(crate) settings: Settings,
56    pub(crate) channels: Arc<Channels>,
57    pub(crate) hub: Arc<hub::Hub>,
58    /// Set at boot, once the secret is known.
59    pub(crate) session: OnceLock<Arc<session::Config>>,
60    pub(crate) policy: OnceLock<origin::Policy>,
61    pub(crate) handshakes: endpoint::Handshakes,
62    pubsub: OnceLock<PubSub>,
63    spy: Mutex<Option<Vec<testing::Sent>>>,
64    /// Recent revocations (core's auth events).
65    pub(crate) revocations: Arc<revocation::Revocations>,
66    /// This process's id on its presence rows (random).
67    pub(crate) process: String,
68    /// The presence store of a process whose PubSub driver is `local`.
69    pub(crate) memory: Arc<presence::memory::MemoryStore>,
70    /// The presence store, once the PubSub driver is known.
71    pub(crate) store: OnceLock<Arc<dyn presence::Store>>,
72    /// What a shared presence store is built from.
73    pub(crate) env: OnceLock<live::StoreEnv>,
74    /// The presence memberships of this process's sockets: (channel, socket) → member.
75    pub(crate) memberships: presence::Memberships,
76    /// Client events per channel this second (`live::CHANNEL_CLIENT_EVENTS_PER_SECOND`).
77    pub(crate) whisper_budget: Mutex<live::WhisperBudgets>,
78    /// The events this process delivers, for core's channel seam (Sparks listeners).
79    pub(crate) listeners: smeltery_core::channels::ChannelEventSender,
80    /// This process is `smeltery anvil`: it serves the socket endpoint whatever `ANVIL_IN_SERVE` says.
81    pub(crate) socket_process: std::sync::atomic::AtomicBool,
82}
83
84/// The app's real-time hub: sends events to the sockets subscribed to their channels, in this process and (through
85/// the app's [PubSub](smeltery_core::pubsub)) in its other processes. A cheap clone; a handler takes it as an
86/// argument (`anvil: Anvil`), jobs and agents get it with [`Anvil::of`].
87#[derive(Clone)]
88pub struct Anvil {
89    pub(crate) inner: Arc<Inner>,
90}
91
92impl std::fmt::Debug for Anvil {
93    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
94        f.debug_struct("Anvil")
95            .field("app_key", &self.inner.settings.app_key)
96            .field("connections", &self.connections())
97            .finish_non_exhaustive()
98    }
99}
100
101/// What a send reached.
102#[derive(Debug, Clone, Copy, PartialEq, Eq)]
103#[non_exhaustive]
104pub struct Delivered {
105    /// Sockets of this process the event was queued for (other processes are not counted).
106    pub local: usize,
107}
108
109/// An event ready to send: await it (`anvil.send(&event).await?`), optionally after
110/// [`except`](Self::except).
111#[must_use = "an event is sent only when awaited"]
112pub struct PendingEvent {
113    anvil: Anvil,
114    prepared: Result<Prepared>,
115    except: Option<SocketId>,
116}
117
118impl std::fmt::Debug for PendingEvent {
119    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
120        f.debug_struct("PendingEvent")
121            .field("except", &self.except)
122            .finish_non_exhaustive()
123    }
124}
125
126/// `channels` without repeats, in their first order (one event reaches a socket once per channel).
127fn distinct(channels: impl IntoIterator<Item = String>) -> Vec<String> {
128    let mut out: Vec<String> = Vec::new();
129    for channel in channels {
130        if !out.contains(&channel) {
131            out.push(channel);
132        }
133    }
134    out
135}
136
137/// The most sends [`testing::AnvilSpy`] keeps (the oldest are dropped).
138const MAX_RECORDED: usize = 10_000;
139
140/// An event's channels, name and data (its JSON as text).
141#[derive(Debug, Clone)]
142struct Prepared {
143    channels: Vec<String>,
144    name: String,
145    data: String,
146}
147
148impl PendingEvent {
149    /// Leave out the socket that made this request (its `X-Socket-ID`), so the client that caused the event does
150    /// not receive it again. `None` leaves nobody out.
151    pub fn except(mut self, socket: impl Into<Option<SocketId>>) -> Self {
152        self.except = socket.into();
153        self
154    }
155}
156
157impl IntoFuture for PendingEvent {
158    type Output = Result<Delivered>;
159    type IntoFuture = Pin<Box<dyn Future<Output = Result<Delivered>> + Send>>;
160
161    fn into_future(self) -> Self::IntoFuture {
162        Box::pin(async move {
163            let prepared = self.prepared?;
164            self.anvil.publish(prepared, self.except).await
165        })
166    }
167}
168
169/// Events to explicit channels without an event type ([`Anvil::to`]).
170#[derive(Debug)]
171#[must_use = "name the event with `.event(...)`"]
172pub struct To {
173    anvil: Anvil,
174    channels: Vec<Channel>,
175}
176
177impl To {
178    /// One more channel.
179    pub fn to(mut self, channel: Channel) -> Self {
180        self.channels.push(channel);
181        self
182    }
183
184    /// The event name (clients listen for exactly this name, with a leading dot in laravel-echo).
185    pub fn event(self, name: impl Into<String>) -> Named {
186        Named {
187            to: self,
188            name: name.into(),
189        }
190    }
191}
192
193/// A named event to explicit channels; [`with`](Self::with) gives its data.
194#[derive(Debug)]
195#[must_use = "give the data with `.with(...)` and await it"]
196pub struct Named {
197    to: To,
198    name: String,
199}
200
201impl Named {
202    /// The event's data (anything that serializes; an object is the usual shape).
203    pub fn with(self, data: &impl Serialize) -> PendingEvent {
204        let prepared = serde_json::to_string(data)
205            .map_err(Error::from)
206            .map(|data| Prepared {
207                channels: distinct(self.to.channels.iter().map(|c| c.name().to_owned())),
208                name: self.name,
209                data,
210            });
211        PendingEvent {
212            anvil: self.to.anvil,
213            prepared,
214            except: None,
215        }
216    }
217}
218
219/// The message between processes.
220#[derive(Debug, Serialize, serde::Deserialize)]
221pub(crate) struct Wire {
222    /// The event name.
223    pub(crate) e: String,
224    /// The channels.
225    pub(crate) c: Vec<String>,
226    /// The data (JSON text).
227    pub(crate) d: String,
228    /// The socket left out.
229    #[serde(default, skip_serializing_if = "Option::is_none")]
230    pub(crate) x: Option<String>,
231    /// What it is: absent for an app's event, `m` a presence member event, `w` a client event.
232    #[serde(default, skip_serializing_if = "Option::is_none")]
233    pub(crate) k: Option<String>,
234    /// A client event's sender on a presence channel.
235    #[serde(default, skip_serializing_if = "Option::is_none")]
236    pub(crate) u: Option<String>,
237}
238
239/// The longest event name.
240const MAX_EVENT_NAME: usize = 200;
241
242/// The most channels one event goes to.
243const MAX_CHANNELS: usize = 100;
244
245impl Anvil {
246    fn new(settings: Settings, channels: Channels) -> Self {
247        // A process that just started has not seen the auth events before it: grants made earlier are refused.
248        let revocations = Arc::new(revocation::Revocations::starting_at(unix_now()));
249        let hub = hub::Hub::new(
250            settings.max_connections,
251            settings.max_connections_per_ip,
252            settings.outbox,
253            Arc::clone(&revocations),
254        );
255        let handshakes = endpoint::Handshakes::new(settings.handshakes_per_minute);
256        Self {
257            inner: Arc::new(Inner {
258                settings,
259                channels: Arc::new(channels),
260                hub: Arc::new(hub),
261                session: OnceLock::new(),
262                policy: OnceLock::new(),
263                handshakes,
264                pubsub: OnceLock::new(),
265                spy: Mutex::new(None),
266                revocations,
267                process: protocol::new_socket_id().unwrap_or_else(|_| format!("{}", unix_now())),
268                memory: Arc::default(),
269                store: OnceLock::new(),
270                env: OnceLock::new(),
271                memberships: presence::Memberships::default(),
272                whisper_budget: Mutex::default(),
273                listeners: smeltery_core::channels::ChannelEventSender::new(),
274                socket_process: std::sync::atomic::AtomicBool::new(false),
275            }),
276        }
277    }
278
279    /// The app's Anvil, when `.anvil(...)` installed it.
280    pub fn of(app: &App) -> Option<Self> {
281        app.service::<Self>().map(|a| (*a).clone())
282    }
283
284    /// The settings.
285    pub fn settings(&self) -> &Settings {
286        &self.inner.settings
287    }
288
289    /// The public app key (the socket path is `/app/<key>`).
290    pub fn app_key(&self) -> &str {
291        &self.inner.settings.app_key
292    }
293
294    /// Whether this process serves the socket endpoint: `serve` unless `ANVIL_IN_SERVE=false`, and the `anvil`
295    /// process always.
296    pub fn serves_sockets(&self) -> bool {
297        self.inner.settings.in_serve
298            || self
299                .inner
300                .socket_process
301                .load(std::sync::atomic::Ordering::Relaxed)
302    }
303
304    /// Open sockets in this process.
305    pub fn connections(&self) -> usize {
306        self.inner.hub.connections()
307    }
308
309    /// Sockets of this process subscribed to `channel` (the full name, `private-orders.7`).
310    pub fn subscribers(&self, channel: &str) -> usize {
311        self.inner.hub.subscribers(channel)
312    }
313
314    /// Send `event` to its channels.
315    ///
316    /// ```no_run
317    /// # use serde::Serialize;
318    /// # use smeltery::anvil::{Anvil, BroadcastEvent, SocketId};
319    /// # #[derive(Serialize, BroadcastEvent)]
320    /// # #[broadcast(private = "orders.{order_id}")]
321    /// # struct OrderShipped { order_id: i64 }
322    /// async fn ship(anvil: Anvil, socket: Option<SocketId>) -> smeltery::Result<&'static str> {
323    ///     anvil.send(&OrderShipped { order_id: 7 }).except(socket).await?;
324    ///     Ok("shipped")
325    /// }
326    /// ```
327    pub fn send<E: BroadcastEvent + ?Sized>(&self, event: &E) -> PendingEvent {
328        let prepared = serde_json::to_string(event)
329            .map_err(Error::from)
330            .map(|data| Prepared {
331                channels: distinct(event.channels().iter().map(|c| c.name().to_owned())),
332                name: event.name().into_owned(),
333                data,
334            });
335        PendingEvent {
336            anvil: self.clone(),
337            prepared,
338            except: None,
339        }
340    }
341
342    /// Events to `channel` without an event type: `anvil.to(Channel::public("news")).event("posted").with(&data)`.
343    pub fn to(&self, channel: Channel) -> To {
344        To {
345            anvil: self.clone(),
346            channels: vec![channel],
347        }
348    }
349
350    /// Check an event before anything is delivered.
351    fn check(&self, prepared: &Prepared, except: Option<&SocketId>) -> Result<String> {
352        let name = &prepared.name;
353        if name.is_empty() || name.len() > MAX_EVENT_NAME || name.chars().any(char::is_control) {
354            return Err(Error::internal(format!(
355                "the event name `{}` must be 1 to {MAX_EVENT_NAME} bytes without control characters",
356                name.escape_debug()
357            )));
358        }
359        if name.starts_with("pusher:") || name.starts_with("pusher_internal:") {
360            return Err(Error::internal(format!(
361                "the event name `{}` uses a prefix the protocol reserves (`pusher:`, `pusher_internal:`)",
362                name.escape_debug()
363            )));
364        }
365        if prepared.channels.is_empty() || prepared.channels.len() > MAX_CHANNELS {
366            return Err(Error::internal(format!(
367                "the event `{name}` must go to 1 to {MAX_CHANNELS} channels"
368            )));
369        }
370        if let Some(bad) = prepared.channels.iter().find(|c| !valid_channel(c)) {
371            return Err(Error::internal(format!(
372                "the event `{name}` names an invalid channel `{}`",
373                bad.escape_debug()
374            )));
375        }
376        let max = self.inner.settings.max_event_size;
377        if prepared.data.len() > max {
378            return Err(Error::internal(format!(
379                "the event `{name}` is {} bytes, more than ANVIL_MAX_EVENT_SIZE ({max})",
380                prepared.data.len()
381            )));
382        }
383        let wire = serde_json::to_string(&Wire {
384            e: name.clone(),
385            c: prepared.channels.clone(),
386            d: prepared.data.clone(),
387            x: except.map(|s| s.as_str().to_owned()),
388            k: None,
389            u: None,
390        })?;
391        // The PubSub envelope adds the topic, the process id and a time stamp.
392        if wire.len() + 256 > smeltery_core::pubsub::MAX_MESSAGE_BYTES {
393            return Err(Error::internal(format!(
394                "the event `{name}` is too large to reach the app's other processes ({} bytes)",
395                wire.len()
396            )));
397        }
398        Ok(wire)
399    }
400
401    async fn publish(&self, prepared: Prepared, except: Option<SocketId>) -> Result<Delivered> {
402        let wire = self.check(&prepared, except.as_ref())?;
403        if let Some(sent) = self
404            .inner
405            .spy
406            .lock()
407            .unwrap_or_else(PoisonError::into_inner)
408            .as_mut()
409        {
410            if sent.len() >= MAX_RECORDED {
411                sent.remove(0);
412            }
413            sent.push(testing::Sent::new(
414                &prepared.name,
415                &prepared.channels,
416                &prepared.data,
417            ));
418        }
419        let local = self.deliver(&prepared, except.as_ref().map(SocketId::as_str));
420        if let Some(pubsub) = self.inner.pubsub.get() {
421            let value: serde_json::Value = serde_json::from_str(&wire)?;
422            pubsub.publish_reserved(TOPIC, &value).await?;
423        }
424        Ok(Delivered { local })
425    }
426
427    /// Queue the event for this process's subscribers; the frame is built once per channel.
428    fn deliver(&self, prepared: &Prepared, except: Option<&str>) -> usize {
429        prepared
430            .channels
431            .iter()
432            .map(|channel| {
433                let frame =
434                    Utf8Bytes::from(protocol::event(&prepared.name, channel, &prepared.data));
435                // Listeners inside the app (Sparks) get every delivered event; they have no socket to leave out.
436                self.inner
437                    .listeners
438                    .send(smeltery_core::channels::ChannelEvent::new(
439                        channel.as_str(),
440                        prepared.name.as_str(),
441                        prepared.data.as_str(),
442                    ));
443                self.inner.hub.deliver(channel, except, &frame)
444            })
445            .sum()
446    }
447
448    /// Subscribe to core's auth events and raise the watermark to `now` (Unix seconds): the events before this
449    /// moment were never seen here, so grants made before it are refused.
450    fn listen_for_revocations(&self, pubsub: &PubSub, now: u64) -> Subscription {
451        let events = pubsub.subscribe(smeltery_core::auth::EVENTS_TOPIC);
452        self.inner.revocations.listening_from(now);
453        events
454    }
455
456    /// Act on core's auth events (this process's and the others'), until shutdown: remember each revocation (a grant
457    /// made before it cannot subscribe) and close the sockets it concerns with 4200.
458    async fn revocations_from(
459        self,
460        events: Subscription,
461        token: tokio_util::sync::CancellationToken,
462    ) {
463        // Each event with the time it was sent (Unix seconds by the sender's clock): a revocation is dated by when it
464        // happened, not by when it arrived.
465        let events = futures_util::stream::unfold(events, |mut sub| async move {
466            let next = sub
467                .recv()
468                .await
469                .map(|m| (m.payload.clone(), m.sent_at / 1000));
470            Some((next, sub))
471        });
472        self.revocations_over(events, token).await;
473    }
474
475    /// The loop of [`revocations_from`](Self::revocations_from) over a stream of event payloads with their send
476    /// times (Unix seconds); ends with the stream, on `Closed` or on the token.
477    ///
478    /// An event this version cannot read that names a user ends every credential of that user; one that names no
479    /// user is handled like a lag (what this process cannot check, it refuses).
480    ///
481    /// After `Lagged` (auth events were lost), every grant made up to that second is refused at once, and every
482    /// socket holding a subscription such a grant authorized closes with 4200 (its client reconnects and authorizes
483    /// again, with a grant this process accepts): from the next second on, each at a random moment within 30 s, at
484    /// most one such mass close a minute (`revocation::LagCloser`).
485    async fn revocations_over(
486        self,
487        events: impl futures_util::Stream<
488            Item = std::result::Result<(serde_json::Value, u64), RecvError>,
489        >,
490        token: tokio_util::sync::CancellationToken,
491    ) {
492        use futures_util::StreamExt as _;
493        let mut events = std::pin::pin!(events);
494        let mut closer = revocation::LagCloser::default();
495        loop {
496            let message = tokio::select! {
497                biased;
498                () = token.cancelled() => return,
499                () = sleep_until_some(closer.next()) => {
500                    let hub = &self.inner.hub;
501                    let due = closer.due(
502                        tokio::time::Instant::now(),
503                        |limit| hub.granted_before(limit),
504                        random_spread,
505                    );
506                    let closed = due
507                        .into_iter()
508                        .filter(|(id, limit)| hub.close_if_granted_before(*id, *limit))
509                        .count();
510                    if closed > 0 {
511                        tracing::info!(closed, "{MASS_CLOSED}");
512                    }
513                    continue;
514                }
515                message = events.next() => message,
516            };
517            match message {
518                None | Some(Err(RecvError::Closed)) => return,
519                Some(Ok((payload, sent))) => {
520                    if self.revoke(&payload, sent).is_none() {
521                        let limit = self.inner.revocations.fell_behind(unix_now());
522                        tracing::error!("{UNREADABLE_EVENT}");
523                        closer.lagged(tokio::time::Instant::now(), until_next_second(), limit);
524                    }
525                }
526                Some(Err(RecvError::Lagged(missed))) => {
527                    let limit = self.inner.revocations.fell_behind(unix_now());
528                    tracing::warn!(missed, "{FELL_BEHIND}");
529                    closer.lagged(tokio::time::Instant::now(), until_next_second(), limit);
530                }
531                Some(Err(_)) => {}
532            }
533        }
534    }
535
536    /// Apply one auth event (its JSON) sent at `sent` (Unix seconds); how many sockets were closed, `None` when the
537    /// event names no user this version can read (the caller fails closed).
538    ///
539    /// An event of a type or credential kind this version does not know (a newer process in a rolling deploy) that
540    /// names a user ends every credential of that user (fail closed).
541    fn revoke(&self, payload: &serde_json::Value, sent: u64) -> Option<usize> {
542        let known = serde_json::from_value::<smeltery_core::auth::AuthEvent>(payload.clone())
543            .ok()
544            .as_ref()
545            .and_then(revocation::Revocation::from_event);
546        let revocation = match known {
547            Some(revocation) => revocation,
548            None => {
549                let user = payload.get("user_id").and_then(serde_json::Value::as_i64)?;
550                tracing::warn!(
551                    user,
552                    "anvil: an auth event this version does not know; every credential of its user is treated as ended"
553                );
554                revocation::Revocation::everything_of(user)
555            }
556        };
557        self.inner.revocations.record(revocation.clone(), sent);
558        let closed = self.inner.hub.revoke(&revocation);
559        if closed > 0 {
560            tracing::info!(closed, "{REVOKED_CLOSED}");
561        }
562        Some(closed)
563    }
564
565    /// Hand the events other processes publish to this process's sockets, until shutdown.
566    async fn relay(self, pubsub: PubSub, token: tokio_util::sync::CancellationToken) {
567        let messages =
568            futures_util::stream::unfold(pubsub.subscribe(TOPIC), |mut sub| async move {
569                let next = sub.recv().await.map(|m| (m.remote, m.payload.clone()));
570                Some((next, sub))
571            });
572        self.relay_from(messages, token).await;
573    }
574
575    /// The relay's loop over a stream of `(remote, payload)` messages; ends with the stream, on `Closed` or on the
576    /// token.
577    async fn relay_from(
578        self,
579        messages: impl futures_util::Stream<
580            Item = std::result::Result<(bool, serde_json::Value), RecvError>,
581        >,
582        token: tokio_util::sync::CancellationToken,
583    ) {
584        use futures_util::StreamExt as _;
585        let mut messages = std::pin::pin!(messages);
586        loop {
587            let message = tokio::select! {
588            () = token.cancelled() => return,
589            message = messages.next() => message,
590            };
591            match message {
592                None | Some(Err(RecvError::Closed)) => return,
593                // This process delivered its own events itself.
594                Some(Ok((false, _))) => {}
595                Some(Ok((true, payload))) => {
596                    self.deliver_remote(payload);
597                }
598                Some(Err(RecvError::Lagged(missed))) => {
599                    tracing::warn!(missed, "anvil: fell behind other processes' events");
600                }
601                Some(Err(_)) => {}
602            }
603        }
604    }
605
606    /// Deliver an event another process published, after the checks of a local send; how many sockets got it.
607    fn deliver_remote(&self, payload: serde_json::Value) -> usize {
608        let Ok(wire) = serde_json::from_value::<Wire>(payload) else {
609            tracing::warn!("anvil: an unreadable event from another process");
610            return 0;
611        };
612        if wire.k.is_some() {
613            return self.deliver_remote_live(wire);
614        }
615        let prepared = Prepared {
616            channels: distinct(wire.c),
617            name: wire.e,
618            data: wire.d,
619        };
620        let except = match wire.x.as_deref() {
621            None => None,
622            Some(id) => match SocketId::parse(id) {
623                Some(id) => Some(id),
624                None => {
625                    tracing::warn!("anvil: an event from another process was refused");
626                    return 0;
627                }
628            },
629        };
630        if self.check(&prepared, except.as_ref()).is_err() {
631            tracing::warn!("anvil: an event from another process was refused");
632            return 0;
633        }
634        self.deliver(&prepared, except.as_ref().map(SocketId::as_str))
635    }
636
637    /// Start recording sends (`testing::AnvilSpy`).
638    pub(crate) fn record(&self) {
639        let mut spy = self
640            .inner
641            .spy
642            .lock()
643            .unwrap_or_else(PoisonError::into_inner);
644        if spy.is_none() {
645            *spy = Some(Vec::new());
646        }
647    }
648
649    pub(crate) fn recorded(&self) -> Vec<testing::Sent> {
650        self.inner
651            .spy
652            .lock()
653            .unwrap_or_else(PoisonError::into_inner)
654            .clone()
655            .unwrap_or_default()
656    }
657
658    /// Prepare what needs the app: the channel secret, the Origin policy and the PubSub.
659    fn boot(&self, app: &App) -> Result<()> {
660        let errors = self.inner.channels.errors();
661        if !errors.is_empty() {
662            return Err(Error::internal(errors.join("; ")));
663        }
664        let core = app.settings();
665        self.inner.settings.check(core)?;
666        let policy = origin::Policy::new(
667            &core.url,
668            &self.inner.settings.allowed_origins,
669            &core.cors_allowed_origins,
670            core.is_local_development(),
671        )?;
672        if policy.allows_any() && core.env != "local" {
673            tracing::warn!(
674                "ANVIL_ALLOWED_ORIGINS contains `*`: pages of any site may open sockets to this app"
675            );
676        }
677        let _ = self.inner.policy.set(policy);
678        let secret = match &self.inner.settings.app_secret {
679            Some(secret) => Some(secret.clone()),
680            // Without a usable APP_KEY the app cannot serve (its web routes need it); console commands still build.
681            None => app
682                .derive_key(SECRET_PURPOSE)
683                .ok()
684                .map(|key| key.iter().map(|b| format!("{b:02x}")).collect::<String>()),
685        };
686        if self.inner.settings.app_key == settings::derived_key(&core.key) && core.is_production() {
687            tracing::info!(
688                "ANVIL_APP_KEY is derived from APP_KEY and changes when APP_KEY changes; set it in .env when clients \
689                 are built with it"
690            );
691        }
692        if let Some(secret) = secret {
693            let s = &self.inner.settings;
694            let _ = self.inner.session.set(Arc::new(session::Config {
695                app_key: s.app_key.clone(),
696                secret,
697                channels: Arc::clone(&self.inner.channels),
698                activity_timeout: s.activity_timeout,
699                ping_interval: s.ping_interval,
700                pong_timeout: s.pong_timeout,
701                max_age: s.max_connection_age,
702                max_subscriptions: s.max_subscriptions,
703                max_presence_channels: s.max_presence_channels,
704                frames_per_second: s.frames_per_second,
705                frame_burst: s.frame_burst,
706                revocations: Arc::clone(&self.inner.revocations),
707            }));
708        }
709        if let Some(pubsub) = PubSub::of(app) {
710            let _ = self.inner.pubsub.set(pubsub);
711        }
712        self.remember_app(app);
713        Ok(())
714    }
715}
716
717/// The log message when the auth subscription lagged.
718const FELL_BEHIND: &str = "anvil: fell behind the auth events; grants made until now are refused, and the sockets \
719                           they authorized reconnect within 30 s";
720
721/// The log message when an auth event names no user this version can read.
722const UNREADABLE_EVENT: &str = "anvil: an auth event names no user this version can read; grants made until now are \
723                                refused, and the sockets they authorized reconnect within 30 s";
724
725/// The log message when sockets closed after a lag.
726const MASS_CLOSED: &str = "anvil: closed sockets authorized before missed auth events";
727
728/// The log message when an auth event closed sockets.
729const REVOKED_CLOSED: &str = "anvil: closed the sockets of ended credentials";
730
731/// A random moment within `revocation::MASS_CLOSE_SPREAD`.
732fn random_spread() -> std::time::Duration {
733    let mut bytes = [0_u8; 4];
734    // Without a random source every socket closes at once (it still closes).
735    let _ = getrandom::fill(&mut bytes);
736    let spread = u32::try_from(revocation::MASS_CLOSE_SPREAD.as_millis()).unwrap_or(30_000);
737    std::time::Duration::from_millis(u64::from(u32::from_le_bytes(bytes) % spread.max(1)))
738}
739
740/// Unix seconds.
741fn unix_now() -> u64 {
742    std::time::SystemTime::now()
743        .duration_since(std::time::UNIX_EPOCH)
744        .map_or(0, |d| d.as_secs())
745}
746
747/// The time until the next second of the clock starts (and a little more).
748fn until_next_second() -> std::time::Duration {
749    let into = std::time::SystemTime::now()
750        .duration_since(std::time::UNIX_EPOCH)
751        .map_or(0, |d| d.subsec_millis());
752    std::time::Duration::from_millis(u64::from(1_000 - into.min(999)) + 50)
753}
754
755/// Sleep until `deadline`; never ends without one.
756async fn sleep_until_some(deadline: Option<tokio::time::Instant>) {
757    match deadline {
758        Some(at) => tokio::time::sleep_until(at).await,
759        None => std::future::pending().await,
760    }
761}
762
763impl axum::extract::FromRequestParts<App> for Anvil {
764    type Rejection = Error;
765
766    async fn from_request_parts(
767        _parts: &mut http::request::Parts,
768        app: &App,
769    ) -> std::result::Result<Self, Self::Rejection> {
770        Self::of(app).ok_or_else(|| {
771            Error::internal("Anvil is not installed: call `.anvil(...)` in bootstrap/app.rs")
772        })
773    }
774}
775
776/// Installs Anvil on an [`AppBuilder`]: `.anvil(routes::channels::register)` in `bootstrap/app.rs`.
777pub trait AnvilExt: Sized {
778    /// Install Anvil with the settings from `.env` and the channels `register` declares: the socket endpoint
779    /// `GET /app/<ANVIL_APP_KEY>` (outside the web stack: no session, no cookies read), the web route
780    /// `POST /broadcasting/auth` (session and CSRF, `throttle:600,1`), `POST /api/broadcasting/auth` (an API route for
781    /// the bearer credentials of the app's stateless guards, `throttle:600,1`; it answers 404 when the app has none),
782    /// and the [`Anvil`] service. An invalid or
783    /// repeated channel pattern, an invalid setting or `ANVIL_MAX_CONNECTIONS` not below
784    /// `SERVER_MAX_CONNECTIONS` fails the build.
785    fn anvil(self, register: impl FnOnce(&mut Channels)) -> Self;
786
787    /// [`anvil`](Self::anvil) with explicit settings.
788    fn anvil_with(self, settings: Settings, register: impl FnOnce(&mut Channels)) -> Self;
789}
790
791impl AnvilExt for AppBuilder {
792    fn anvil(self, register: impl FnOnce(&mut Channels)) -> Self {
793        let settings = Settings::from_env(self.settings());
794        self.anvil_with(settings, register)
795    }
796
797    fn anvil_with(self, settings: Settings, register: impl FnOnce(&mut Channels)) -> Self {
798        let mut channels = Channels::new();
799        register(&mut channels);
800        let key = settings.app_key.clone();
801        let anvil = Anvil::new(settings, channels);
802        let booted = anvil.clone();
803        let relayed = anvil.clone();
804        let builder = self
805            .channel_authorizer(listeners::Listeners(anvil.clone()))
806            .service(anvil)
807            .on_boot(move |app| async move { booted.boot(&app) })
808            .on_serve(move |app| async move {
809                if relayed.serves_sockets() {
810                    tracing::info!(path = %format!("/app/{}", relayed.app_key()), "anvil: socket endpoint");
811                } else {
812                    // The sockets are in the `anvil` process: events this process sends must reach it (under
813                    // `PUBSUB_DRIVER=auto` the shared driver; A3).
814                    app.set_web_only();
815                    tracing::info!("anvil: ANVIL_IN_SERVE=false; the `anvil` process serves the sockets");
816                }
817                if let Some(pubsub) = PubSub::of(&app) {
818                    let token = app.shutdown_token().clone();
819                    // Subscribed here, before the server accepts: from now on no auth event is missed.
820                    let events = relayed.listen_for_revocations(&pubsub, unix_now());
821                    app.spawn_owned(relayed.clone().revocations_from(events, token.clone()));
822                    // Presence: this process's rows stay alive, gone processes are swept, and at shutdown this
823                    // process's members leave.
824                    app.spawn_owned(relayed.clone().presence_heartbeat(token.clone()));
825                    app.spawn_owned(relayed.relay(pubsub, token));
826                }
827                Ok(())
828            })
829            .routes(|r| {
830                r.post("/broadcasting/auth", auth_route::authorize)
831                    .name("anvil.auth")
832                    .middleware("throttle:600,1");
833            })
834            .serve_command(process::NAME, process::ABOUT, process::run)
835            .api_routes(|r| {
836                r.post("/broadcasting/auth", auth_route::authorize_token)
837                    .name("anvil.token_auth")
838                    .middleware("throttle:600,1");
839            });
840        if settings::valid_app_key(&key) {
841            let path = format!("/{key}");
842            builder.api_routes_at("/app", move |r| {
843                r.get(&path, endpoint::socket).name("anvil.socket");
844            })
845        } else {
846            // `boot` reports the invalid key; no route is built from it.
847            builder
848        }
849    }
850}
851
852#[cfg(test)]
853mod tests {
854    #![allow(clippy::unwrap_used, clippy::expect_used)]
855
856    use super::*;
857    use serde_json::json;
858
859    fn anvil() -> Anvil {
860        let core = smeltery_core::config::Settings::from_env();
861        let mut channels = Channels::new();
862        channels.public("news");
863        Anvil::new(Settings::from_env(&core), channels)
864    }
865
866    #[test]
867    fn a_channel_carries_a_bounded_number_of_client_events_a_second() {
868        let anvil = anvil();
869        for n in 0..live::CHANNEL_CLIENT_EVENTS_PER_SECOND {
870            // Distinct addresses: only the channel's budget is in play.
871            let client = Some(std::net::IpAddr::from([198, 51, 100, n as u8]));
872            assert!(
873                anvil
874                    .whisper("1.1", client, "private-chat.1", "client-x", &json!(1), None)
875                    .is_ok()
876            );
877        }
878        assert_eq!(
879            anvil.whisper("1.2", None, "private-chat.1", "client-x", &json!(1), None),
880            Err(live::CHANNEL_BUDGET_SPENT),
881            "the channel's budget, whoever sends"
882        );
883        assert!(
884            anvil
885                .whisper("1.1", None, "private-chat.2", "client-x", &json!(1), None)
886                .is_ok()
887        );
888    }
889
890    /// Sweep W5-01: one client address cannot send more than `ANVIL_CLIENT_EVENTS_PER_CLIENT` client events a second
891    /// over all its sockets and channels, nor all clients together more than `ANVIL_CLIENT_EVENTS_PER_SECOND`: each
892    /// one reaches the other processes (a database row with the `database` driver).
893    #[test]
894    fn client_events_are_budgeted_per_address_and_per_process() {
895        let core = smeltery_core::config::Settings::from_env();
896        let mut settings = Settings::from_env(&core);
897        settings.client_events_per_client = 5;
898        settings.client_events_per_second = 12;
899        let anvil = Anvil::new(settings, Channels::new());
900        let one = Some(std::net::IpAddr::from([203, 0, 113, 7]));
901        // One address, a new socket and channel each time (the per-channel budget never binds).
902        for n in 0..5 {
903            let channel = format!("private-chat.{n}");
904            assert!(
905                anvil
906                    .whisper(
907                        &format!("1.{n}"),
908                        one,
909                        &channel,
910                        "client-x",
911                        &json!(1),
912                        None
913                    )
914                    .is_ok()
915            );
916        }
917        assert_eq!(
918            anvil.whisper("1.9", one, "private-chat.9", "client-x", &json!(1), None),
919            Err(live::CLIENT_BUDGET_SPENT)
920        );
921        // Other addresses go on until the process's budget is spent.
922        for n in 0..7u8 {
923            let other = Some(std::net::IpAddr::from([198, 51, 100, n]));
924            assert!(
925                anvil
926                    .whisper("2.1", other, "private-chat.x", "client-x", &json!(1), None)
927                    .is_ok(),
928                "{n}"
929            );
930        }
931        let late = Some(std::net::IpAddr::from([192, 0, 2, 1]));
932        assert_eq!(
933            anvil.whisper("3.1", late, "private-chat.y", "client-x", &json!(1), None),
934            Err(live::PROCESS_BUDGET_SPENT)
935        );
936    }
937
938    #[test]
939    fn remote_member_events_need_a_well_formed_user_id() {
940        let mut channels = Channels::new();
941        channels.presence("room.{r}", |_| async { Ok(None) });
942        let core = smeltery_core::config::Settings::from_env();
943        let anvil = Anvil::new(Settings::from_env(&core), channels);
944        let mut socket = anvil.inner.hub.register("1.1", None).ok().unwrap();
945        assert!(socket.registration.join("presence-room.1", None));
946        let member = |data: &str| json!({ "e": "pusher_internal:member_removed", "c": ["presence-room.1"], "d": data, "k": "m" });
947        for bad in [
948            r#"{"user_id":""}"#,
949            r#"{"user_id":7}"#,
950            r#"{"name":"x"}"#,
951            "nope",
952        ] {
953            assert_eq!(anvil.deliver_remote(member(bad)), 0, "{bad}");
954        }
955        assert_eq!(anvil.deliver_remote(member(r#"{"user_id":"7"}"#)), 1);
956        assert!(socket.outbox.try_recv().is_ok());
957    }
958
959    #[tokio::test]
960    async fn a_join_that_fails_is_undone_and_forgotten() {
961        use crate::presence::{Joined, Member, Store};
962        /// A store whose joins write the socket and then fail (a timeout after the first statement).
963        #[derive(Default)]
964        struct Failing(presence::memory::MemoryStore);
965        impl Store for Failing {
966            fn join<'a>(
967                &'a self,
968                channel: &'a str,
969                socket: &'a str,
970                member: &'a Member,
971                max: usize,
972            ) -> smeltery_core::BoxFuture<'a, Result<Joined>> {
973                Box::pin(async move {
974                    self.0.join(channel, socket, member, max).await?;
975                    Err(Error::internal("cut off"))
976                })
977            }
978            fn leave<'a>(
979                &'a self,
980                channel: &'a str,
981                socket: &'a str,
982                user_id: &'a str,
983            ) -> smeltery_core::BoxFuture<'a, Result<bool>> {
984                self.0.leave(channel, socket, user_id)
985            }
986            fn members<'a>(
987                &'a self,
988                channel: &'a str,
989                max: usize,
990            ) -> smeltery_core::BoxFuture<'a, Result<Vec<Member>>> {
991                self.0.members(channel, max)
992            }
993        }
994        let anvil = anvil();
995        let failing: Arc<dyn Store> = Arc::new(Failing::default());
996        let _ = anvil.inner.store.set(Arc::clone(&failing));
997        let result = anvil
998            .presence_join("1.1", "presence-room.1", &Member::new(7))
999            .await;
1000        assert!(result.is_err());
1001        assert!(anvil.memberships_of("1.1").is_empty(), "forgotten");
1002        assert!(
1003            failing
1004                .members("presence-room.1", 10)
1005                .await
1006                .unwrap()
1007                .is_empty(),
1008            "the half-made join is undone"
1009        );
1010    }
1011
1012    fn wire(name: &str, channels: &[&str], except: Option<&str>) -> serde_json::Value {
1013        let mut value = json!({ "e": name, "c": channels, "d": "{\"id\":1}" });
1014        if let Some(id) = except {
1015            value["x"] = json!(id);
1016        }
1017        value
1018    }
1019
1020    #[tokio::test]
1021    async fn the_relay_survives_lag_and_refuses_bad_remote_events() {
1022        let anvil = anvil();
1023        let mut socket = anvil.inner.hub.register("1.1", None).ok().unwrap();
1024        assert!(socket.registration.join("news", None));
1025        let messages = vec![
1026            Err(RecvError::Lagged(7)),
1027            // Unreadable, an invalid channel, a reserved name, a bad socket id, too large.
1028            Ok((true, json!({ "nonsense": true }))),
1029            Ok((true, wire("posted", &["bad channel"], None))),
1030            Ok((
1031                true,
1032                wire("pusher_internal:subscription_succeeded", &["news"], None),
1033            )),
1034            Ok((true, wire("posted", &["news"], Some("1.1; x")))),
1035            Ok((
1036                true,
1037                json!({ "e": "big", "c": ["news"], "d": "x".repeat(anvil.settings().max_event_size + 1) }),
1038            )),
1039            // This process's own event: delivered when it was sent, not again.
1040            Ok((false, wire("own", &["news"], None))),
1041            // A good one, with a repeated channel: delivered once.
1042            Ok((true, wire("posted", &["news", "news"], None))),
1043            Err(RecvError::Closed),
1044            Ok((true, wire("after-close", &["news"], None))),
1045        ];
1046        let token = tokio_util::sync::CancellationToken::new();
1047        anvil
1048            .clone()
1049            .relay_from(futures_util::stream::iter(messages), token)
1050            .await;
1051        let frame = socket.outbox.try_recv().unwrap();
1052        assert!(frame.as_str().contains("\"posted\""), "{}", frame.as_str());
1053        assert!(
1054            socket.outbox.try_recv().is_err(),
1055            "exactly one event got through"
1056        );
1057    }
1058
1059    #[test]
1060    fn a_new_process_refuses_grants_made_before_it_started() {
1061        let anvil = anvil();
1062        let holder = |issued: u64| crate::revocation::Holder {
1063            user: 7,
1064            key: "fake:token:1".into(),
1065            issued,
1066        };
1067        let now = unix_now();
1068        assert!(anvil.inner.revocations.refuses_holder(&holder(now - 10)));
1069        assert!(!anvil.inner.revocations.refuses_holder(&holder(now + 1)));
1070    }
1071
1072    #[tokio::test]
1073    async fn listening_to_the_auth_events_refuses_grants_made_before() {
1074        let app = smeltery_core::AppBuilder::new(smeltery_core::config::Settings::from_env())
1075            .build()
1076            .await
1077            .unwrap()
1078            .app;
1079        let pubsub = PubSub::of(&app).unwrap();
1080        let anvil = anvil();
1081        let later = unix_now() + 100;
1082        let holder = |issued: u64| crate::revocation::Holder {
1083            user: 7,
1084            key: "fake:token:1".into(),
1085            issued,
1086        };
1087        assert!(!anvil.inner.revocations.refuses_holder(&holder(later - 1)));
1088        let _events = anvil.listen_for_revocations(&pubsub, later);
1089        assert!(
1090            anvil.inner.revocations.refuses_holder(&holder(later - 1)),
1091            "made before the subscription existed"
1092        );
1093        assert!(!anvil.inner.revocations.refuses_holder(&holder(later)));
1094    }
1095
1096    #[test]
1097    fn log_messages_have_no_runs_of_spaces() {
1098        for message in [FELL_BEHIND, MASS_CLOSED, REVOKED_CLOSED, UNREADABLE_EVENT] {
1099            assert!(!message.contains("  "), "{message}");
1100        }
1101    }
1102
1103    /// Sweep W5-05: an auth event of a type or credential kind this version does not know (a newer process in a
1104    /// rolling deploy) ends every credential of the user it names; one that names no user is answered as `None` (the
1105    /// loop fails closed, see `an_auth_event_naming_no_user_fails_closed`).
1106    #[test]
1107    fn auth_events_this_version_cannot_read_fail_closed() {
1108        use crate::revocation::Holder;
1109        let anvil = anvil();
1110        let now = unix_now();
1111        let holder = |user: i64, key: &str| Holder {
1112            user,
1113            key: key.into(),
1114            issued: now,
1115        };
1116        let mut seven = anvil.inner.hub.register("1.1", None).ok().unwrap();
1117        assert!(
1118            seven
1119                .registration
1120                .join("private-a", Some(holder(7, "web:session:aa")))
1121        );
1122        let mut nine = anvil.inner.hub.register("1.2", None).ok().unwrap();
1123        assert!(
1124            nine.registration
1125                .join("private-a", Some(holder(9, "fake:token:3")))
1126        );
1127        let mut other = anvil.inner.hub.register("1.3", None).ok().unwrap();
1128        assert!(
1129            other
1130                .registration
1131                .join("private-a", Some(holder(8, "web:session:bb")))
1132        );
1133        let unknown_kind = json!({ "type": "revoked_all", "user_id": 7, "kind": "passkeys" });
1134        assert_eq!(anvil.revoke(&unknown_kind, now), Some(1));
1135        assert_eq!(*seven.close.borrow_and_update(), Some(CloseCode::Reconnect));
1136        assert!(
1137            anvil
1138                .inner
1139                .revocations
1140                .refuses_holder(&holder(7, "fake:token:4")),
1141            "every credential of user 7"
1142        );
1143        let unknown_type = json!({ "type": "revoked_device", "user_id": 9, "device": "x" });
1144        assert_eq!(anvil.revoke(&unknown_type, now), Some(1));
1145        assert_eq!(*nine.close.borrow_and_update(), Some(CloseCode::Reconnect));
1146        assert_eq!(*other.close.borrow_and_update(), None, "another user");
1147        assert_eq!(anvil.revoke(&json!({ "type": "rotated" }), now), None);
1148    }
1149
1150    /// Sweep W5-06: a revocation is dated by when it was sent, not when it arrived: a grant made well after the
1151    /// event (beyond the clock margin) is not refused because the event arrived late or again.
1152    #[test]
1153    fn revocations_are_dated_by_when_they_were_sent() {
1154        use crate::revocation::Holder;
1155        let anvil = anvil();
1156        let now = unix_now();
1157        let fresh = Holder {
1158            user: 7,
1159            key: "web:session:aa".into(),
1160            issued: now,
1161        };
1162        let logout = json!({ "type": "revoked", "user_id": 7, "key": "web:session:aa" });
1163        // Sent two minutes ago (before the grant, beyond the 30 s margin).
1164        assert_eq!(anvil.revoke(&logout, now - 120), Some(0));
1165        assert!(!anvil.inner.revocations.refuses_holder(&fresh));
1166        // Sent now: the grant is refused.
1167        assert_eq!(anvil.revoke(&logout, now), Some(0));
1168        assert!(anvil.inner.revocations.refuses_holder(&fresh));
1169    }
1170
1171    /// Sweep W5-05: an auth event naming no user this version can read is handled like a lag: grants made until
1172    /// now are refused and the sockets holding them close (spread over 30 s).
1173    #[tokio::test(start_paused = true)]
1174    async fn an_auth_event_naming_no_user_fails_closed() {
1175        use crate::revocation::Holder;
1176        use futures_util::StreamExt as _;
1177        let anvil = anvil();
1178        let now = unix_now();
1179        let mut old = anvil.inner.hub.register("1.1", None).ok().unwrap();
1180        assert!(old.registration.join(
1181            "private-a",
1182            Some(Holder {
1183                user: 7,
1184                key: "fake:token:1".into(),
1185                issued: now,
1186            })
1187        ));
1188        let token = tokio_util::sync::CancellationToken::new();
1189        let events = futures_util::stream::iter(vec![Ok((json!({ "type": "rotated" }), now))])
1190            .chain(futures_util::stream::pending());
1191        let task = tokio::spawn(anvil.clone().revocations_over(events, token.clone()));
1192        tokio::time::timeout(std::time::Duration::from_secs(32), old.close.changed())
1193            .await
1194            .expect("closed after the unreadable event")
1195            .unwrap();
1196        assert_eq!(*old.close.borrow_and_update(), Some(CloseCode::Reconnect));
1197        token.cancel();
1198        task.await.unwrap();
1199    }
1200
1201    #[tokio::test(start_paused = true)]
1202    async fn falling_behind_the_auth_events_fails_closed() {
1203        use crate::revocation::Holder;
1204        use futures_util::StreamExt as _;
1205        let anvil = anvil();
1206        let now = unix_now();
1207        let holder = |key: &str, issued: u64| Holder {
1208            user: 7,
1209            key: key.into(),
1210            issued,
1211        };
1212        let mut old = anvil.inner.hub.register("1.1", None).ok().unwrap();
1213        assert!(
1214            old.registration
1215                .join("private-a", Some(holder("fake:token:1", now)))
1216        );
1217        let mut fresh = anvil.inner.hub.register("1.2", None).ok().unwrap();
1218        assert!(
1219            fresh
1220                .registration
1221                .join("private-a", Some(holder("fake:token:2", now + 5)))
1222        );
1223        let mut public = anvil.inner.hub.register("1.3", None).ok().unwrap();
1224        assert!(public.registration.join("news", None));
1225
1226        let token = tokio_util::sync::CancellationToken::new();
1227        let events = futures_util::stream::iter(vec![Err(RecvError::Lagged(3))])
1228            .chain(futures_util::stream::pending());
1229        let task = tokio::spawn(anvil.clone().revocations_over(events, token.clone()));
1230        // Within the next second plus the 30 s spread (paused time).
1231        tokio::time::timeout(std::time::Duration::from_secs(32), old.close.changed())
1232            .await
1233            .expect("closed after the lag")
1234            .unwrap();
1235        assert_eq!(*old.close.borrow_and_update(), Some(CloseCode::Reconnect));
1236        assert_eq!(*fresh.close.borrow_and_update(), None, "a later grant");
1237        assert_eq!(*public.close.borrow_and_update(), None, "no grant");
1238        assert!(
1239            anvil
1240                .inner
1241                .revocations
1242                .refuses_holder(&holder("fake:token:3", now)),
1243            "a grant made before the lag cannot subscribe"
1244        );
1245        token.cancel();
1246        task.await.unwrap();
1247    }
1248
1249    #[tokio::test]
1250    async fn reserved_names_are_refused_and_channels_sent_once() {
1251        let anvil = anvil();
1252        let mut socket = anvil.inner.hub.register("1.1", None).ok().unwrap();
1253        assert!(socket.registration.join("news", None));
1254        let err = anvil
1255            .to(Channel::public("news"))
1256            .event("pusher:error")
1257            .with(&json!({}))
1258            .await
1259            .unwrap_err();
1260        assert!(err.to_string().contains("reserves"), "{err}");
1261        assert!(socket.outbox.try_recv().is_err());
1262        let delivered = anvil
1263            .to(Channel::public("news"))
1264            .to(Channel::public("news"))
1265            .event("posted")
1266            .with(&json!({}))
1267            .await
1268            .unwrap();
1269        assert_eq!(delivered.local, 1);
1270    }
1271
1272    #[tokio::test]
1273    async fn the_spy_keeps_a_bounded_record() {
1274        let anvil = anvil();
1275        anvil.record();
1276        for _ in 0..(MAX_RECORDED + 5) {
1277            anvil
1278                .to(Channel::public("news"))
1279                .event("posted")
1280                .with(&json!({}))
1281                .await
1282                .unwrap();
1283        }
1284        assert_eq!(anvil.recorded().len(), MAX_RECORDED);
1285    }
1286}