Skip to main content

evento_remote/
client.rs

1//! Client side: an [`Executor`] that forwards every call to a remote server.
2//!
3//! Each pooled connection ([`ClientBuilder::connections`], default 1) is owned
4//! by one actor: it multiplexes requests dispatched to it over its TCP stream,
5//! demultiplexes replies by correlation id, and keeps the connection alive
6//! with capped exponential backoff (eager reconnect — the push channel must
7//! stay alive even while the client is idle-subscribed). The delay before
8//! each attempt is drawn at random below the backoff, so the clients a server
9//! restart dropped together do not come back together
10//! ([`ClientBuilder::reconnect_backoff`]). Requests are
11//! round-robined across connections.
12//!
13//! **Write wakes.** A watcher ([`Executor::write_watch_for`], what a
14//! subscription waits on) registers its filter with the server, on the first
15//! connection, and is woken by the pushed [`ServerFrame::Notify`] frames that
16//! name it: the writes that hold an event it would read, and no other. The
17//! server also names a watch when it registers it, so a watcher is woken once
18//! at the start and once after every reconnect, and reads what it could have
19//! missed.
20//!
21//! **At-most-once semantics.** A request in flight when the connection drops
22//! fails at the caller, but may still have executed on the server — a retried
23//! `write` can then surface `InvalidOriginalVersion`. This is the same
24//! contract as any RPC store. Requests on different pooled connections have no
25//! ordering relative to each other; the executor contract is request/response,
26//! and callers await a reply before issuing a dependent call.
27//!
28//! **Addresses.** The server is named by an [`Address`]: an IP and port, or a
29//! `host:port` looked up again on every connection attempt, each address it
30//! resolves to tried in turn. A reconnect therefore follows the name — a load
31//! balancer's current address, a DNS failover, a geo-routed record — rather
32//! than redialling the node it first reached.
33
34use std::collections::HashMap;
35use std::fmt;
36use std::net::SocketAddr;
37use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
38use std::sync::Arc;
39use std::time::Duration;
40
41use anyhow::Context as _;
42use bytes::Bytes;
43use evento_core::cursor::{Args, ReadResult, Value};
44use evento_core::{
45    Event, EventFilter, Executor, RoutingKey, TenantFilter, WriteError, WriteInterest,
46};
47use futures_util::{SinkExt, StreamExt};
48use tokio::net::TcpStream;
49use tokio::sync::{mpsc, oneshot, watch};
50use tokio_rustls::rustls::pki_types::ServerName;
51use tokio_rustls::TlsConnector;
52use tokio_util::codec::{Framed, LengthDelimitedCodec};
53use tokio_util::either::Either;
54use ulid::Ulid;
55
56use crate::wire::{
57    decode_tagged, encode_tagged, ClientFrame, HelloOk, RecordKind, Rejection, Request, Response,
58    ServerFrame, Token, MAX_FRAME_LENGTH,
59};
60
61/// Outbound connection: plain TCP, or a client-side TLS session over it.
62type ClientStream = Either<TcpStream, tokio_rustls::client::TlsStream<TcpStream>>;
63
64/// Client-side TLS settings: the rustls connector and the server name
65/// presented for certificate verification.
66#[derive(Clone)]
67struct Tls {
68    connector: TlsConnector,
69    server_name: ServerName<'static>,
70}
71
72/// Where a [`Client`] connects: an IP and port, or a `host:port` looked up
73/// again on every connection attempt (the first connect and each reconnect).
74///
75/// ```
76/// # use evento_remote::Address;
77/// let ip: Address = "127.0.0.1:4321".into();
78/// let name: Address = "store.example.com:4321".into();
79/// let socket: Address = std::net::SocketAddr::from(([127, 0, 0, 1], 4321)).into();
80/// # let _ = (ip, name, socket);
81/// ```
82#[derive(Clone, Debug, PartialEq, Eq)]
83pub struct Address(String);
84
85impl From<SocketAddr> for Address {
86    fn from(addr: SocketAddr) -> Self {
87        Address(addr.to_string())
88    }
89}
90
91impl From<&str> for Address {
92    fn from(addr: &str) -> Self {
93        Address(addr.to_owned())
94    }
95}
96
97impl From<String> for Address {
98    fn from(addr: String) -> Self {
99        Address(addr)
100    }
101}
102
103impl fmt::Display for Address {
104    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
105        f.write_str(&self.0)
106    }
107}
108
109/// Everything a connection attempt needs; shared by the first connect and
110/// every reconnect.
111#[derive(Clone)]
112struct Dial {
113    addr: Address,
114    token: Option<Token>,
115    tls: Option<Tls>,
116}
117
118const DEFAULT_REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
119const RECONNECT_MIN: Duration = Duration::from_millis(100);
120const RECONNECT_MAX: Duration = Duration::from_secs(5);
121
122/// The reconnect schedule: the bound on the delay starts at `min` and doubles
123/// up to `max` with every failed attempt. The delay itself is drawn uniformly
124/// in `[0, bound]` (full jitter): a fixed schedule would bring every client
125/// that lost its connection at the same instant back at the same instants.
126#[derive(Clone, Copy)]
127struct Backoff {
128    min: Duration,
129    max: Duration,
130}
131
132impl Backoff {
133    /// The delay before the next attempt, drawn below `bound`, which then
134    /// doubles. The first attempt is delayed like the others.
135    fn next_delay(&self, bound: &mut Duration) -> Duration {
136        let delay = bound.mul_f64(rand::random_range(0.0..=1.0));
137        *bound = bound.saturating_mul(2).min(self.max);
138        delay
139    }
140}
141/// Correlation id of the per-connection `Hello` exchange; regular ids start at 1.
142const HELLO_ID: u64 = 0;
143/// Capacity of each connection's outbound request queue.
144const REQ_CAPACITY: usize = 1024;
145/// Capacity of the per-connection encoded-frame queue feeding the writer task.
146const OUT_CAPACITY: usize = 1024;
147
148/// The cached stability watermark pushed by the server, max-merged across
149/// every connection's responses and notifies (so cross-connection reordering
150/// can never move it backwards). `0` = not reported yet — the watermark is
151/// microseconds since the Unix epoch, so a real report is never 0.
152type Stable = Arc<AtomicU64>;
153
154fn codec() -> LengthDelimitedCodec {
155    LengthDelimitedCodec::builder()
156        .max_frame_length(MAX_FRAME_LENGTH)
157        .new_codec()
158}
159
160/// A request dispatched to a connection actor: the reply channel travels with
161/// it, and the actor owns its own pending map — no shared lock.
162type Dispatch = (u64, Request, oneshot::Sender<Response>);
163
164/// What a watcher asks of the connection that carries the watches.
165enum WatchCommand {
166    /// Register `interest` under `watch` and wake `tx` when the server names
167    /// it.
168    Add {
169        watch: u64,
170        interest: WriteInterest,
171        tx: watch::Sender<u64>,
172    },
173    /// The last receiver of `watch` is gone.
174    Remove { watch: u64 },
175}
176
177/// The [`ClientFrame::Watch`] that registers `interest` under `watch`.
178fn watch_frame(watch: u64, interest: &WriteInterest) -> ClientFrame {
179    ClientFrame::Watch {
180        watch,
181        filters: interest.filters.as_deref().map(<[EventFilter]>::to_vec),
182        routing_key: interest.routing_key.clone(),
183        tenant: interest.tenant.clone(),
184    }
185}
186
187/// A remote [`Executor`]: forwards every call to a [`serve`](crate::serve)d
188/// executor over TCP. Cheap to clone; all clones share the connection pool.
189#[derive(Clone)]
190pub struct Client {
191    inner: Arc<Inner>,
192}
193
194struct Inner {
195    /// One request queue per pooled connection.
196    lanes: Vec<mpsc::Sender<Dispatch>>,
197    round_robin: AtomicUsize,
198    next_id: AtomicU64,
199    /// The watches, handed to the first connection's actor.
200    watches: mpsc::UnboundedSender<WatchCommand>,
201    next_watch: AtomicU64,
202    /// The channel of the unfiltered [`write_watch`](Executor::write_watch),
203    /// registered with the server by its first caller and kept from then on.
204    watch_all: std::sync::OnceLock<watch::Sender<u64>>,
205    /// Where the tasks that wait for a watcher to go away are spawned.
206    runtime: tokio::runtime::Handle,
207    /// Fixed at server executor construction; snapshotted once from the first
208    /// connection's `Hello` (it is borrowed as `&str`, so it must stay fixed).
209    default_routing_key: Option<String>,
210    stable: Stable,
211    request_timeout: Duration,
212}
213
214/// Builds a [`Client`] with non-default settings.
215pub struct ClientBuilder {
216    dial: Dial,
217    request_timeout: Duration,
218    connections: usize,
219    backoff: Backoff,
220}
221
222impl ClientBuilder {
223    /// The credential presented at `Hello`, on every connection of the pool
224    /// and again on each reconnect. What it means is the server's resolver's
225    /// business; an open server ignores it.
226    pub fn token(mut self, token: impl Into<String>) -> Self {
227        self.dial.token = Some(Token(token.into()));
228        self
229    }
230
231    /// Wraps every connection in TLS: `connector` carries the trust roots
232    /// (and the client certificate, for mutual TLS), `server_name` is what the
233    /// server's certificate is verified against.
234    pub fn tls(mut self, connector: TlsConnector, server_name: ServerName<'static>) -> Self {
235        self.dial.tls = Some(Tls {
236            connector,
237            server_name,
238        });
239        self
240    }
241
242    /// How long a single request may wait for its reply before failing
243    /// (default 30s). Raise for servers that can stall, e.g. an Accord-backed
244    /// store during recovery.
245    pub fn request_timeout(mut self, timeout: Duration) -> Self {
246        self.request_timeout = timeout;
247        self
248    }
249
250    /// Number of TCP connections in the pool (default 1, clamped to ≥ 1).
251    /// Requests are round-robined across them, so concurrent callers stop
252    /// sharing one socket's head-of-line and one server-side in-flight budget.
253    pub fn connections(mut self, n: usize) -> Self {
254        self.connections = n.max(1);
255        self
256    }
257
258    /// Bounds of the delay before a reconnect attempt (default 100ms and 5s;
259    /// `max` is raised to `min` when below it). The bound starts at `min`
260    /// and doubles up to `max` with every failed attempt; each delay, the one
261    /// before the first attempt included, is drawn at random in `[0, bound]`,
262    /// so clients dropped together spread out instead of redialling in step. Raise both when many clients share one
263    /// server: the wider the bound, the thinner the wave of `Hello`s a
264    /// restarted server takes.
265    pub fn reconnect_backoff(mut self, min: Duration, max: Duration) -> Self {
266        self.backoff = Backoff {
267            min,
268            max: max.max(min),
269        };
270        self
271    }
272
273    /// Connects and performs the `Hello` exchange on every pooled connection.
274    /// Fails fast when the server is unreachable or refuses the credential
275    /// (the error then carries the [`Rejection`]); after that each connection
276    /// is kept alive with automatic reconnect.
277    pub async fn connect(self) -> anyhow::Result<Client> {
278        let stable: Stable = Arc::new(AtomicU64::new(0));
279        let (watches, watch_rx) = mpsc::unbounded_channel();
280        let mut watch_rx = Some(watch_rx);
281
282        let mut lanes = Vec::with_capacity(self.connections);
283        let mut default_routing_key = None;
284        for lane in 0..self.connections {
285            let (framed, hello) =
286                connect_and_hello(&self.dial, &stable, self.request_timeout).await?;
287            if lane == 0 {
288                default_routing_key = hello.default_routing_key;
289            }
290            let (req_tx, req_rx) = mpsc::channel(REQ_CAPACITY);
291            tokio::spawn(
292                Actor {
293                    dial: self.dial.clone(),
294                    req_rx,
295                    // One connection carries the watches: a watch is a
296                    // level-triggered wakeup, registering it on every
297                    // connection would add nothing.
298                    watch_rx: if lane == 0 { watch_rx.take() } else { None },
299                    watches: HashMap::new(),
300                    stable: Arc::clone(&stable),
301                    request_timeout: self.request_timeout,
302                    backoff: self.backoff,
303                }
304                .run(framed),
305            );
306            lanes.push(req_tx);
307        }
308
309        Ok(Client {
310            inner: Arc::new(Inner {
311                lanes,
312                round_robin: AtomicUsize::new(0),
313                next_id: AtomicU64::new(HELLO_ID + 1),
314                watches,
315                next_watch: AtomicU64::new(0),
316                watch_all: std::sync::OnceLock::new(),
317                runtime: tokio::runtime::Handle::current(),
318                default_routing_key,
319                stable,
320                request_timeout: self.request_timeout,
321            }),
322        })
323    }
324}
325
326impl Client {
327    /// Connects with default settings. See [`Client::builder`] for knobs.
328    pub async fn connect(addr: impl Into<Address>) -> anyhow::Result<Client> {
329        Client::builder(addr).connect().await
330    }
331
332    /// Starts building a client for `addr` (token, TLS, request timeout,
333    /// connection count, reconnect backoff). A host name in `addr` is looked up on every
334    /// connection attempt, not once here.
335    pub fn builder(addr: impl Into<Address>) -> ClientBuilder {
336        ClientBuilder {
337            dial: Dial {
338                addr: addr.into(),
339                token: None,
340                tls: None,
341            },
342            request_timeout: DEFAULT_REQUEST_TIMEOUT,
343            connections: 1,
344            backoff: Backoff {
345                min: RECONNECT_MIN,
346                max: RECONNECT_MAX,
347            },
348        }
349    }
350
351    async fn request(&self, request: Request) -> anyhow::Result<Response> {
352        // Ids only need uniqueness per connection (they are per-connection on
353        // the wire); a global counter is a superset of that.
354        let id = self.inner.next_id.fetch_add(1, Ordering::Relaxed);
355        let lane = self.inner.round_robin.fetch_add(1, Ordering::Relaxed) % self.inner.lanes.len();
356        let (tx, rx) = oneshot::channel();
357        if self.inner.lanes[lane]
358            .send((id, request, tx))
359            .await
360            .is_err()
361        {
362            anyhow::bail!("remote executor connection is closed");
363        }
364        match tokio::time::timeout(self.inner.request_timeout, rx).await {
365            Ok(Ok(response)) => Ok(response),
366            Ok(Err(_)) => anyhow::bail!("remote executor connection lost"),
367            Err(_) => anyhow::bail!(
368                "remote request timed out after {:?}",
369                self.inner.request_timeout
370            ),
371        }
372    }
373}
374
375impl Inner {
376    /// Registers `interest` with the server: the id of the watch and the
377    /// sender of the channel its notifies wake.
378    fn watch(&self, interest: WriteInterest) -> (u64, watch::Sender<u64>) {
379        let (tx, _) = watch::channel(0u64);
380        let watch = self.next_watch.fetch_add(1, Ordering::Relaxed);
381        // A closed channel means the actor is gone, and with it every
382        // connection: nothing would wake the watcher anyway.
383        let _ = self.watches.send(WatchCommand::Add {
384            watch,
385            interest,
386            tx: tx.clone(),
387        });
388        (watch, tx)
389    }
390}
391
392fn protocol_err(what: &str) -> anyhow::Error {
393    anyhow::anyhow!("remote protocol error: unexpected response to {what}")
394}
395
396#[async_trait::async_trait]
397impl Executor for Client {
398    fn default_routing_key(&self) -> Option<&str> {
399        self.inner.default_routing_key.as_deref()
400    }
401
402    fn write_watch(&self) -> Option<watch::Receiver<u64>> {
403        let all = self.inner.watch_all.get_or_init(|| {
404            self.inner
405                .watch(WriteInterest {
406                    filters: None,
407                    routing_key: None,
408                    tenant: TenantFilter::All,
409                })
410                .1
411        });
412        Some(all.subscribe())
413    }
414
415    fn write_watch_for(&self, interest: &WriteInterest) -> Option<watch::Receiver<u64>> {
416        let (watch, tx) = self.inner.watch(interest.clone());
417        let rx = tx.subscribe();
418        // The server keeps a watch until it is told to drop it: say so once
419        // the last receiver is gone.
420        let watches = self.inner.watches.clone();
421        self.inner.runtime.spawn(async move {
422            tx.closed().await;
423            let _ = watches.send(WatchCommand::Remove { watch });
424        });
425        Some(rx)
426    }
427
428    async fn stable_timestamp(&self) -> anyhow::Result<Option<u64>> {
429        // The cached watermark is refreshed by every response and notify frame;
430        // a polling subscription issues several requests per pass, so the cache
431        // is at most one round-trip stale by the time the gate consults it.
432        match self.inner.stable.load(Ordering::Acquire) {
433            0 => Ok(None),
434            v => Ok(Some(v)),
435        }
436    }
437
438    async fn write(&self, events: Vec<Event>) -> Result<(), WriteError> {
439        match self.request(Request::Write { events }).await {
440            Ok(Response::Write(Ok(()))) => Ok(()),
441            Ok(Response::Write(Err(e))) => Err(e.into()),
442            Ok(_) => Err(WriteError::Unknown(protocol_err("write"))),
443            Err(e) => Err(WriteError::Unknown(e)),
444        }
445    }
446
447    async fn replicate(&self, events: Vec<Event>) -> Result<(), WriteError> {
448        match self.request(Request::Replicate { events }).await {
449            Ok(Response::Write(Ok(()))) => Ok(()),
450            Ok(Response::Write(Err(e))) => Err(e.into()),
451            Ok(_) => Err(WriteError::Unknown(protocol_err("replicate"))),
452            Err(e) => Err(WriteError::Unknown(e)),
453        }
454    }
455
456    async fn read(
457        &self,
458        aggregators: Option<Arc<[EventFilter]>>,
459        routing_key: Option<RoutingKey>,
460        tenant: TenantFilter,
461        args: Args,
462        to_micros: Option<u64>,
463    ) -> anyhow::Result<ReadResult<Event>> {
464        match self
465            .request(Request::Read {
466                aggregators: aggregators.map(|a| a.to_vec()),
467                routing_key,
468                tenant,
469                args,
470                to_micros,
471            })
472            .await?
473        {
474            Response::Read(Ok(wire)) => Ok(wire.into()),
475            Response::Read(Err(msg)) => Err(anyhow::anyhow!(msg)),
476            _ => Err(protocol_err("read")),
477        }
478    }
479
480    async fn latest_timestamp(
481        &self,
482        aggregators: Option<Arc<[EventFilter]>>,
483        routing_key: Option<RoutingKey>,
484        tenant: TenantFilter,
485    ) -> anyhow::Result<u64> {
486        match self
487            .request(Request::LatestTimestamp {
488                aggregators: aggregators.map(|a| a.to_vec()),
489                routing_key,
490                tenant,
491            })
492            .await?
493        {
494            Response::LatestTimestamp(Ok(v)) => Ok(v),
495            Response::LatestTimestamp(Err(msg)) => Err(anyhow::anyhow!(msg)),
496            _ => Err(protocol_err("latest_timestamp")),
497        }
498    }
499
500    async fn get_subscriber_cursor(&self, key: String) -> anyhow::Result<Option<Value>> {
501        match self.request(Request::GetSubscriberCursor { key }).await? {
502            Response::SubscriberCursor(Ok(v)) => Ok(v),
503            Response::SubscriberCursor(Err(msg)) => Err(anyhow::anyhow!(msg)),
504            _ => Err(protocol_err("get_subscriber_cursor")),
505        }
506    }
507
508    async fn is_subscriber_running(&self, key: String, worker_id: Ulid) -> anyhow::Result<bool> {
509        match self
510            .request(Request::IsSubscriberRunning { key, worker_id })
511            .await?
512        {
513            Response::SubscriberRunning(Ok(v)) => Ok(v),
514            Response::SubscriberRunning(Err(msg)) => Err(anyhow::anyhow!(msg)),
515            _ => Err(protocol_err("is_subscriber_running")),
516        }
517    }
518
519    async fn subscriber_status(
520        &self,
521        key: String,
522        worker_id: Ulid,
523    ) -> anyhow::Result<evento_core::SubscriberStatus> {
524        match self
525            .request(Request::SubscriberStatus { key, worker_id })
526            .await?
527        {
528            Response::SubscriberStatus(Ok(v)) => Ok(v),
529            Response::SubscriberStatus(Err(msg)) => Err(anyhow::anyhow!(msg)),
530            _ => Err(protocol_err("subscriber_status")),
531        }
532    }
533
534    async fn latest_version(
535        &self,
536        tenant: String,
537        aggregate_type: String,
538        aggregate_id: String,
539    ) -> anyhow::Result<u16> {
540        match self
541            .request(Request::LatestVersion {
542                tenant,
543                aggregate_type,
544                aggregate_id,
545            })
546            .await?
547        {
548            Response::LatestVersion(Ok(v)) => Ok(v),
549            Response::LatestVersion(Err(msg)) => Err(anyhow::anyhow!(msg)),
550            _ => Err(protocol_err("latest_version")),
551        }
552    }
553
554    async fn stream_routing_key(
555        &self,
556        tenant: String,
557        aggregate_type: String,
558        aggregate_id: String,
559    ) -> anyhow::Result<Option<Option<String>>> {
560        match self
561            .request(Request::StreamRoutingKey {
562                tenant,
563                aggregate_type,
564                aggregate_id,
565            })
566            .await?
567        {
568            Response::StreamRoutingKey(Ok(v)) => Ok(v),
569            Response::StreamRoutingKey(Err(msg)) => Err(anyhow::anyhow!(msg)),
570            _ => Err(protocol_err("stream_routing_key")),
571        }
572    }
573
574    async fn upsert_subscriber(&self, key: String, worker_id: Ulid) -> anyhow::Result<()> {
575        match self
576            .request(Request::UpsertSubscriber { key, worker_id })
577            .await?
578        {
579            Response::Unit(Ok(())) => Ok(()),
580            Response::Unit(Err(msg)) => Err(anyhow::anyhow!(msg)),
581            _ => Err(protocol_err("upsert_subscriber")),
582        }
583    }
584
585    async fn acknowledge(
586        &self,
587        key: String,
588        worker_id: Ulid,
589        cursor: Value,
590        lag: u64,
591    ) -> anyhow::Result<bool> {
592        match self
593            .request(Request::Acknowledge {
594                key,
595                worker_id,
596                cursor,
597                lag,
598            })
599            .await?
600        {
601            Response::Acknowledge(Ok(v)) => Ok(v),
602            Response::Acknowledge(Err(msg)) => Err(anyhow::anyhow!(msg)),
603            _ => Err(protocol_err("acknowledge")),
604        }
605    }
606
607    async fn get_snapshot(
608        &self,
609        tenant: String,
610        aggregate_type: String,
611        projection: String,
612        aggregate_revision: String,
613        id: String,
614    ) -> anyhow::Result<Option<(Vec<u8>, Value)>> {
615        match self
616            .request(Request::GetSnapshot {
617                tenant,
618                aggregate_type,
619                projection,
620                aggregate_revision,
621                id,
622            })
623            .await?
624        {
625            Response::Snapshot(Ok(v)) => Ok(v),
626            Response::Snapshot(Err(msg)) => Err(anyhow::anyhow!(msg)),
627            _ => Err(protocol_err("get_snapshot")),
628        }
629    }
630
631    async fn save_snapshot(
632        &self,
633        tenant: String,
634        aggregate_type: String,
635        projection: String,
636        aggregate_revision: String,
637        id: String,
638        data: Vec<u8>,
639        cursor: Value,
640    ) -> anyhow::Result<()> {
641        match self
642            .request(Request::SaveSnapshot {
643                tenant,
644                aggregate_type,
645                projection,
646                aggregate_revision,
647                id,
648                data,
649                cursor,
650            })
651            .await?
652        {
653            Response::Unit(Ok(())) => Ok(()),
654            Response::Unit(Err(msg)) => Err(anyhow::anyhow!(msg)),
655            _ => Err(protocol_err("save_snapshot")),
656        }
657    }
658
659    async fn delete_snapshot(
660        &self,
661        tenant: String,
662        aggregate_type: String,
663        projection: String,
664        id: String,
665    ) -> anyhow::Result<()> {
666        match self
667            .request(Request::DeleteSnapshot {
668                tenant,
669                aggregate_type,
670                projection,
671                id,
672            })
673            .await?
674        {
675            Response::Unit(Ok(())) => Ok(()),
676            Response::Unit(Err(msg)) => Err(anyhow::anyhow!(msg)),
677            _ => Err(protocol_err("delete_snapshot")),
678        }
679    }
680}
681
682/// Owns one framed connection of the pool: sends dispatched requests,
683/// demultiplexes replies via its **own** pending map (the reply channel
684/// arrives with each request, so there is no shared lock), on the first
685/// connection registers the watches and wakes the ones a pushed notification
686/// names, reconnects with capped exponential backoff. Exits when every
687/// [`Client`] clone is dropped.
688struct Actor {
689    dial: Dial,
690    req_rx: mpsc::Receiver<Dispatch>,
691    /// `Some` on the connection that carries the watches only.
692    watch_rx: Option<mpsc::UnboundedReceiver<WatchCommand>>,
693    /// The watches registered through this connection, by id. They outlive
694    /// a connection: each is registered again on the next one.
695    watches: HashMap<u64, (WriteInterest, watch::Sender<u64>)>,
696    stable: Stable,
697    request_timeout: Duration,
698    backoff: Backoff,
699}
700
701impl Actor {
702    async fn run(mut self, initial: Framed<ClientStream, LengthDelimitedCodec>) {
703        let mut conn = Some(initial);
704        // In-flight requests on the current connection. Cleared (failing the
705        // callers) on disconnect: a possibly-executed request must never be
706        // replayed on the next connection — the caller may retry it under a
707        // fresh id. Requests still queued in `req_rx` at that point were never
708        // sent, so their callers keep waiting and they go out on the new
709        // connection safely.
710        let mut pending: HashMap<u64, oneshot::Sender<Response>> = HashMap::new();
711        loop {
712            let framed = match conn.take() {
713                Some(framed) => framed,
714                None => match self.reconnect().await {
715                    Some(framed) => framed,
716                    None => return, // all clients dropped
717                },
718            };
719            let (mut sink, mut inbound) = framed.split();
720            // Dedicated writer task: the read side must keep draining even
721            // while an outbound send is blocked on TCP backpressure, or a
722            // pipelining client and a slow server can deadlock head-of-line
723            // (each side blocked sending, neither reading).
724            let (out_tx, mut out_rx) = mpsc::channel::<Bytes>(OUT_CAPACITY);
725            let writer = tokio::spawn(async move {
726                // Coalesce: feed this frame plus everything already queued,
727                // then flush once — one syscall per burst instead of one per
728                // frame (`send` = `feed` + `flush`).
729                'writer: while let Some(bytes) = out_rx.recv().await {
730                    if sink.feed(bytes).await.is_err() {
731                        break;
732                    }
733                    while let Ok(bytes) = out_rx.try_recv() {
734                        if sink.feed(bytes).await.is_err() {
735                            break 'writer;
736                        }
737                    }
738                    if sink.flush().await.is_err() {
739                        break;
740                    }
741                }
742            });
743            // The server knows the watches of a connection, not of a client:
744            // a new connection is told all of them again. It names each one
745            // as it registers it, which wakes the watchers for whatever was
746            // written while there was no connection.
747            'connected: {
748                for (watch, (interest, _)) in &self.watches {
749                    let Ok(bytes) =
750                        encode_tagged(RecordKind::ClientFrame, &watch_frame(*watch, interest))
751                    else {
752                        continue;
753                    };
754                    if out_tx.send(Bytes::from(bytes)).await.is_err() {
755                        break 'connected;
756                    }
757                }
758                loop {
759                    tokio::select! {
760                        command = next_watch_command(&mut self.watch_rx) => {
761                            let frame = match command {
762                                WatchCommand::Add { watch, interest, tx } => {
763                                    let frame = watch_frame(watch, &interest);
764                                    self.watches.insert(watch, (interest, tx));
765                                    frame
766                                }
767                                WatchCommand::Remove { watch } => {
768                                    self.watches.remove(&watch);
769                                    ClientFrame::Unwatch { watch }
770                                }
771                            };
772                            let Ok(bytes) = encode_tagged(RecordKind::ClientFrame, &frame) else {
773                                continue;
774                            };
775                            if out_tx.send(Bytes::from(bytes)).await.is_err() {
776                                break;
777                            }
778                        }
779                        item = self.req_rx.recv() => {
780                            let Some((id, request, reply_tx)) = item else { break };
781                            let frame = ClientFrame::Request { id, request };
782                            let Ok(bytes) = encode_tagged(RecordKind::ClientFrame, &frame) else {
783                                continue; // dropping reply_tx fails the caller
784                            };
785                            pending.insert(id, reply_tx);
786                            if out_tx.send(Bytes::from(bytes)).await.is_err() {
787                                pending.remove(&id);
788                                break;
789                            }
790                        }
791                        item = inbound.next() => {
792                            let Some(Ok(bytes)) = item else { break };
793                            match decode_tagged::<ServerFrame>(RecordKind::ServerFrame, &bytes) {
794                                Ok(ServerFrame::Response { id, response, stable_timestamp }) => {
795                                    self.update_stable(stable_timestamp);
796                                    // An unknown id is a reply to a request that
797                                    // already timed out (its caller is gone — the
798                                    // send below just fails harmlessly).
799                                    if let Some(tx) = pending.remove(&id) {
800                                        let _ = tx.send(response);
801                                    }
802                                }
803                                Ok(ServerFrame::Notify { watches, stable_timestamp }) => {
804                                    self.update_stable(stable_timestamp);
805                                    for watch in watches {
806                                        // An unknown id is a watch dropped while
807                                        // its notify was on the way.
808                                        if let Some((_, tx)) = self.watches.get(&watch) {
809                                            tx.send_modify(|g| *g += 1);
810                                        }
811                                    }
812                                }
813                                Err(err) => {
814                                    tracing::debug!(?err, "undecodable frame, reconnecting");
815                                    break;
816                                }
817                            }
818                        }
819                    }
820                }
821            }
822            // Disconnected (or all clients dropped): stop the writer, fail
823            // everything in flight (dropping the reply senders surfaces
824            // "connection lost" at each caller), then reconnect.
825            drop(out_tx);
826            let _ = writer.await;
827            pending.clear();
828            if self.req_rx.is_closed() {
829                return;
830            }
831        }
832    }
833
834    /// Reconnects with capped, jittered exponential backoff. `None` when every
835    /// client clone has been dropped (nothing left to serve).
836    async fn reconnect(&mut self) -> Option<Framed<ClientStream, LengthDelimitedCodec>> {
837        let mut bound = self.backoff.min;
838        loop {
839            // The first attempt waits too: where a load balancer moved the
840            // name, it succeeds, and every client dropped with this one would
841            // otherwise say `Hello` to the new node at the same instant.
842            tokio::time::sleep(self.backoff.next_delay(&mut bound)).await;
843            if self.req_rx.is_closed() {
844                return None;
845            }
846            match connect_and_hello(&self.dial, &self.stable, self.request_timeout).await {
847                // The routing key from a re-Hello is ignored: it is fixed at
848                // server construction, and the cached one must stay stable for
849                // the lifetime of this Client (it is borrowed as &str).
850                Ok((framed, _)) => return Some(framed),
851                Err(err) => {
852                    // A refusal is the server's decision (the credential was
853                    // revoked, the store is down), worth more than a debug
854                    // line; it is retried like any failure, the resolver may
855                    // accept again.
856                    if let Some(rejection) = err.downcast_ref::<Rejection>() {
857                        tracing::warn!(%rejection, addr = %self.dial.addr, "the server refused the reconnect, backing off");
858                    } else {
859                        tracing::debug!(?err, addr = %self.dial.addr, "reconnect failed, backing off");
860                    }
861                }
862            }
863        }
864    }
865
866    fn update_stable(&self, stable_timestamp: Option<u64>) {
867        if let Some(v) = stable_timestamp {
868            self.stable.fetch_max(v, Ordering::AcqRel);
869        }
870    }
871}
872
873/// The next watch command; never resolves on a connection that carries no
874/// watches, or once every [`Client`] clone is gone (the request queue closing
875/// is what ends the actor).
876async fn next_watch_command(
877    rx: &mut Option<mpsc::UnboundedReceiver<WatchCommand>>,
878) -> WatchCommand {
879    if let Some(commands) = rx {
880        if let Some(command) = commands.recv().await {
881            return command;
882        }
883        *rx = None;
884    }
885    std::future::pending().await
886}
887
888/// Establishes one connection (TLS handshake included) and performs the
889/// `Hello` exchange, returning the framed stream and what the server answered
890/// (its `default_routing_key`). Seeds the stable watermark from the reply.
891/// Frames
892/// pushed before the reply (notifies) only update the watermark. A refusal
893/// comes back as an error whose source is the [`Rejection`].
894async fn connect_and_hello(
895    dial: &Dial,
896    stable: &AtomicU64,
897    timeout: Duration,
898) -> anyhow::Result<(Framed<ClientStream, LengthDelimitedCodec>, HelloOk)> {
899    // A host name is looked up here, on every attempt, and each address it
900    // resolves to is tried in turn; an IP is used as is.
901    let stream = TcpStream::connect(dial.addr.0.as_str())
902        .await
903        .with_context(|| format!("cannot connect to {}", dial.addr))?;
904    let _ = stream.set_nodelay(true);
905    let stream = match &dial.tls {
906        None => Either::Left(stream),
907        Some(tls) => Either::Right(
908            tokio::time::timeout(
909                timeout,
910                tls.connector.connect(tls.server_name.clone(), stream),
911            )
912            .await
913            .map_err(|_| anyhow::anyhow!("TLS handshake timed out after {timeout:?}"))??,
914        ),
915    };
916    let mut framed = Framed::new(stream, codec());
917
918    let hello = ClientFrame::Request {
919        id: HELLO_ID,
920        request: Request::Hello {
921            token: dial.token.clone(),
922        },
923    };
924    framed
925        .send(Bytes::from(encode_tagged(RecordKind::ClientFrame, &hello)?))
926        .await?;
927
928    let hello = tokio::time::timeout(timeout, async {
929        loop {
930            let Some(item) = framed.next().await else {
931                anyhow::bail!("connection closed during hello");
932            };
933            match decode_tagged::<ServerFrame>(RecordKind::ServerFrame, &item?)? {
934                ServerFrame::Response {
935                    id: HELLO_ID,
936                    response: Response::Hello(reply),
937                    stable_timestamp,
938                } => {
939                    if let Some(v) = stable_timestamp {
940                        stable.fetch_max(v, Ordering::AcqRel);
941                    }
942                    let ok = reply
943                        .map_err(anyhow::Error::new)
944                        .context("the server refused the connection")?;
945                    return Ok(ok);
946                }
947                ServerFrame::Notify {
948                    stable_timestamp, ..
949                } => {
950                    if let Some(v) = stable_timestamp {
951                        stable.fetch_max(v, Ordering::AcqRel);
952                    }
953                }
954                _ => anyhow::bail!("unexpected frame during hello"),
955            }
956        }
957    })
958    .await
959    .map_err(|_| anyhow::anyhow!("hello timed out after {timeout:?}"))??;
960
961    Ok((framed, hello))
962}
963
964#[cfg(test)]
965mod tests {
966    use super::*;
967
968    /// Issue #535: a watcher's filter is sent to the server, and the server
969    /// is told once its last receiver is gone. The unfiltered watch is
970    /// registered by its first caller only.
971    #[tokio::test]
972    async fn a_watch_is_registered_and_dropped_with_its_receivers() {
973        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
974        let addr = listener.local_addr().unwrap();
975        let server = tokio::spawn(async move {
976            let (stream, _) = listener.accept().await.unwrap();
977            let mut wire = Framed::new(stream, codec());
978            let mut seen = Vec::new();
979            while seen.len() < 4 {
980                let bytes = wire.next().await.unwrap().unwrap();
981                match decode_tagged::<ClientFrame>(RecordKind::ClientFrame, &bytes).unwrap() {
982                    ClientFrame::Request { id, .. } => {
983                        let reply = ServerFrame::Response {
984                            id,
985                            response: Response::Hello(Ok(HelloOk {
986                                default_routing_key: None,
987                            })),
988                            stable_timestamp: None,
989                        };
990                        let bytes = encode_tagged(RecordKind::ServerFrame, &reply).unwrap();
991                        wire.send(Bytes::from(bytes)).await.unwrap();
992                    }
993                    frame => seen.push(frame),
994                }
995            }
996            seen
997        });
998
999        let client = Client::connect(addr).await.unwrap();
1000        let interest = WriteInterest {
1001            filters: Some([EventFilter::by_type_raw("test/Account")].into()),
1002            routing_key: Some(RoutingKey::Value(None)),
1003            tenant: TenantFilter::All,
1004        };
1005        let accounts = client.write_watch_for(&interest).unwrap();
1006        let clone = accounts.clone();
1007        let all = client.write_watch().unwrap();
1008        let all_again = client.write_watch().unwrap();
1009        drop(accounts);
1010        let cards = client
1011            .write_watch_for(&WriteInterest {
1012                filters: Some([EventFilter::by_type_raw("test/Card")].into()),
1013                ..interest
1014            })
1015            .unwrap();
1016        drop(clone);
1017
1018        let seen = tokio::time::timeout(Duration::from_secs(5), server)
1019            .await
1020            .expect("four watch frames within 5s")
1021            .unwrap();
1022        let [ClientFrame::Watch {
1023            watch: accounts,
1024            filters: Some(filters),
1025            routing_key: Some(RoutingKey::Value(None)),
1026            tenant: TenantFilter::All,
1027        }, ClientFrame::Watch {
1028            watch: unfiltered,
1029            filters: None,
1030            routing_key: None,
1031            tenant: TenantFilter::All,
1032        }, ClientFrame::Watch {
1033            watch: cards_id, ..
1034        }, ClientFrame::Unwatch { watch: dropped }] = &seen[..]
1035        else {
1036            panic!("unexpected frames: {seen:?}");
1037        };
1038        assert_eq!(filters, &[EventFilter::by_type_raw("test/Account")]);
1039        assert_eq!(dropped, accounts);
1040        assert!(accounts != unfiltered && unfiltered != cards_id && accounts != cards_id);
1041        drop((all, all_again, cards));
1042    }
1043
1044    #[test]
1045    fn the_reconnect_delay_stays_below_a_doubling_bound() {
1046        let backoff = Backoff {
1047            min: Duration::from_millis(100),
1048            max: Duration::from_secs(5),
1049        };
1050        let mut bound = backoff.min;
1051        let mut bounds = Vec::new();
1052        for _ in 0..8 {
1053            let before = bound;
1054            assert!(backoff.next_delay(&mut bound) <= before);
1055            bounds.push(before.as_millis());
1056        }
1057        assert_eq!(bounds, [100, 200, 400, 800, 1600, 3200, 5000, 5000]);
1058    }
1059
1060    #[test]
1061    fn two_clients_do_not_share_a_reconnect_schedule() {
1062        let backoff = Backoff {
1063            min: Duration::from_secs(5),
1064            max: Duration::from_secs(5),
1065        };
1066        let schedule = || {
1067            let mut bound = backoff.min;
1068            (0..16)
1069                .map(|_| backoff.next_delay(&mut bound))
1070                .collect::<Vec<_>>()
1071        };
1072        assert_ne!(schedule(), schedule());
1073    }
1074
1075    #[test]
1076    fn a_reconnect_maximum_below_the_minimum_is_raised_to_it() {
1077        let builder = Client::builder("127.0.0.1:1")
1078            .reconnect_backoff(Duration::from_secs(2), Duration::from_secs(1));
1079        assert_eq!(builder.backoff.min, Duration::from_secs(2));
1080        assert_eq!(builder.backoff.max, Duration::from_secs(2));
1081    }
1082}