Skip to main content

evento_remote/
server.rs

1//! Server side: expose executors over framed TCP, plain or TLS.
2//!
3//! Every connection starts with a [`Hello`] exchange: the client presents an
4//! optional token, and a [`Resolver`] decides which executor that connection
5//! is served with — or refuses it with a [`Rejection`]. [`serve`] is the
6//! one-executor case (an open server, every connection gets the same store);
7//! [`serve_with`] takes the resolver that a host authenticating credentials
8//! and multiplexing stores needs. The `_tls` variants complete a rustls
9//! handshake first, mirroring `evento-cluster`'s `serve`/`serve_tls` split.
10//!
11//! Requests on one connection are handled **concurrently** (spawn per
12//! request) — multiple application tasks share one client connection, and a
13//! slow `write` (e.g. a consensus round on an Accord-backed server) must not
14//! head-of-line-block unrelated reads; correlation ids make out-of-order
15//! replies safe.
16//!
17//! A connection is told of the writes its client watches for, and of no
18//! other: the client registers each watch with the filter its reads use
19//! ([`ClientFrame::Watch`]), and a [`ServerFrame::Notify`] names the watches a
20//! write wakes. A watch is fed from the executor's own `write_watch_for` when
21//! it has a write signal, otherwise from a server-wide signal fired after
22//! each successful `write` handled by any connection (still low-latency for
23//! writes through this server; writers bypassing it are covered by the
24//! client's poll-interval fallback, the same caveat every `write_watch`
25//! backend documents). A connection without a write signal may thus be woken
26//! by a write to another executor; the watch is level-triggered, so a
27//! spurious wakeup costs one empty poll. On a connection serving a tenant a
28//! watch follows that tenant's writes alone.
29//!
30//! A session may serve one tenant ([`Session::tenant`]): every request on the
31//! connection then acts in that tenant, whatever tenant it names, and the
32//! requests that would reach past it are refused.
33
34use std::any::Any;
35use std::collections::HashMap;
36use std::net::SocketAddr;
37use std::sync::Arc;
38use std::time::Duration;
39
40use bytes::Bytes;
41use evento_core::{Executor, TenantError, TenantFilter, WriteInterest, WriteSignal};
42use futures_util::stream::SplitStream;
43use futures_util::{SinkExt, StreamExt};
44use tokio::net::{TcpListener, TcpStream};
45use tokio::sync::{mpsc, watch};
46use tokio::task::{AbortHandle, JoinHandle};
47use tokio_rustls::TlsAcceptor;
48use tokio_util::codec::{Framed, LengthDelimitedCodec};
49use tokio_util::either::Either;
50
51use crate::wire::{
52    encode_tagged, ClientFrame, HelloOk, RecordKind, Rejection, Request, Response, ServerFrame,
53    WireReadResult, WireWriteError, MAX_FRAME_LENGTH,
54};
55
56/// Capacity of a connection's outbound queue. Sends into it are awaited, so a
57/// slow client backpressures its own request handlers instead of shedding
58/// replies.
59const OUT_CAPACITY: usize = 1024;
60
61/// Maximum concurrently-executing request handlers per connection. Without a
62/// cap a client pipelining arbitrarily many large frames could spawn unbounded
63/// tasks; at the cap the read loop stops pulling frames (TCP backpressure).
64const MAX_INFLIGHT_PER_CONNECTION: usize = 256;
65
66/// Maximum watches a connection may hold, each a task and a receiver on the
67/// executor's write signal. Past it a watch is not registered and its
68/// watcher is left to its poll interval.
69const MAX_WATCHES_PER_CONNECTION: usize = 65_536;
70
71/// Capacity of the queue between a connection's watches and the task that
72/// names them in a [`ServerFrame::Notify`]. A watch that finds it full waits:
73/// the writes that land meanwhile are covered by the one notify that follows.
74const WOKEN_CAPACITY: usize = 1024;
75
76/// A dialer that never completes the TLS handshake, or never sends `Hello`,
77/// must not pin a task forever.
78const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);
79const HELLO_TIMEOUT: Duration = Duration::from_secs(10);
80
81/// Inbound connection: plain TCP, or a server-side TLS session over it.
82type ServerStream = Either<TcpStream, tokio_rustls::server::TlsStream<TcpStream>>;
83
84fn codec() -> LengthDelimitedCodec {
85    LengthDelimitedCodec::builder()
86        .max_frame_length(MAX_FRAME_LENGTH)
87        .new_codec()
88}
89
90/// What a client said when it connected, and where from.
91#[derive(Debug, Clone)]
92pub struct Hello {
93    /// The credential presented, if any. Its meaning is the resolver's.
94    pub token: Option<String>,
95    /// The peer address (after TLS, if any).
96    pub peer: SocketAddr,
97}
98
99/// What a connection is served with, as a [`Resolver`] builds it.
100pub struct Session<E> {
101    executor: E,
102    tenant: Option<Arc<str>>,
103    guard: Option<Box<dyn Any + Send + Sync>>,
104    close: Option<watch::Receiver<bool>>,
105}
106
107impl<E> Session<E> {
108    /// A session over `executor`, kept for the whole connection.
109    pub fn new(executor: E) -> Self {
110        Self {
111            executor,
112            tenant: None,
113            guard: None,
114            close: None,
115        }
116    }
117
118    /// Serves the connection in `tenant` alone, whatever tenant its requests
119    /// name: reads and loads see only `tenant`'s events, writes land in
120    /// `tenant`, and the version, routing-key and snapshot lookups are
121    /// `tenant`'s. A host that maps a client's credential to a tenant in its
122    /// [`Resolver`] confines the client this way, over one store shared by
123    /// every tenant.
124    ///
125    /// The client sees a store of its own: the events it reads come back in
126    /// the default tenant (`""`), so it names no tenant anywhere — its
127    /// projections, snapshots and cursors are kept as against a store of a
128    /// single tenant.
129    ///
130    /// Requests that would reach past the tenant are refused: `replicate`,
131    /// which keeps the caller's timestamps (an event backdated behind the
132    /// cursor of a subscription over the shared store would never reach it),
133    /// and the subscriber calls, since a subscriber row belongs to no tenant.
134    /// A tenant's application keeps its subscriptions in a store of its own:
135    /// a `Sidecar` (evento-core's `rw` feature) with the client as its events
136    /// source.
137    ///
138    /// # Errors
139    ///
140    /// [`TenantError`] for `""` — the default tenant, which a session serving
141    /// no tenant reaches along with every other — or for a tenant id longer
142    /// than a store keeps.
143    pub fn tenant(mut self, tenant: impl Into<String>) -> Result<Self, TenantError> {
144        let tenant = tenant.into();
145        TenantError::check(&tenant)?;
146        self.tenant = Some(tenant.into());
147        Ok(self)
148    }
149
150    /// Keeps `guard` alive until the connection ends, then drops it. A host
151    /// leases a tenant to the connection this way: the lease is released when
152    /// the client goes away, whatever the reason.
153    pub fn keep(mut self, guard: impl Any + Send + Sync) -> Self {
154        self.guard = Some(Box::new(guard));
155        self
156    }
157
158    /// Ends the connection from the server's side when `signal` turns `true`
159    /// (a credential was revoked). In-flight requests fail at the client, and
160    /// its reconnect goes through the resolver again, which can now refuse it.
161    pub fn close_when(mut self, signal: watch::Receiver<bool>) -> Self {
162        self.close = Some(signal);
163        self
164    }
165}
166
167/// Picks the executor a connection is served with, per connection, from what
168/// the client presented at [`Hello`]: authentication and store multiplexing
169/// in one hook. Called once per connection (and again on each reconnect of a
170/// client, which is a new connection), before any request is handled.
171#[async_trait::async_trait]
172pub trait Resolver: Send + Sync + 'static {
173    /// What every connection of this server is served with.
174    type Executor: Executor + Clone;
175
176    /// The session for a connection that said `hello`, or why it is refused.
177    /// The refusal reaches the client as the error of its `connect`.
178    async fn resolve(&self, hello: &Hello) -> Result<Session<Self::Executor>, Rejection>;
179}
180
181/// The trivial resolver: every connection gets the same executor.
182struct Static<E>(E);
183
184#[async_trait::async_trait]
185impl<E: Executor + Clone> Resolver for Static<E> {
186    type Executor = E;
187
188    async fn resolve(&self, _hello: &Hello) -> Result<Session<E>, Rejection> {
189        Ok(Session::new(self.0.clone()))
190    }
191}
192
193/// Handle to a running server. Dropping it does **not** stop the server; call
194/// [`shutdown`](ServerHandle::shutdown).
195pub struct ServerHandle {
196    accept_task: JoinHandle<()>,
197    shutdown_tx: watch::Sender<bool>,
198}
199
200impl ServerHandle {
201    /// Stops accepting connections and closes every established one. In-flight
202    /// request handlers may still complete against the executor, but their
203    /// replies are no longer delivered.
204    pub async fn shutdown(self) {
205        let _ = self.shutdown_tx.send(true);
206        let _ = self.accept_task.await;
207    }
208}
209
210/// Serves `executor` to every connection on `listener`, plaintext, until
211/// [`ServerHandle::shutdown`]. Tokens presented at `Hello` are ignored.
212pub fn serve<E: Executor + Clone>(listener: TcpListener, executor: E) -> ServerHandle {
213    serve_with(listener, Static(executor))
214}
215
216/// Serves each connection on `listener` with the executor `resolver` picks
217/// for it, plaintext, until [`ServerHandle::shutdown`].
218pub fn serve_with<R: Resolver>(listener: TcpListener, resolver: R) -> ServerHandle {
219    start(listener, resolver, None)
220}
221
222/// Like [`serve`], completing a TLS handshake with `acceptor` on each
223/// connection first. Client-certificate verification is the acceptor's
224/// configuration; the resolver does not see the certificate.
225pub fn serve_tls<E: Executor + Clone>(
226    listener: TcpListener,
227    acceptor: TlsAcceptor,
228    executor: E,
229) -> ServerHandle {
230    serve_tls_with(listener, acceptor, Static(executor))
231}
232
233/// Like [`serve_with`], completing a TLS handshake with `acceptor` on each
234/// connection first.
235pub fn serve_tls_with<R: Resolver>(
236    listener: TcpListener,
237    acceptor: TlsAcceptor,
238    resolver: R,
239) -> ServerHandle {
240    start(listener, resolver, Some(acceptor))
241}
242
243/// The accept loop behind every `serve*` flavour.
244fn start<R: Resolver>(
245    listener: TcpListener,
246    resolver: R,
247    tls: Option<TlsAcceptor>,
248) -> ServerHandle {
249    let (shutdown_tx, mut shutdown) = watch::channel(false);
250    // Wakes the connections whose executor has no `write_watch` of its own.
251    let bump = WriteSignal::new();
252    let resolver = Arc::new(resolver);
253
254    let accept_task = tokio::spawn(async move {
255        loop {
256            tokio::select! {
257                accepted = listener.accept() => match accepted {
258                    Ok((stream, peer)) => {
259                        let _ = stream.set_nodelay(true);
260                        tokio::spawn(accept(
261                            stream,
262                            peer,
263                            Arc::clone(&resolver),
264                            tls.clone(),
265                            bump.clone(),
266                            shutdown.clone(),
267                        ));
268                    }
269                    // Transient errors (EMFILE, ECONNABORTED, …) must not end
270                    // the accept loop.
271                    Err(err) => {
272                        tracing::warn!(error = %err, "accept failed; retrying");
273                        tokio::time::sleep(Duration::from_millis(100)).await;
274                    }
275                },
276                _ = shutdown.changed() => break,
277            }
278        }
279    });
280
281    ServerHandle {
282        accept_task,
283        shutdown_tx,
284    }
285}
286
287/// Completes the (optional) TLS handshake, then serves the connection.
288async fn accept<R: Resolver>(
289    stream: TcpStream,
290    peer: SocketAddr,
291    resolver: Arc<R>,
292    tls: Option<TlsAcceptor>,
293    bump: WriteSignal,
294    shutdown: watch::Receiver<bool>,
295) {
296    let stream = match tls {
297        None => Either::Left(stream),
298        Some(acceptor) => {
299            match tokio::time::timeout(HANDSHAKE_TIMEOUT, acceptor.accept(stream)).await {
300                Ok(Ok(session)) => Either::Right(session),
301                Ok(Err(err)) => {
302                    tracing::debug!(%peer, ?err, "TLS handshake failed");
303                    return;
304                }
305                Err(_) => {
306                    tracing::debug!(%peer, "TLS handshake timed out");
307                    return;
308                }
309            }
310        }
311    };
312    connection(stream, peer, resolver, bump, shutdown).await;
313}
314
315type Inbound = SplitStream<Framed<ServerStream, LengthDelimitedCodec>>;
316
317/// Reads the first frame, which must be `Hello`: its correlation id and token.
318/// `None` closes the connection (timeout, garbage, or a protocol violation).
319async fn hello_phase(inbound: &mut Inbound, peer: SocketAddr) -> Option<(u64, Option<String>)> {
320    let item = match tokio::time::timeout(HELLO_TIMEOUT, inbound.next()).await {
321        Ok(Some(Ok(bytes))) => bytes,
322        Ok(_) => return None, // closed, or a transport error
323        Err(_) => {
324            tracing::debug!(%peer, "no hello within the timeout, closing connection");
325            return None;
326        }
327    };
328    let frame: ClientFrame = match crate::wire::decode_tagged(RecordKind::ClientFrame, &item) {
329        Ok(frame) => frame,
330        Err(err) => {
331            tracing::debug!(%peer, ?err, "undecodable first frame, closing connection");
332            return None;
333        }
334    };
335    match frame {
336        ClientFrame::Request {
337            id,
338            request: Request::Hello { token },
339        } => Some((id, token.map(|t| t.0))),
340        _ => {
341            tracing::debug!(%peer, "first frame is not hello, closing connection");
342            None
343        }
344    }
345}
346
347/// Resolves when the session's close signal turns `true`; never, without one
348/// (or once its sender is gone).
349async fn closed(close: &mut Option<watch::Receiver<bool>>) {
350    if let Some(rx) = close {
351        if rx.wait_for(|closed| *closed).await.is_ok() {
352            return;
353        }
354    }
355    std::future::pending::<()>().await
356}
357
358/// One established connection: a writer task draining the outbound queue, the
359/// `Hello` exchange that resolves its executor, a notify task forwarding write
360/// notifications, and the inbound read loop spawning a handler per request.
361async fn connection<R: Resolver>(
362    stream: ServerStream,
363    peer: SocketAddr,
364    resolver: Arc<R>,
365    bump: WriteSignal,
366    mut shutdown: watch::Receiver<bool>,
367) {
368    let framed = Framed::new(stream, codec());
369    let (mut sink, mut inbound) = framed.split();
370    let (out_tx, mut out_rx) = mpsc::channel::<ServerFrame>(OUT_CAPACITY);
371    let inflight = Arc::new(tokio::sync::Semaphore::new(MAX_INFLIGHT_PER_CONNECTION));
372
373    let writer = tokio::spawn(async move {
374        // Coalesce: feed this frame plus everything already queued, then flush
375        // once — one syscall per burst instead of one per frame.
376        'writer: while let Some(frame) = out_rx.recv().await {
377            let Ok(bytes) = encode_tagged(RecordKind::ServerFrame, &frame) else {
378                continue;
379            };
380            if sink.feed(Bytes::from(bytes)).await.is_err() {
381                break;
382            }
383            while let Ok(frame) = out_rx.try_recv() {
384                let Ok(bytes) = encode_tagged(RecordKind::ServerFrame, &frame) else {
385                    continue;
386                };
387                if sink.feed(Bytes::from(bytes)).await.is_err() {
388                    break 'writer;
389                }
390            }
391            if sink.flush().await.is_err() {
392                break;
393            }
394        }
395    });
396
397    // Hello: the first frame names the credential; the resolver turns it into
398    // the executor this connection is served with, or a refusal.
399    let hello = tokio::select! {
400        hello = hello_phase(&mut inbound, peer) => hello,
401        _ = shutdown.changed() => None,
402    };
403    let Some((hello_id, token)) = hello else {
404        drop(out_tx);
405        let _ = writer.await;
406        return;
407    };
408    let session = match resolver.resolve(&Hello { token, peer }).await {
409        Ok(session) => session,
410        Err(rejection) => {
411            tracing::debug!(%peer, %rejection, "connection refused");
412            let _ = out_tx
413                .send(ServerFrame::Response {
414                    id: hello_id,
415                    response: Response::Hello(Err(rejection)),
416                    stable_timestamp: None,
417                })
418                .await;
419            drop(out_tx);
420            let _ = writer.await;
421            return;
422        }
423    };
424    let Session {
425        executor,
426        tenant,
427        guard: _guard,
428        mut close,
429    } = session;
430    let hello_ok = HelloOk {
431        default_routing_key: executor.default_routing_key().map(str::to_owned),
432    };
433    let frame = ServerFrame::Response {
434        id: hello_id,
435        response: Response::Hello(Ok(hello_ok.clone())),
436        stable_timestamp: executor.stable_timestamp().await.ok().flatten(),
437    };
438    if out_tx.send(frame).await.is_err() {
439        let _ = writer.await;
440        return;
441    }
442
443    // A backend with its own write signal covers writes from every process
444    // sharing the store; the watches are fed from it. Otherwise the Write
445    // handlers fire the server-wide signal (see `handle`).
446    let bump_on_write = executor.write_watch().is_none();
447    // The watches of the connection, by the client's id: each a task that
448    // queues its id when a write it follows lands. The notifier turns what
449    // is queued into one frame.
450    let mut watches: HashMap<u64, AbortHandle> = HashMap::new();
451    let (woken_tx, mut woken_rx) = mpsc::channel::<u64>(WOKEN_CAPACITY);
452    let notifier = {
453        let out_tx = out_tx.clone();
454        let executor = executor.clone();
455        let mut shutdown = shutdown.clone();
456        tokio::spawn(async move {
457            loop {
458                tokio::select! {
459                    woken = woken_rx.recv() => {
460                        let Some(watch) = woken else { break };
461                        // One write often wakes several watches: name them
462                        // together.
463                        let mut woken = vec![watch];
464                        while let Ok(watch) = woken_rx.try_recv() {
465                            if !woken.contains(&watch) {
466                                woken.push(watch);
467                            }
468                        }
469                        let stable_timestamp =
470                            executor.stable_timestamp().await.ok().flatten();
471                        let frame = ServerFrame::Notify {
472                            watches: woken,
473                            stable_timestamp,
474                        };
475                        if out_tx.send(frame).await.is_err() {
476                            break;
477                        }
478                    }
479                    _ = shutdown.changed() => break,
480                }
481            }
482        })
483    };
484
485    loop {
486        let item = tokio::select! {
487            item = inbound.next() => item,
488            _ = closed(&mut close) => {
489                tracing::debug!(%peer, "connection closed by the server");
490                break;
491            }
492            _ = shutdown.changed() => break,
493        };
494        let Some(Ok(bytes)) = item else {
495            break; // closed, or a transport error
496        };
497        let frame: ClientFrame = match crate::wire::decode_tagged(RecordKind::ClientFrame, &bytes) {
498            Ok(frame) => frame,
499            Err(err) => {
500                // An RPC peer speaking garbage is unrecoverable; drop the link.
501                tracing::debug!(%peer, ?err, "undecodable frame, closing connection");
502                break;
503            }
504        };
505        let (id, request) = match frame {
506            ClientFrame::Request { id, request } => (id, request),
507            ClientFrame::Watch {
508                watch,
509                filters,
510                routing_key,
511                tenant: named,
512            } => {
513                let interest = WriteInterest {
514                    filters: filters.map(Arc::from),
515                    routing_key,
516                    // A session serving a tenant watches that tenant's
517                    // writes, whatever tenant the watch names (see `confine`).
518                    tenant: match &tenant {
519                        Some(tenant) => TenantFilter::Only(tenant.to_string()),
520                        None => named,
521                    },
522                };
523                // An id registered twice is the client's mistake; the later
524                // filter replaces the earlier.
525                if let Some(replaced) = watches.remove(&watch) {
526                    replaced.abort();
527                }
528                if watches.len() >= MAX_WATCHES_PER_CONNECTION {
529                    tracing::warn!(
530                        %peer,
531                        max = MAX_WATCHES_PER_CONNECTION,
532                        "too many watches on one connection: the watch is not registered \
533                         and its subscription is left to its poll interval"
534                    );
535                    continue;
536                }
537                let mut rx = executor
538                    .write_watch_for(&interest)
539                    .unwrap_or_else(|| bump.subscribe_to(interest));
540                let woken_tx = woken_tx.clone();
541                let task = tokio::spawn(async move {
542                    // Named once before any write: the watcher reads what
543                    // was written before the watch was in place.
544                    while woken_tx.send(watch).await.is_ok() {
545                        if rx.changed().await.is_err() {
546                            break;
547                        }
548                        rx.borrow_and_update();
549                    }
550                });
551                watches.insert(watch, task.abort_handle());
552                continue;
553            }
554            ClientFrame::Unwatch { watch } => {
555                if let Some(dropped) = watches.remove(&watch) {
556                    dropped.abort();
557                }
558                continue;
559            }
560        };
561
562        // A repeated hello is answered from the established session, never
563        // resolved again.
564        if let Request::Hello { .. } = request {
565            let frame = ServerFrame::Response {
566                id,
567                response: Response::Hello(Ok(hello_ok.clone())),
568                stable_timestamp: executor.stable_timestamp().await.ok().flatten(),
569            };
570            if out_tx.send(frame).await.is_err() {
571                break;
572            }
573            continue;
574        }
575
576        // Bound concurrent handlers: at the cap, stop pulling frames until one
577        // finishes (backpressure), instead of spawning without limit.
578        let Ok(permit) = Arc::clone(&inflight).acquire_owned().await else {
579            break;
580        };
581        let executor = executor.clone();
582        let tenant = tenant.clone();
583        let out_tx = out_tx.clone();
584        let bump = bump_on_write.then(|| bump.clone());
585        tokio::spawn(async move {
586            let _permit = permit;
587            let response = handle(&executor, tenant.as_deref(), request, bump.as_ref()).await;
588            let frame = ServerFrame::Response {
589                id,
590                response,
591                stable_timestamp: executor.stable_timestamp().await.ok().flatten(),
592            };
593            let _ = out_tx.send(frame).await;
594        });
595    }
596
597    notifier.abort();
598    for (_, watch) in watches {
599        watch.abort();
600    }
601    drop(out_tx);
602    let _ = writer.await;
603}
604
605fn err_string<T>(result: anyhow::Result<T>) -> Result<T, String> {
606    result.map_err(|e| format!("{e:#}"))
607}
608
609/// Why a session serving a tenant refuses `replicate`.
610const REPLICATE_REFUSED: &str = "replicate is refused on a connection serving a tenant";
611
612/// The answer a session serving a tenant gives the subscriber calls.
613fn subscribers_refused<T>() -> Result<T, String> {
614    Err(
615        "subscriptions are not served on a connection serving a tenant: keep their \
616         cursors in a store of your own (a Sidecar)"
617            .to_owned(),
618    )
619}
620
621/// Puts a request of a session serving `tenant` in that tenant, whatever
622/// tenant it names, or answers it with a refusal when it would reach past the
623/// tenant (see [`Session::tenant`]).
624fn confine(request: &mut Request, tenant: &str) -> Option<Response> {
625    match request {
626        Request::Hello { .. } => None,
627        Request::Write { events } => {
628            for event in events {
629                event.tenant = tenant.to_owned();
630            }
631            None
632        }
633        Request::Replicate { .. } => Some(Response::Write(Err(WireWriteError::Other(
634            REPLICATE_REFUSED.to_owned(),
635        )))),
636        Request::Read { tenant: named, .. } | Request::LatestTimestamp { tenant: named, .. } => {
637            *named = TenantFilter::Only(tenant.to_owned());
638            None
639        }
640        Request::LatestVersion { tenant: named, .. }
641        | Request::StreamRoutingKey { tenant: named, .. }
642        | Request::GetSnapshot { tenant: named, .. }
643        | Request::SaveSnapshot { tenant: named, .. }
644        | Request::DeleteSnapshot { tenant: named, .. } => {
645            *named = tenant.to_owned();
646            None
647        }
648        Request::GetSubscriberCursor { .. } => {
649            Some(Response::SubscriberCursor(subscribers_refused()))
650        }
651        Request::IsSubscriberRunning { .. } => {
652            Some(Response::SubscriberRunning(subscribers_refused()))
653        }
654        Request::SubscriberStatus { .. } => Some(Response::SubscriberStatus(subscribers_refused())),
655        Request::UpsertSubscriber { .. } => Some(Response::Unit(subscribers_refused())),
656        Request::Acknowledge { .. } => Some(Response::Acknowledge(subscribers_refused())),
657    }
658}
659
660/// What a client of a session serving a tenant reads: that tenant's events, in
661/// the default tenant (see [`Session::tenant`]).
662fn present(response: &mut Response) {
663    if let Response::Read(Ok(page)) = response {
664        for event in &mut page.events {
665            event.tenant.clear();
666        }
667    }
668}
669
670/// Serves one request with the session's executor, in the session's tenant
671/// when it serves one (see [`confine`] and [`present`]). A write that went
672/// through fires `bump`, the server-wide signal of the executors that have
673/// no `write_watch` of their own.
674async fn handle<E: Executor>(
675    executor: &E,
676    tenant: Option<&str>,
677    mut request: Request,
678    bump: Option<&WriteSignal>,
679) -> Response {
680    if let Some(tenant) = tenant {
681        if let Some(refusal) = confine(&mut request, tenant) {
682            return refusal;
683        }
684    }
685    // Picked once the events carry the tenant they are written in.
686    let wake = match (&request, bump) {
687        (Request::Write { events } | Request::Replicate { events }, Some(bump)) => {
688            Some(bump.prepare(events))
689        }
690        _ => None,
691    };
692    let mut response = dispatch(executor, request).await;
693    if let (Some(wake), Response::Write(Ok(()))) = (wake, &response) {
694        wake.fire();
695    }
696    if tenant.is_some() {
697        present(&mut response);
698    }
699    response
700}
701
702/// Serves one request with the session's executor, its tenants as given.
703async fn dispatch<E: Executor>(executor: &E, request: Request) -> Response {
704    match request {
705        Request::Hello { .. } => unreachable!("Hello is answered inline by the read loop"),
706        Request::Write { events } => Response::Write(
707            executor
708                .write(events)
709                .await
710                .map_err(|e| WireWriteError::from(&e)),
711        ),
712        Request::Replicate { events } => Response::Write(
713            executor
714                .replicate(events)
715                .await
716                .map_err(|e| WireWriteError::from(&e)),
717        ),
718        Request::Read {
719            aggregators,
720            routing_key,
721            tenant,
722            args,
723            to_micros,
724        } => {
725            // The bound comes from the client's cached watermark, which may
726            // have been reported by another node (a reconnect through a load
727            // balancer) or be a round trip stale. This node only vouches for
728            // its own watermark, so a bounded read never goes past it: events
729            // below the client's bound that this node has not applied yet are
730            // read on a later pass instead of being skipped.
731            let to_micros = match to_micros {
732                Some(bound) => match executor.stable_timestamp().await {
733                    Ok(Some(own)) => Some(bound.min(own)),
734                    _ => Some(bound),
735                },
736                None => None,
737            };
738            Response::Read(err_string(
739                executor
740                    .read(
741                        aggregators.map(Arc::from),
742                        routing_key,
743                        tenant,
744                        args,
745                        to_micros,
746                    )
747                    .await
748                    .map(WireReadResult::from),
749            ))
750        }
751        Request::LatestTimestamp {
752            aggregators,
753            routing_key,
754            tenant,
755        } => Response::LatestTimestamp(err_string(
756            executor
757                .latest_timestamp(aggregators.map(Arc::from), routing_key, tenant)
758                .await,
759        )),
760        Request::GetSubscriberCursor { key } => {
761            Response::SubscriberCursor(err_string(executor.get_subscriber_cursor(key).await))
762        }
763        Request::IsSubscriberRunning { key, worker_id } => Response::SubscriberRunning(err_string(
764            executor.is_subscriber_running(key, worker_id).await,
765        )),
766        Request::SubscriberStatus { key, worker_id } => {
767            Response::SubscriberStatus(err_string(executor.subscriber_status(key, worker_id).await))
768        }
769        Request::LatestVersion {
770            tenant,
771            aggregate_type,
772            aggregate_id,
773        } => Response::LatestVersion(err_string(
774            executor
775                .latest_version(tenant, aggregate_type, aggregate_id)
776                .await,
777        )),
778        Request::StreamRoutingKey {
779            tenant,
780            aggregate_type,
781            aggregate_id,
782        } => Response::StreamRoutingKey(err_string(
783            executor
784                .stream_routing_key(tenant, aggregate_type, aggregate_id)
785                .await,
786        )),
787        Request::UpsertSubscriber { key, worker_id } => {
788            Response::Unit(err_string(executor.upsert_subscriber(key, worker_id).await))
789        }
790        Request::Acknowledge {
791            key,
792            worker_id,
793            cursor,
794            lag,
795        } => Response::Acknowledge(err_string(
796            executor.acknowledge(key, worker_id, cursor, lag).await,
797        )),
798        Request::GetSnapshot {
799            tenant,
800            aggregate_type,
801            projection,
802            aggregate_revision,
803            id,
804        } => Response::Snapshot(err_string(
805            executor
806                .get_snapshot(tenant, aggregate_type, projection, aggregate_revision, id)
807                .await,
808        )),
809        Request::SaveSnapshot {
810            tenant,
811            aggregate_type,
812            projection,
813            aggregate_revision,
814            id,
815            data,
816            cursor,
817        } => Response::Unit(err_string(
818            executor
819                .save_snapshot(
820                    tenant,
821                    aggregate_type,
822                    projection,
823                    aggregate_revision,
824                    id,
825                    data,
826                    cursor,
827                )
828                .await,
829        )),
830        Request::DeleteSnapshot {
831            tenant,
832            aggregate_type,
833            projection,
834            id,
835        } => Response::Unit(err_string(
836            executor
837                .delete_snapshot(tenant, aggregate_type, projection, id)
838                .await,
839        )),
840    }
841}
842
843#[cfg(test)]
844mod tests {
845    use evento_core::cursor::{Args, Value};
846    use evento_core::{Event, TenantFilter};
847    use ulid::Ulid;
848
849    use super::{confine, present};
850    use crate::wire::{Request, Response, WireReadResult, WireWriteError};
851
852    fn acme() -> String {
853        "acme".to_owned()
854    }
855
856    #[test]
857    fn a_tenant_session_acts_in_its_tenant_whatever_a_request_names() {
858        let mut write = Request::Write {
859            events: vec![
860                Event::default(),
861                Event {
862                    tenant: "other".to_owned(),
863                    ..Default::default()
864                },
865            ],
866        };
867        assert!(confine(&mut write, "acme").is_none());
868        let Request::Write { events } = write else {
869            unreachable!()
870        };
871        assert!(events.iter().all(|e| e.tenant == "acme"));
872
873        for tenant in [TenantFilter::All, TenantFilter::Only("other".to_owned())] {
874            let mut read = Request::Read {
875                aggregators: None,
876                routing_key: None,
877                tenant: tenant.clone(),
878                args: Args::forward(1, None),
879                to_micros: None,
880            };
881            assert!(confine(&mut read, "acme").is_none());
882            assert!(
883                matches!(&read, Request::Read { tenant: TenantFilter::Only(t), .. } if *t == acme())
884            );
885            let mut latest = Request::LatestTimestamp {
886                aggregators: None,
887                routing_key: None,
888                tenant,
889            };
890            assert!(confine(&mut latest, "acme").is_none());
891            assert!(
892                matches!(&latest, Request::LatestTimestamp { tenant: TenantFilter::Only(t), .. } if *t == acme())
893            );
894        }
895
896        let other = || "other".to_owned();
897        let mut requests = [
898            Request::LatestVersion {
899                tenant: other(),
900                aggregate_type: other(),
901                aggregate_id: other(),
902            },
903            Request::StreamRoutingKey {
904                tenant: other(),
905                aggregate_type: other(),
906                aggregate_id: other(),
907            },
908            Request::GetSnapshot {
909                tenant: other(),
910                aggregate_type: other(),
911                projection: other(),
912                aggregate_revision: other(),
913                id: other(),
914            },
915            Request::SaveSnapshot {
916                tenant: other(),
917                aggregate_type: other(),
918                projection: other(),
919                aggregate_revision: other(),
920                id: other(),
921                data: vec![],
922                cursor: Value(String::new()),
923            },
924            Request::DeleteSnapshot {
925                tenant: other(),
926                aggregate_type: other(),
927                projection: other(),
928                id: other(),
929            },
930        ];
931        for request in &mut requests {
932            assert!(confine(request, "acme").is_none());
933            let (Request::LatestVersion { tenant, .. }
934            | Request::StreamRoutingKey { tenant, .. }
935            | Request::GetSnapshot { tenant, .. }
936            | Request::SaveSnapshot { tenant, .. }
937            | Request::DeleteSnapshot { tenant, .. }) = &*request
938            else {
939                unreachable!()
940            };
941            assert_eq!(*tenant, acme(), "{request:?}");
942        }
943    }
944
945    #[test]
946    fn a_tenant_session_reads_its_events_in_the_default_tenant() {
947        let event = |tenant: &str| Event {
948            tenant: tenant.to_owned(),
949            ..Default::default()
950        };
951        let mut response = Response::Read(Ok(WireReadResult {
952            cursors: vec![Value(String::new()), Value(String::new())],
953            events: vec![event("acme"), event("acme")],
954            page_info: Default::default(),
955        }));
956        present(&mut response);
957        let Response::Read(Ok(page)) = response else {
958            unreachable!()
959        };
960        assert!(page.events.iter().all(|e| e.tenant.is_empty()));
961    }
962
963    #[test]
964    fn a_tenant_session_refuses_replicate_and_the_subscriber_calls() {
965        let key = || "key".to_owned();
966        let worker_id = Ulid::nil();
967        let refusal = |mut request: Request| confine(&mut request, "acme");
968
969        assert!(matches!(
970            refusal(Request::Replicate {
971                events: vec![Event::default()]
972            }),
973            Some(Response::Write(Err(WireWriteError::Other(_))))
974        ));
975        assert!(matches!(
976            refusal(Request::GetSubscriberCursor { key: key() }),
977            Some(Response::SubscriberCursor(Err(_)))
978        ));
979        assert!(matches!(
980            refusal(Request::IsSubscriberRunning {
981                key: key(),
982                worker_id
983            }),
984            Some(Response::SubscriberRunning(Err(_)))
985        ));
986        assert!(matches!(
987            refusal(Request::SubscriberStatus {
988                key: key(),
989                worker_id
990            }),
991            Some(Response::SubscriberStatus(Err(_)))
992        ));
993        assert!(matches!(
994            refusal(Request::UpsertSubscriber {
995                key: key(),
996                worker_id
997            }),
998            Some(Response::Unit(Err(_)))
999        ));
1000        assert!(matches!(
1001            refusal(Request::Acknowledge {
1002                key: key(),
1003                worker_id,
1004                cursor: Value(String::new()),
1005                lag: 0
1006            }),
1007            Some(Response::Acknowledge(Err(_)))
1008        ));
1009    }
1010}
1011
1012#[cfg(test)]
1013mod watch_tests {
1014    use super::*;
1015    use evento_core::{Event, EventFilter, RoutingKey};
1016
1017    type Wire = Framed<TcpStream, LengthDelimitedCodec>;
1018
1019    async fn send(wire: &mut Wire, frame: ClientFrame) {
1020        let bytes = encode_tagged(RecordKind::ClientFrame, &frame).unwrap();
1021        wire.send(Bytes::from(bytes)).await.unwrap();
1022    }
1023
1024    async fn next(wire: &mut Wire) -> ServerFrame {
1025        let bytes = tokio::time::timeout(Duration::from_secs(5), wire.next())
1026            .await
1027            .expect("a frame within 5s")
1028            .expect("the connection is open")
1029            .unwrap();
1030        crate::wire::decode_tagged(RecordKind::ServerFrame, &bytes).unwrap()
1031    }
1032
1033    /// The watches of the next `Notify`; a response met on the way is skipped.
1034    async fn notified(wire: &mut Wire) -> Vec<u64> {
1035        loop {
1036            if let ServerFrame::Notify { mut watches, .. } = next(wire).await {
1037                watches.sort_unstable();
1038                return watches;
1039            }
1040        }
1041    }
1042
1043    /// A `Hello` and its reply: the frames sent before it have been taken in
1044    /// by then, the read loop handles them in order.
1045    async fn settle(wire: &mut Wire, id: u64) {
1046        send(
1047            wire,
1048            ClientFrame::Request {
1049                id,
1050                request: Request::Hello { token: None },
1051            },
1052        )
1053        .await;
1054        loop {
1055            if let ServerFrame::Response { id: answered, .. } = next(wire).await {
1056                if answered == id {
1057                    return;
1058                }
1059            }
1060        }
1061    }
1062
1063    fn watch(watch: u64, aggregate_type: &str) -> ClientFrame {
1064        ClientFrame::Watch {
1065            watch,
1066            filters: Some(vec![EventFilter::by_type_raw(aggregate_type)]),
1067            routing_key: Some(RoutingKey::Value(None)),
1068            tenant: TenantFilter::All,
1069        }
1070    }
1071
1072    fn event(aggregate_type: &str, id: &str) -> Event {
1073        Event {
1074            id: ulid::Ulid::generate(),
1075            aggregate_type: aggregate_type.to_owned(),
1076            aggregate_id: id.to_owned(),
1077            version: 1,
1078            name: "Probed".to_owned(),
1079            ..Default::default()
1080        }
1081    }
1082
1083    /// Issue #535: a `Notify` names the watches a write wakes, a connection
1084    /// that watches nothing is sent none, and a watch dropped with `Unwatch`
1085    /// is no longer named.
1086    #[tokio::test]
1087    async fn a_notify_names_the_watches_a_write_wakes() {
1088        let store = evento_fjall::Fjall::temporary().unwrap();
1089        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
1090        let addr = listener.local_addr().unwrap();
1091        let _server = serve(listener, store.clone());
1092        let mut wire = Framed::new(TcpStream::connect(addr).await.unwrap(), codec());
1093        settle(&mut wire, 0).await;
1094
1095        // Nothing is watched yet: the write is answered by nothing.
1096        store
1097            .write(vec![event("test/Account", "a0")])
1098            .await
1099            .unwrap();
1100        let quiet = tokio::time::timeout(Duration::from_millis(300), wire.next()).await;
1101        assert!(
1102            quiet.is_err(),
1103            "a connection that watches nothing is sent nothing"
1104        );
1105
1106        // A watch is named once as it is registered.
1107        send(&mut wire, watch(1, "test/Account")).await;
1108        assert_eq!(notified(&mut wire).await, [1]);
1109        send(&mut wire, watch(2, "test/Account")).await;
1110        assert_eq!(notified(&mut wire).await, [2]);
1111        send(&mut wire, watch(3, "test/Card")).await;
1112        assert_eq!(notified(&mut wire).await, [3]);
1113
1114        store
1115            .write(vec![event("test/Account", "a1")])
1116            .await
1117            .unwrap();
1118        let mut woken = notified(&mut wire).await;
1119        if woken.len() < 2 {
1120            // The two watches were queued either side of the notifier's wake.
1121            woken.extend(notified(&mut wire).await);
1122            woken.sort_unstable();
1123        }
1124        assert_eq!(woken, [1, 2]);
1125
1126        send(&mut wire, ClientFrame::Unwatch { watch: 2 }).await;
1127        settle(&mut wire, 1).await;
1128        store
1129            .write(vec![event("test/Account", "a2")])
1130            .await
1131            .unwrap();
1132        assert_eq!(notified(&mut wire).await, [1]);
1133        store.write(vec![event("test/Card", "c1")]).await.unwrap();
1134        assert_eq!(notified(&mut wire).await, [3]);
1135    }
1136}