1use std::collections::HashMap;
35use std::fmt;
36use std::net::SocketAddr;
37use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
38use std::sync::Arc;
39use std::time::Duration;
40
41use anyhow::Context as _;
42use bytes::Bytes;
43use evento_core::cursor::{Args, ReadResult, Value};
44use evento_core::{
45 Event, EventFilter, Executor, RoutingKey, TenantFilter, WriteError, WriteInterest,
46};
47use futures_util::{SinkExt, StreamExt};
48use tokio::net::TcpStream;
49use tokio::sync::{mpsc, oneshot, watch};
50use tokio_rustls::rustls::pki_types::ServerName;
51use tokio_rustls::TlsConnector;
52use tokio_util::codec::{Framed, LengthDelimitedCodec};
53use tokio_util::either::Either;
54use ulid::Ulid;
55
56use crate::wire::{
57 decode_tagged, encode_tagged, ClientFrame, HelloOk, RecordKind, Rejection, Request, Response,
58 ServerFrame, Token, MAX_FRAME_LENGTH,
59};
60
61type ClientStream = Either<TcpStream, tokio_rustls::client::TlsStream<TcpStream>>;
63
64#[derive(Clone)]
67struct Tls {
68 connector: TlsConnector,
69 server_name: ServerName<'static>,
70}
71
72#[derive(Clone, Debug, PartialEq, Eq)]
83pub struct Address(String);
84
85impl From<SocketAddr> for Address {
86 fn from(addr: SocketAddr) -> Self {
87 Address(addr.to_string())
88 }
89}
90
91impl From<&str> for Address {
92 fn from(addr: &str) -> Self {
93 Address(addr.to_owned())
94 }
95}
96
97impl From<String> for Address {
98 fn from(addr: String) -> Self {
99 Address(addr)
100 }
101}
102
103impl fmt::Display for Address {
104 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
105 f.write_str(&self.0)
106 }
107}
108
109#[derive(Clone)]
112struct Dial {
113 addr: Address,
114 token: Option<Token>,
115 tls: Option<Tls>,
116}
117
118const DEFAULT_REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
119const RECONNECT_MIN: Duration = Duration::from_millis(100);
120const RECONNECT_MAX: Duration = Duration::from_secs(5);
121
122#[derive(Clone, Copy)]
127struct Backoff {
128 min: Duration,
129 max: Duration,
130}
131
132impl Backoff {
133 fn next_delay(&self, bound: &mut Duration) -> Duration {
136 let delay = bound.mul_f64(rand::random_range(0.0..=1.0));
137 *bound = bound.saturating_mul(2).min(self.max);
138 delay
139 }
140}
141const HELLO_ID: u64 = 0;
143const REQ_CAPACITY: usize = 1024;
145const OUT_CAPACITY: usize = 1024;
147
148type Stable = Arc<AtomicU64>;
153
154fn codec() -> LengthDelimitedCodec {
155 LengthDelimitedCodec::builder()
156 .max_frame_length(MAX_FRAME_LENGTH)
157 .new_codec()
158}
159
160type Dispatch = (u64, Request, oneshot::Sender<Response>);
163
164enum WatchCommand {
166 Add {
169 watch: u64,
170 interest: WriteInterest,
171 tx: watch::Sender<u64>,
172 },
173 Remove { watch: u64 },
175}
176
177fn watch_frame(watch: u64, interest: &WriteInterest) -> ClientFrame {
179 ClientFrame::Watch {
180 watch,
181 filters: interest.filters.as_deref().map(<[EventFilter]>::to_vec),
182 routing_key: interest.routing_key.clone(),
183 tenant: interest.tenant.clone(),
184 }
185}
186
187#[derive(Clone)]
190pub struct Client {
191 inner: Arc<Inner>,
192}
193
194struct Inner {
195 lanes: Vec<mpsc::Sender<Dispatch>>,
197 round_robin: AtomicUsize,
198 next_id: AtomicU64,
199 watches: mpsc::UnboundedSender<WatchCommand>,
201 next_watch: AtomicU64,
202 watch_all: std::sync::OnceLock<watch::Sender<u64>>,
205 runtime: tokio::runtime::Handle,
207 default_routing_key: Option<String>,
210 stable: Stable,
211 request_timeout: Duration,
212}
213
214pub struct ClientBuilder {
216 dial: Dial,
217 request_timeout: Duration,
218 connections: usize,
219 backoff: Backoff,
220}
221
222impl ClientBuilder {
223 pub fn token(mut self, token: impl Into<String>) -> Self {
227 self.dial.token = Some(Token(token.into()));
228 self
229 }
230
231 pub fn tls(mut self, connector: TlsConnector, server_name: ServerName<'static>) -> Self {
235 self.dial.tls = Some(Tls {
236 connector,
237 server_name,
238 });
239 self
240 }
241
242 pub fn request_timeout(mut self, timeout: Duration) -> Self {
246 self.request_timeout = timeout;
247 self
248 }
249
250 pub fn connections(mut self, n: usize) -> Self {
254 self.connections = n.max(1);
255 self
256 }
257
258 pub fn reconnect_backoff(mut self, min: Duration, max: Duration) -> Self {
266 self.backoff = Backoff {
267 min,
268 max: max.max(min),
269 };
270 self
271 }
272
273 pub async fn connect(self) -> anyhow::Result<Client> {
278 let stable: Stable = Arc::new(AtomicU64::new(0));
279 let (watches, watch_rx) = mpsc::unbounded_channel();
280 let mut watch_rx = Some(watch_rx);
281
282 let mut lanes = Vec::with_capacity(self.connections);
283 let mut default_routing_key = None;
284 for lane in 0..self.connections {
285 let (framed, hello) =
286 connect_and_hello(&self.dial, &stable, self.request_timeout).await?;
287 if lane == 0 {
288 default_routing_key = hello.default_routing_key;
289 }
290 let (req_tx, req_rx) = mpsc::channel(REQ_CAPACITY);
291 tokio::spawn(
292 Actor {
293 dial: self.dial.clone(),
294 req_rx,
295 watch_rx: if lane == 0 { watch_rx.take() } else { None },
299 watches: HashMap::new(),
300 stable: Arc::clone(&stable),
301 request_timeout: self.request_timeout,
302 backoff: self.backoff,
303 }
304 .run(framed),
305 );
306 lanes.push(req_tx);
307 }
308
309 Ok(Client {
310 inner: Arc::new(Inner {
311 lanes,
312 round_robin: AtomicUsize::new(0),
313 next_id: AtomicU64::new(HELLO_ID + 1),
314 watches,
315 next_watch: AtomicU64::new(0),
316 watch_all: std::sync::OnceLock::new(),
317 runtime: tokio::runtime::Handle::current(),
318 default_routing_key,
319 stable,
320 request_timeout: self.request_timeout,
321 }),
322 })
323 }
324}
325
326impl Client {
327 pub async fn connect(addr: impl Into<Address>) -> anyhow::Result<Client> {
329 Client::builder(addr).connect().await
330 }
331
332 pub fn builder(addr: impl Into<Address>) -> ClientBuilder {
336 ClientBuilder {
337 dial: Dial {
338 addr: addr.into(),
339 token: None,
340 tls: None,
341 },
342 request_timeout: DEFAULT_REQUEST_TIMEOUT,
343 connections: 1,
344 backoff: Backoff {
345 min: RECONNECT_MIN,
346 max: RECONNECT_MAX,
347 },
348 }
349 }
350
351 async fn request(&self, request: Request) -> anyhow::Result<Response> {
352 let id = self.inner.next_id.fetch_add(1, Ordering::Relaxed);
355 let lane = self.inner.round_robin.fetch_add(1, Ordering::Relaxed) % self.inner.lanes.len();
356 let (tx, rx) = oneshot::channel();
357 if self.inner.lanes[lane]
358 .send((id, request, tx))
359 .await
360 .is_err()
361 {
362 anyhow::bail!("remote executor connection is closed");
363 }
364 match tokio::time::timeout(self.inner.request_timeout, rx).await {
365 Ok(Ok(response)) => Ok(response),
366 Ok(Err(_)) => anyhow::bail!("remote executor connection lost"),
367 Err(_) => anyhow::bail!(
368 "remote request timed out after {:?}",
369 self.inner.request_timeout
370 ),
371 }
372 }
373}
374
375impl Inner {
376 fn watch(&self, interest: WriteInterest) -> (u64, watch::Sender<u64>) {
379 let (tx, _) = watch::channel(0u64);
380 let watch = self.next_watch.fetch_add(1, Ordering::Relaxed);
381 let _ = self.watches.send(WatchCommand::Add {
384 watch,
385 interest,
386 tx: tx.clone(),
387 });
388 (watch, tx)
389 }
390}
391
392fn protocol_err(what: &str) -> anyhow::Error {
393 anyhow::anyhow!("remote protocol error: unexpected response to {what}")
394}
395
396#[async_trait::async_trait]
397impl Executor for Client {
398 fn default_routing_key(&self) -> Option<&str> {
399 self.inner.default_routing_key.as_deref()
400 }
401
402 fn write_watch(&self) -> Option<watch::Receiver<u64>> {
403 let all = self.inner.watch_all.get_or_init(|| {
404 self.inner
405 .watch(WriteInterest {
406 filters: None,
407 routing_key: None,
408 tenant: TenantFilter::All,
409 })
410 .1
411 });
412 Some(all.subscribe())
413 }
414
415 fn write_watch_for(&self, interest: &WriteInterest) -> Option<watch::Receiver<u64>> {
416 let (watch, tx) = self.inner.watch(interest.clone());
417 let rx = tx.subscribe();
418 let watches = self.inner.watches.clone();
421 self.inner.runtime.spawn(async move {
422 tx.closed().await;
423 let _ = watches.send(WatchCommand::Remove { watch });
424 });
425 Some(rx)
426 }
427
428 async fn stable_timestamp(&self) -> anyhow::Result<Option<u64>> {
429 match self.inner.stable.load(Ordering::Acquire) {
433 0 => Ok(None),
434 v => Ok(Some(v)),
435 }
436 }
437
438 async fn write(&self, events: Vec<Event>) -> Result<(), WriteError> {
439 match self.request(Request::Write { events }).await {
440 Ok(Response::Write(Ok(()))) => Ok(()),
441 Ok(Response::Write(Err(e))) => Err(e.into()),
442 Ok(_) => Err(WriteError::Unknown(protocol_err("write"))),
443 Err(e) => Err(WriteError::Unknown(e)),
444 }
445 }
446
447 async fn replicate(&self, events: Vec<Event>) -> Result<(), WriteError> {
448 match self.request(Request::Replicate { events }).await {
449 Ok(Response::Write(Ok(()))) => Ok(()),
450 Ok(Response::Write(Err(e))) => Err(e.into()),
451 Ok(_) => Err(WriteError::Unknown(protocol_err("replicate"))),
452 Err(e) => Err(WriteError::Unknown(e)),
453 }
454 }
455
456 async fn read(
457 &self,
458 aggregators: Option<Arc<[EventFilter]>>,
459 routing_key: Option<RoutingKey>,
460 tenant: TenantFilter,
461 args: Args,
462 to_micros: Option<u64>,
463 ) -> anyhow::Result<ReadResult<Event>> {
464 match self
465 .request(Request::Read {
466 aggregators: aggregators.map(|a| a.to_vec()),
467 routing_key,
468 tenant,
469 args,
470 to_micros,
471 })
472 .await?
473 {
474 Response::Read(Ok(wire)) => Ok(wire.into()),
475 Response::Read(Err(msg)) => Err(anyhow::anyhow!(msg)),
476 _ => Err(protocol_err("read")),
477 }
478 }
479
480 async fn latest_timestamp(
481 &self,
482 aggregators: Option<Arc<[EventFilter]>>,
483 routing_key: Option<RoutingKey>,
484 tenant: TenantFilter,
485 ) -> anyhow::Result<u64> {
486 match self
487 .request(Request::LatestTimestamp {
488 aggregators: aggregators.map(|a| a.to_vec()),
489 routing_key,
490 tenant,
491 })
492 .await?
493 {
494 Response::LatestTimestamp(Ok(v)) => Ok(v),
495 Response::LatestTimestamp(Err(msg)) => Err(anyhow::anyhow!(msg)),
496 _ => Err(protocol_err("latest_timestamp")),
497 }
498 }
499
500 async fn get_subscriber_cursor(&self, key: String) -> anyhow::Result<Option<Value>> {
501 match self.request(Request::GetSubscriberCursor { key }).await? {
502 Response::SubscriberCursor(Ok(v)) => Ok(v),
503 Response::SubscriberCursor(Err(msg)) => Err(anyhow::anyhow!(msg)),
504 _ => Err(protocol_err("get_subscriber_cursor")),
505 }
506 }
507
508 async fn is_subscriber_running(&self, key: String, worker_id: Ulid) -> anyhow::Result<bool> {
509 match self
510 .request(Request::IsSubscriberRunning { key, worker_id })
511 .await?
512 {
513 Response::SubscriberRunning(Ok(v)) => Ok(v),
514 Response::SubscriberRunning(Err(msg)) => Err(anyhow::anyhow!(msg)),
515 _ => Err(protocol_err("is_subscriber_running")),
516 }
517 }
518
519 async fn subscriber_status(
520 &self,
521 key: String,
522 worker_id: Ulid,
523 ) -> anyhow::Result<evento_core::SubscriberStatus> {
524 match self
525 .request(Request::SubscriberStatus { key, worker_id })
526 .await?
527 {
528 Response::SubscriberStatus(Ok(v)) => Ok(v),
529 Response::SubscriberStatus(Err(msg)) => Err(anyhow::anyhow!(msg)),
530 _ => Err(protocol_err("subscriber_status")),
531 }
532 }
533
534 async fn latest_version(
535 &self,
536 tenant: String,
537 aggregate_type: String,
538 aggregate_id: String,
539 ) -> anyhow::Result<u16> {
540 match self
541 .request(Request::LatestVersion {
542 tenant,
543 aggregate_type,
544 aggregate_id,
545 })
546 .await?
547 {
548 Response::LatestVersion(Ok(v)) => Ok(v),
549 Response::LatestVersion(Err(msg)) => Err(anyhow::anyhow!(msg)),
550 _ => Err(protocol_err("latest_version")),
551 }
552 }
553
554 async fn stream_routing_key(
555 &self,
556 tenant: String,
557 aggregate_type: String,
558 aggregate_id: String,
559 ) -> anyhow::Result<Option<Option<String>>> {
560 match self
561 .request(Request::StreamRoutingKey {
562 tenant,
563 aggregate_type,
564 aggregate_id,
565 })
566 .await?
567 {
568 Response::StreamRoutingKey(Ok(v)) => Ok(v),
569 Response::StreamRoutingKey(Err(msg)) => Err(anyhow::anyhow!(msg)),
570 _ => Err(protocol_err("stream_routing_key")),
571 }
572 }
573
574 async fn upsert_subscriber(&self, key: String, worker_id: Ulid) -> anyhow::Result<()> {
575 match self
576 .request(Request::UpsertSubscriber { key, worker_id })
577 .await?
578 {
579 Response::Unit(Ok(())) => Ok(()),
580 Response::Unit(Err(msg)) => Err(anyhow::anyhow!(msg)),
581 _ => Err(protocol_err("upsert_subscriber")),
582 }
583 }
584
585 async fn acknowledge(
586 &self,
587 key: String,
588 worker_id: Ulid,
589 cursor: Value,
590 lag: u64,
591 ) -> anyhow::Result<bool> {
592 match self
593 .request(Request::Acknowledge {
594 key,
595 worker_id,
596 cursor,
597 lag,
598 })
599 .await?
600 {
601 Response::Acknowledge(Ok(v)) => Ok(v),
602 Response::Acknowledge(Err(msg)) => Err(anyhow::anyhow!(msg)),
603 _ => Err(protocol_err("acknowledge")),
604 }
605 }
606
607 async fn get_snapshot(
608 &self,
609 tenant: String,
610 aggregate_type: String,
611 projection: String,
612 aggregate_revision: String,
613 id: String,
614 ) -> anyhow::Result<Option<(Vec<u8>, Value)>> {
615 match self
616 .request(Request::GetSnapshot {
617 tenant,
618 aggregate_type,
619 projection,
620 aggregate_revision,
621 id,
622 })
623 .await?
624 {
625 Response::Snapshot(Ok(v)) => Ok(v),
626 Response::Snapshot(Err(msg)) => Err(anyhow::anyhow!(msg)),
627 _ => Err(protocol_err("get_snapshot")),
628 }
629 }
630
631 async fn save_snapshot(
632 &self,
633 tenant: String,
634 aggregate_type: String,
635 projection: String,
636 aggregate_revision: String,
637 id: String,
638 data: Vec<u8>,
639 cursor: Value,
640 ) -> anyhow::Result<()> {
641 match self
642 .request(Request::SaveSnapshot {
643 tenant,
644 aggregate_type,
645 projection,
646 aggregate_revision,
647 id,
648 data,
649 cursor,
650 })
651 .await?
652 {
653 Response::Unit(Ok(())) => Ok(()),
654 Response::Unit(Err(msg)) => Err(anyhow::anyhow!(msg)),
655 _ => Err(protocol_err("save_snapshot")),
656 }
657 }
658
659 async fn delete_snapshot(
660 &self,
661 tenant: String,
662 aggregate_type: String,
663 projection: String,
664 id: String,
665 ) -> anyhow::Result<()> {
666 match self
667 .request(Request::DeleteSnapshot {
668 tenant,
669 aggregate_type,
670 projection,
671 id,
672 })
673 .await?
674 {
675 Response::Unit(Ok(())) => Ok(()),
676 Response::Unit(Err(msg)) => Err(anyhow::anyhow!(msg)),
677 _ => Err(protocol_err("delete_snapshot")),
678 }
679 }
680}
681
682struct Actor {
689 dial: Dial,
690 req_rx: mpsc::Receiver<Dispatch>,
691 watch_rx: Option<mpsc::UnboundedReceiver<WatchCommand>>,
693 watches: HashMap<u64, (WriteInterest, watch::Sender<u64>)>,
696 stable: Stable,
697 request_timeout: Duration,
698 backoff: Backoff,
699}
700
701impl Actor {
702 async fn run(mut self, initial: Framed<ClientStream, LengthDelimitedCodec>) {
703 let mut conn = Some(initial);
704 let mut pending: HashMap<u64, oneshot::Sender<Response>> = HashMap::new();
711 loop {
712 let framed = match conn.take() {
713 Some(framed) => framed,
714 None => match self.reconnect().await {
715 Some(framed) => framed,
716 None => return, },
718 };
719 let (mut sink, mut inbound) = framed.split();
720 let (out_tx, mut out_rx) = mpsc::channel::<Bytes>(OUT_CAPACITY);
725 let writer = tokio::spawn(async move {
726 'writer: while let Some(bytes) = out_rx.recv().await {
730 if sink.feed(bytes).await.is_err() {
731 break;
732 }
733 while let Ok(bytes) = out_rx.try_recv() {
734 if sink.feed(bytes).await.is_err() {
735 break 'writer;
736 }
737 }
738 if sink.flush().await.is_err() {
739 break;
740 }
741 }
742 });
743 'connected: {
748 for (watch, (interest, _)) in &self.watches {
749 let Ok(bytes) =
750 encode_tagged(RecordKind::ClientFrame, &watch_frame(*watch, interest))
751 else {
752 continue;
753 };
754 if out_tx.send(Bytes::from(bytes)).await.is_err() {
755 break 'connected;
756 }
757 }
758 loop {
759 tokio::select! {
760 command = next_watch_command(&mut self.watch_rx) => {
761 let frame = match command {
762 WatchCommand::Add { watch, interest, tx } => {
763 let frame = watch_frame(watch, &interest);
764 self.watches.insert(watch, (interest, tx));
765 frame
766 }
767 WatchCommand::Remove { watch } => {
768 self.watches.remove(&watch);
769 ClientFrame::Unwatch { watch }
770 }
771 };
772 let Ok(bytes) = encode_tagged(RecordKind::ClientFrame, &frame) else {
773 continue;
774 };
775 if out_tx.send(Bytes::from(bytes)).await.is_err() {
776 break;
777 }
778 }
779 item = self.req_rx.recv() => {
780 let Some((id, request, reply_tx)) = item else { break };
781 let frame = ClientFrame::Request { id, request };
782 let Ok(bytes) = encode_tagged(RecordKind::ClientFrame, &frame) else {
783 continue; };
785 pending.insert(id, reply_tx);
786 if out_tx.send(Bytes::from(bytes)).await.is_err() {
787 pending.remove(&id);
788 break;
789 }
790 }
791 item = inbound.next() => {
792 let Some(Ok(bytes)) = item else { break };
793 match decode_tagged::<ServerFrame>(RecordKind::ServerFrame, &bytes) {
794 Ok(ServerFrame::Response { id, response, stable_timestamp }) => {
795 self.update_stable(stable_timestamp);
796 if let Some(tx) = pending.remove(&id) {
800 let _ = tx.send(response);
801 }
802 }
803 Ok(ServerFrame::Notify { watches, stable_timestamp }) => {
804 self.update_stable(stable_timestamp);
805 for watch in watches {
806 if let Some((_, tx)) = self.watches.get(&watch) {
809 tx.send_modify(|g| *g += 1);
810 }
811 }
812 }
813 Err(err) => {
814 tracing::debug!(?err, "undecodable frame, reconnecting");
815 break;
816 }
817 }
818 }
819 }
820 }
821 }
822 drop(out_tx);
826 let _ = writer.await;
827 pending.clear();
828 if self.req_rx.is_closed() {
829 return;
830 }
831 }
832 }
833
834 async fn reconnect(&mut self) -> Option<Framed<ClientStream, LengthDelimitedCodec>> {
837 let mut bound = self.backoff.min;
838 loop {
839 tokio::time::sleep(self.backoff.next_delay(&mut bound)).await;
843 if self.req_rx.is_closed() {
844 return None;
845 }
846 match connect_and_hello(&self.dial, &self.stable, self.request_timeout).await {
847 Ok((framed, _)) => return Some(framed),
851 Err(err) => {
852 if let Some(rejection) = err.downcast_ref::<Rejection>() {
857 tracing::warn!(%rejection, addr = %self.dial.addr, "the server refused the reconnect, backing off");
858 } else {
859 tracing::debug!(?err, addr = %self.dial.addr, "reconnect failed, backing off");
860 }
861 }
862 }
863 }
864 }
865
866 fn update_stable(&self, stable_timestamp: Option<u64>) {
867 if let Some(v) = stable_timestamp {
868 self.stable.fetch_max(v, Ordering::AcqRel);
869 }
870 }
871}
872
873async fn next_watch_command(
877 rx: &mut Option<mpsc::UnboundedReceiver<WatchCommand>>,
878) -> WatchCommand {
879 if let Some(commands) = rx {
880 if let Some(command) = commands.recv().await {
881 return command;
882 }
883 *rx = None;
884 }
885 std::future::pending().await
886}
887
888async fn connect_and_hello(
895 dial: &Dial,
896 stable: &AtomicU64,
897 timeout: Duration,
898) -> anyhow::Result<(Framed<ClientStream, LengthDelimitedCodec>, HelloOk)> {
899 let stream = TcpStream::connect(dial.addr.0.as_str())
902 .await
903 .with_context(|| format!("cannot connect to {}", dial.addr))?;
904 let _ = stream.set_nodelay(true);
905 let stream = match &dial.tls {
906 None => Either::Left(stream),
907 Some(tls) => Either::Right(
908 tokio::time::timeout(
909 timeout,
910 tls.connector.connect(tls.server_name.clone(), stream),
911 )
912 .await
913 .map_err(|_| anyhow::anyhow!("TLS handshake timed out after {timeout:?}"))??,
914 ),
915 };
916 let mut framed = Framed::new(stream, codec());
917
918 let hello = ClientFrame::Request {
919 id: HELLO_ID,
920 request: Request::Hello {
921 token: dial.token.clone(),
922 },
923 };
924 framed
925 .send(Bytes::from(encode_tagged(RecordKind::ClientFrame, &hello)?))
926 .await?;
927
928 let hello = tokio::time::timeout(timeout, async {
929 loop {
930 let Some(item) = framed.next().await else {
931 anyhow::bail!("connection closed during hello");
932 };
933 match decode_tagged::<ServerFrame>(RecordKind::ServerFrame, &item?)? {
934 ServerFrame::Response {
935 id: HELLO_ID,
936 response: Response::Hello(reply),
937 stable_timestamp,
938 } => {
939 if let Some(v) = stable_timestamp {
940 stable.fetch_max(v, Ordering::AcqRel);
941 }
942 let ok = reply
943 .map_err(anyhow::Error::new)
944 .context("the server refused the connection")?;
945 return Ok(ok);
946 }
947 ServerFrame::Notify {
948 stable_timestamp, ..
949 } => {
950 if let Some(v) = stable_timestamp {
951 stable.fetch_max(v, Ordering::AcqRel);
952 }
953 }
954 _ => anyhow::bail!("unexpected frame during hello"),
955 }
956 }
957 })
958 .await
959 .map_err(|_| anyhow::anyhow!("hello timed out after {timeout:?}"))??;
960
961 Ok((framed, hello))
962}
963
964#[cfg(test)]
965mod tests {
966 use super::*;
967
968 #[tokio::test]
972 async fn a_watch_is_registered_and_dropped_with_its_receivers() {
973 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
974 let addr = listener.local_addr().unwrap();
975 let server = tokio::spawn(async move {
976 let (stream, _) = listener.accept().await.unwrap();
977 let mut wire = Framed::new(stream, codec());
978 let mut seen = Vec::new();
979 while seen.len() < 4 {
980 let bytes = wire.next().await.unwrap().unwrap();
981 match decode_tagged::<ClientFrame>(RecordKind::ClientFrame, &bytes).unwrap() {
982 ClientFrame::Request { id, .. } => {
983 let reply = ServerFrame::Response {
984 id,
985 response: Response::Hello(Ok(HelloOk {
986 default_routing_key: None,
987 })),
988 stable_timestamp: None,
989 };
990 let bytes = encode_tagged(RecordKind::ServerFrame, &reply).unwrap();
991 wire.send(Bytes::from(bytes)).await.unwrap();
992 }
993 frame => seen.push(frame),
994 }
995 }
996 seen
997 });
998
999 let client = Client::connect(addr).await.unwrap();
1000 let interest = WriteInterest {
1001 filters: Some([EventFilter::by_type_raw("test/Account")].into()),
1002 routing_key: Some(RoutingKey::Value(None)),
1003 tenant: TenantFilter::All,
1004 };
1005 let accounts = client.write_watch_for(&interest).unwrap();
1006 let clone = accounts.clone();
1007 let all = client.write_watch().unwrap();
1008 let all_again = client.write_watch().unwrap();
1009 drop(accounts);
1010 let cards = client
1011 .write_watch_for(&WriteInterest {
1012 filters: Some([EventFilter::by_type_raw("test/Card")].into()),
1013 ..interest
1014 })
1015 .unwrap();
1016 drop(clone);
1017
1018 let seen = tokio::time::timeout(Duration::from_secs(5), server)
1019 .await
1020 .expect("four watch frames within 5s")
1021 .unwrap();
1022 let [ClientFrame::Watch {
1023 watch: accounts,
1024 filters: Some(filters),
1025 routing_key: Some(RoutingKey::Value(None)),
1026 tenant: TenantFilter::All,
1027 }, ClientFrame::Watch {
1028 watch: unfiltered,
1029 filters: None,
1030 routing_key: None,
1031 tenant: TenantFilter::All,
1032 }, ClientFrame::Watch {
1033 watch: cards_id, ..
1034 }, ClientFrame::Unwatch { watch: dropped }] = &seen[..]
1035 else {
1036 panic!("unexpected frames: {seen:?}");
1037 };
1038 assert_eq!(filters, &[EventFilter::by_type_raw("test/Account")]);
1039 assert_eq!(dropped, accounts);
1040 assert!(accounts != unfiltered && unfiltered != cards_id && accounts != cards_id);
1041 drop((all, all_again, cards));
1042 }
1043
1044 #[test]
1045 fn the_reconnect_delay_stays_below_a_doubling_bound() {
1046 let backoff = Backoff {
1047 min: Duration::from_millis(100),
1048 max: Duration::from_secs(5),
1049 };
1050 let mut bound = backoff.min;
1051 let mut bounds = Vec::new();
1052 for _ in 0..8 {
1053 let before = bound;
1054 assert!(backoff.next_delay(&mut bound) <= before);
1055 bounds.push(before.as_millis());
1056 }
1057 assert_eq!(bounds, [100, 200, 400, 800, 1600, 3200, 5000, 5000]);
1058 }
1059
1060 #[test]
1061 fn two_clients_do_not_share_a_reconnect_schedule() {
1062 let backoff = Backoff {
1063 min: Duration::from_secs(5),
1064 max: Duration::from_secs(5),
1065 };
1066 let schedule = || {
1067 let mut bound = backoff.min;
1068 (0..16)
1069 .map(|_| backoff.next_delay(&mut bound))
1070 .collect::<Vec<_>>()
1071 };
1072 assert_ne!(schedule(), schedule());
1073 }
1074
1075 #[test]
1076 fn a_reconnect_maximum_below_the_minimum_is_raised_to_it() {
1077 let builder = Client::builder("127.0.0.1:1")
1078 .reconnect_backoff(Duration::from_secs(2), Duration::from_secs(1));
1079 assert_eq!(builder.backoff.min, Duration::from_secs(2));
1080 assert_eq!(builder.backoff.max, Duration::from_secs(2));
1081 }
1082}