use std::any::Any;
use std::collections::HashMap;
use std::net::SocketAddr;
use std::sync::Arc;
use std::time::Duration;
use bytes::Bytes;
use evento_core::{Executor, TenantError, TenantFilter, WriteInterest, WriteSignal};
use futures_util::stream::SplitStream;
use futures_util::{SinkExt, StreamExt};
use tokio::net::{TcpListener, TcpStream};
use tokio::sync::{mpsc, watch};
use tokio::task::{AbortHandle, JoinHandle};
use tokio_rustls::TlsAcceptor;
use tokio_util::codec::{Framed, LengthDelimitedCodec};
use tokio_util::either::Either;
use crate::wire::{
encode_tagged, ClientFrame, HelloOk, RecordKind, Rejection, Request, Response, ServerFrame,
WireReadResult, WireWriteError, MAX_FRAME_LENGTH,
};
const OUT_CAPACITY: usize = 1024;
const MAX_INFLIGHT_PER_CONNECTION: usize = 256;
const MAX_WATCHES_PER_CONNECTION: usize = 65_536;
const WOKEN_CAPACITY: usize = 1024;
const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);
const HELLO_TIMEOUT: Duration = Duration::from_secs(10);
type ServerStream = Either<TcpStream, tokio_rustls::server::TlsStream<TcpStream>>;
fn codec() -> LengthDelimitedCodec {
LengthDelimitedCodec::builder()
.max_frame_length(MAX_FRAME_LENGTH)
.new_codec()
}
#[derive(Debug, Clone)]
pub struct Hello {
pub token: Option<String>,
pub peer: SocketAddr,
}
pub struct Session<E> {
executor: E,
tenant: Option<Arc<str>>,
guard: Option<Box<dyn Any + Send + Sync>>,
close: Option<watch::Receiver<bool>>,
}
impl<E> Session<E> {
pub fn new(executor: E) -> Self {
Self {
executor,
tenant: None,
guard: None,
close: None,
}
}
pub fn tenant(mut self, tenant: impl Into<String>) -> Result<Self, TenantError> {
let tenant = tenant.into();
TenantError::check(&tenant)?;
self.tenant = Some(tenant.into());
Ok(self)
}
pub fn keep(mut self, guard: impl Any + Send + Sync) -> Self {
self.guard = Some(Box::new(guard));
self
}
pub fn close_when(mut self, signal: watch::Receiver<bool>) -> Self {
self.close = Some(signal);
self
}
}
#[async_trait::async_trait]
pub trait Resolver: Send + Sync + 'static {
type Executor: Executor + Clone;
async fn resolve(&self, hello: &Hello) -> Result<Session<Self::Executor>, Rejection>;
}
struct Static<E>(E);
#[async_trait::async_trait]
impl<E: Executor + Clone> Resolver for Static<E> {
type Executor = E;
async fn resolve(&self, _hello: &Hello) -> Result<Session<E>, Rejection> {
Ok(Session::new(self.0.clone()))
}
}
pub struct ServerHandle {
accept_task: JoinHandle<()>,
shutdown_tx: watch::Sender<bool>,
}
impl ServerHandle {
pub async fn shutdown(self) {
let _ = self.shutdown_tx.send(true);
let _ = self.accept_task.await;
}
}
pub fn serve<E: Executor + Clone>(listener: TcpListener, executor: E) -> ServerHandle {
serve_with(listener, Static(executor))
}
pub fn serve_with<R: Resolver>(listener: TcpListener, resolver: R) -> ServerHandle {
start(listener, resolver, None)
}
pub fn serve_tls<E: Executor + Clone>(
listener: TcpListener,
acceptor: TlsAcceptor,
executor: E,
) -> ServerHandle {
serve_tls_with(listener, acceptor, Static(executor))
}
pub fn serve_tls_with<R: Resolver>(
listener: TcpListener,
acceptor: TlsAcceptor,
resolver: R,
) -> ServerHandle {
start(listener, resolver, Some(acceptor))
}
fn start<R: Resolver>(
listener: TcpListener,
resolver: R,
tls: Option<TlsAcceptor>,
) -> ServerHandle {
let (shutdown_tx, mut shutdown) = watch::channel(false);
let bump = WriteSignal::new();
let resolver = Arc::new(resolver);
let accept_task = tokio::spawn(async move {
loop {
tokio::select! {
accepted = listener.accept() => match accepted {
Ok((stream, peer)) => {
let _ = stream.set_nodelay(true);
tokio::spawn(accept(
stream,
peer,
Arc::clone(&resolver),
tls.clone(),
bump.clone(),
shutdown.clone(),
));
}
Err(err) => {
tracing::warn!(error = %err, "accept failed; retrying");
tokio::time::sleep(Duration::from_millis(100)).await;
}
},
_ = shutdown.changed() => break,
}
}
});
ServerHandle {
accept_task,
shutdown_tx,
}
}
async fn accept<R: Resolver>(
stream: TcpStream,
peer: SocketAddr,
resolver: Arc<R>,
tls: Option<TlsAcceptor>,
bump: WriteSignal,
shutdown: watch::Receiver<bool>,
) {
let stream = match tls {
None => Either::Left(stream),
Some(acceptor) => {
match tokio::time::timeout(HANDSHAKE_TIMEOUT, acceptor.accept(stream)).await {
Ok(Ok(session)) => Either::Right(session),
Ok(Err(err)) => {
tracing::debug!(%peer, ?err, "TLS handshake failed");
return;
}
Err(_) => {
tracing::debug!(%peer, "TLS handshake timed out");
return;
}
}
}
};
connection(stream, peer, resolver, bump, shutdown).await;
}
type Inbound = SplitStream<Framed<ServerStream, LengthDelimitedCodec>>;
async fn hello_phase(inbound: &mut Inbound, peer: SocketAddr) -> Option<(u64, Option<String>)> {
let item = match tokio::time::timeout(HELLO_TIMEOUT, inbound.next()).await {
Ok(Some(Ok(bytes))) => bytes,
Ok(_) => return None, Err(_) => {
tracing::debug!(%peer, "no hello within the timeout, closing connection");
return None;
}
};
let frame: ClientFrame = match crate::wire::decode_tagged(RecordKind::ClientFrame, &item) {
Ok(frame) => frame,
Err(err) => {
tracing::debug!(%peer, ?err, "undecodable first frame, closing connection");
return None;
}
};
match frame {
ClientFrame::Request {
id,
request: Request::Hello { token },
} => Some((id, token.map(|t| t.0))),
_ => {
tracing::debug!(%peer, "first frame is not hello, closing connection");
None
}
}
}
async fn closed(close: &mut Option<watch::Receiver<bool>>) {
if let Some(rx) = close {
if rx.wait_for(|closed| *closed).await.is_ok() {
return;
}
}
std::future::pending::<()>().await
}
async fn connection<R: Resolver>(
stream: ServerStream,
peer: SocketAddr,
resolver: Arc<R>,
bump: WriteSignal,
mut shutdown: watch::Receiver<bool>,
) {
let framed = Framed::new(stream, codec());
let (mut sink, mut inbound) = framed.split();
let (out_tx, mut out_rx) = mpsc::channel::<ServerFrame>(OUT_CAPACITY);
let inflight = Arc::new(tokio::sync::Semaphore::new(MAX_INFLIGHT_PER_CONNECTION));
let writer = tokio::spawn(async move {
'writer: while let Some(frame) = out_rx.recv().await {
let Ok(bytes) = encode_tagged(RecordKind::ServerFrame, &frame) else {
continue;
};
if sink.feed(Bytes::from(bytes)).await.is_err() {
break;
}
while let Ok(frame) = out_rx.try_recv() {
let Ok(bytes) = encode_tagged(RecordKind::ServerFrame, &frame) else {
continue;
};
if sink.feed(Bytes::from(bytes)).await.is_err() {
break 'writer;
}
}
if sink.flush().await.is_err() {
break;
}
}
});
let hello = tokio::select! {
hello = hello_phase(&mut inbound, peer) => hello,
_ = shutdown.changed() => None,
};
let Some((hello_id, token)) = hello else {
drop(out_tx);
let _ = writer.await;
return;
};
let session = match resolver.resolve(&Hello { token, peer }).await {
Ok(session) => session,
Err(rejection) => {
tracing::debug!(%peer, %rejection, "connection refused");
let _ = out_tx
.send(ServerFrame::Response {
id: hello_id,
response: Response::Hello(Err(rejection)),
stable_timestamp: None,
})
.await;
drop(out_tx);
let _ = writer.await;
return;
}
};
let Session {
executor,
tenant,
guard: _guard,
mut close,
} = session;
let hello_ok = HelloOk {
default_routing_key: executor.default_routing_key().map(str::to_owned),
};
let frame = ServerFrame::Response {
id: hello_id,
response: Response::Hello(Ok(hello_ok.clone())),
stable_timestamp: executor.stable_timestamp().await.ok().flatten(),
};
if out_tx.send(frame).await.is_err() {
let _ = writer.await;
return;
}
let bump_on_write = executor.write_watch().is_none();
let mut watches: HashMap<u64, AbortHandle> = HashMap::new();
let (woken_tx, mut woken_rx) = mpsc::channel::<u64>(WOKEN_CAPACITY);
let notifier = {
let out_tx = out_tx.clone();
let executor = executor.clone();
let mut shutdown = shutdown.clone();
tokio::spawn(async move {
loop {
tokio::select! {
woken = woken_rx.recv() => {
let Some(watch) = woken else { break };
let mut woken = vec![watch];
while let Ok(watch) = woken_rx.try_recv() {
if !woken.contains(&watch) {
woken.push(watch);
}
}
let stable_timestamp =
executor.stable_timestamp().await.ok().flatten();
let frame = ServerFrame::Notify {
watches: woken,
stable_timestamp,
};
if out_tx.send(frame).await.is_err() {
break;
}
}
_ = shutdown.changed() => break,
}
}
})
};
loop {
let item = tokio::select! {
item = inbound.next() => item,
_ = closed(&mut close) => {
tracing::debug!(%peer, "connection closed by the server");
break;
}
_ = shutdown.changed() => break,
};
let Some(Ok(bytes)) = item else {
break; };
let frame: ClientFrame = match crate::wire::decode_tagged(RecordKind::ClientFrame, &bytes) {
Ok(frame) => frame,
Err(err) => {
tracing::debug!(%peer, ?err, "undecodable frame, closing connection");
break;
}
};
let (id, request) = match frame {
ClientFrame::Request { id, request } => (id, request),
ClientFrame::Watch {
watch,
filters,
routing_key,
tenant: named,
} => {
let interest = WriteInterest {
filters: filters.map(Arc::from),
routing_key,
tenant: match &tenant {
Some(tenant) => TenantFilter::Only(tenant.to_string()),
None => named,
},
};
if let Some(replaced) = watches.remove(&watch) {
replaced.abort();
}
if watches.len() >= MAX_WATCHES_PER_CONNECTION {
tracing::warn!(
%peer,
max = MAX_WATCHES_PER_CONNECTION,
"too many watches on one connection: the watch is not registered \
and its subscription is left to its poll interval"
);
continue;
}
let mut rx = executor
.write_watch_for(&interest)
.unwrap_or_else(|| bump.subscribe_to(interest));
let woken_tx = woken_tx.clone();
let task = tokio::spawn(async move {
while woken_tx.send(watch).await.is_ok() {
if rx.changed().await.is_err() {
break;
}
rx.borrow_and_update();
}
});
watches.insert(watch, task.abort_handle());
continue;
}
ClientFrame::Unwatch { watch } => {
if let Some(dropped) = watches.remove(&watch) {
dropped.abort();
}
continue;
}
};
if let Request::Hello { .. } = request {
let frame = ServerFrame::Response {
id,
response: Response::Hello(Ok(hello_ok.clone())),
stable_timestamp: executor.stable_timestamp().await.ok().flatten(),
};
if out_tx.send(frame).await.is_err() {
break;
}
continue;
}
let Ok(permit) = Arc::clone(&inflight).acquire_owned().await else {
break;
};
let executor = executor.clone();
let tenant = tenant.clone();
let out_tx = out_tx.clone();
let bump = bump_on_write.then(|| bump.clone());
tokio::spawn(async move {
let _permit = permit;
let response = handle(&executor, tenant.as_deref(), request, bump.as_ref()).await;
let frame = ServerFrame::Response {
id,
response,
stable_timestamp: executor.stable_timestamp().await.ok().flatten(),
};
let _ = out_tx.send(frame).await;
});
}
notifier.abort();
for (_, watch) in watches {
watch.abort();
}
drop(out_tx);
let _ = writer.await;
}
fn err_string<T>(result: anyhow::Result<T>) -> Result<T, String> {
result.map_err(|e| format!("{e:#}"))
}
const REPLICATE_REFUSED: &str = "replicate is refused on a connection serving a tenant";
fn subscribers_refused<T>() -> Result<T, String> {
Err(
"subscriptions are not served on a connection serving a tenant: keep their \
cursors in a store of your own (a Sidecar)"
.to_owned(),
)
}
fn confine(request: &mut Request, tenant: &str) -> Option<Response> {
match request {
Request::Hello { .. } => None,
Request::Write { events } => {
for event in events {
event.tenant = tenant.to_owned();
}
None
}
Request::Replicate { .. } => Some(Response::Write(Err(WireWriteError::Other(
REPLICATE_REFUSED.to_owned(),
)))),
Request::Read { tenant: named, .. } | Request::LatestTimestamp { tenant: named, .. } => {
*named = TenantFilter::Only(tenant.to_owned());
None
}
Request::LatestVersion { tenant: named, .. }
| Request::StreamRoutingKey { tenant: named, .. }
| Request::GetSnapshot { tenant: named, .. }
| Request::SaveSnapshot { tenant: named, .. }
| Request::DeleteSnapshot { tenant: named, .. } => {
*named = tenant.to_owned();
None
}
Request::GetSubscriberCursor { .. } => {
Some(Response::SubscriberCursor(subscribers_refused()))
}
Request::IsSubscriberRunning { .. } => {
Some(Response::SubscriberRunning(subscribers_refused()))
}
Request::SubscriberStatus { .. } => Some(Response::SubscriberStatus(subscribers_refused())),
Request::UpsertSubscriber { .. } => Some(Response::Unit(subscribers_refused())),
Request::Acknowledge { .. } => Some(Response::Acknowledge(subscribers_refused())),
}
}
fn present(response: &mut Response) {
if let Response::Read(Ok(page)) = response {
for event in &mut page.events {
event.tenant.clear();
}
}
}
async fn handle<E: Executor>(
executor: &E,
tenant: Option<&str>,
mut request: Request,
bump: Option<&WriteSignal>,
) -> Response {
if let Some(tenant) = tenant {
if let Some(refusal) = confine(&mut request, tenant) {
return refusal;
}
}
let wake = match (&request, bump) {
(Request::Write { events } | Request::Replicate { events }, Some(bump)) => {
Some(bump.prepare(events))
}
_ => None,
};
let mut response = dispatch(executor, request).await;
if let (Some(wake), Response::Write(Ok(()))) = (wake, &response) {
wake.fire();
}
if tenant.is_some() {
present(&mut response);
}
response
}
async fn dispatch<E: Executor>(executor: &E, request: Request) -> Response {
match request {
Request::Hello { .. } => unreachable!("Hello is answered inline by the read loop"),
Request::Write { events } => Response::Write(
executor
.write(events)
.await
.map_err(|e| WireWriteError::from(&e)),
),
Request::Replicate { events } => Response::Write(
executor
.replicate(events)
.await
.map_err(|e| WireWriteError::from(&e)),
),
Request::Read {
aggregators,
routing_key,
tenant,
args,
to_micros,
} => {
let to_micros = match to_micros {
Some(bound) => match executor.stable_timestamp().await {
Ok(Some(own)) => Some(bound.min(own)),
_ => Some(bound),
},
None => None,
};
Response::Read(err_string(
executor
.read(
aggregators.map(Arc::from),
routing_key,
tenant,
args,
to_micros,
)
.await
.map(WireReadResult::from),
))
}
Request::LatestTimestamp {
aggregators,
routing_key,
tenant,
} => Response::LatestTimestamp(err_string(
executor
.latest_timestamp(aggregators.map(Arc::from), routing_key, tenant)
.await,
)),
Request::GetSubscriberCursor { key } => {
Response::SubscriberCursor(err_string(executor.get_subscriber_cursor(key).await))
}
Request::IsSubscriberRunning { key, worker_id } => Response::SubscriberRunning(err_string(
executor.is_subscriber_running(key, worker_id).await,
)),
Request::SubscriberStatus { key, worker_id } => {
Response::SubscriberStatus(err_string(executor.subscriber_status(key, worker_id).await))
}
Request::LatestVersion {
tenant,
aggregate_type,
aggregate_id,
} => Response::LatestVersion(err_string(
executor
.latest_version(tenant, aggregate_type, aggregate_id)
.await,
)),
Request::StreamRoutingKey {
tenant,
aggregate_type,
aggregate_id,
} => Response::StreamRoutingKey(err_string(
executor
.stream_routing_key(tenant, aggregate_type, aggregate_id)
.await,
)),
Request::UpsertSubscriber { key, worker_id } => {
Response::Unit(err_string(executor.upsert_subscriber(key, worker_id).await))
}
Request::Acknowledge {
key,
worker_id,
cursor,
lag,
} => Response::Acknowledge(err_string(
executor.acknowledge(key, worker_id, cursor, lag).await,
)),
Request::GetSnapshot {
tenant,
aggregate_type,
projection,
aggregate_revision,
id,
} => Response::Snapshot(err_string(
executor
.get_snapshot(tenant, aggregate_type, projection, aggregate_revision, id)
.await,
)),
Request::SaveSnapshot {
tenant,
aggregate_type,
projection,
aggregate_revision,
id,
data,
cursor,
} => Response::Unit(err_string(
executor
.save_snapshot(
tenant,
aggregate_type,
projection,
aggregate_revision,
id,
data,
cursor,
)
.await,
)),
Request::DeleteSnapshot {
tenant,
aggregate_type,
projection,
id,
} => Response::Unit(err_string(
executor
.delete_snapshot(tenant, aggregate_type, projection, id)
.await,
)),
}
}
#[cfg(test)]
mod tests {
use evento_core::cursor::{Args, Value};
use evento_core::{Event, TenantFilter};
use ulid::Ulid;
use super::{confine, present};
use crate::wire::{Request, Response, WireReadResult, WireWriteError};
fn acme() -> String {
"acme".to_owned()
}
#[test]
fn a_tenant_session_acts_in_its_tenant_whatever_a_request_names() {
let mut write = Request::Write {
events: vec![
Event::default(),
Event {
tenant: "other".to_owned(),
..Default::default()
},
],
};
assert!(confine(&mut write, "acme").is_none());
let Request::Write { events } = write else {
unreachable!()
};
assert!(events.iter().all(|e| e.tenant == "acme"));
for tenant in [TenantFilter::All, TenantFilter::Only("other".to_owned())] {
let mut read = Request::Read {
aggregators: None,
routing_key: None,
tenant: tenant.clone(),
args: Args::forward(1, None),
to_micros: None,
};
assert!(confine(&mut read, "acme").is_none());
assert!(
matches!(&read, Request::Read { tenant: TenantFilter::Only(t), .. } if *t == acme())
);
let mut latest = Request::LatestTimestamp {
aggregators: None,
routing_key: None,
tenant,
};
assert!(confine(&mut latest, "acme").is_none());
assert!(
matches!(&latest, Request::LatestTimestamp { tenant: TenantFilter::Only(t), .. } if *t == acme())
);
}
let other = || "other".to_owned();
let mut requests = [
Request::LatestVersion {
tenant: other(),
aggregate_type: other(),
aggregate_id: other(),
},
Request::StreamRoutingKey {
tenant: other(),
aggregate_type: other(),
aggregate_id: other(),
},
Request::GetSnapshot {
tenant: other(),
aggregate_type: other(),
projection: other(),
aggregate_revision: other(),
id: other(),
},
Request::SaveSnapshot {
tenant: other(),
aggregate_type: other(),
projection: other(),
aggregate_revision: other(),
id: other(),
data: vec![],
cursor: Value(String::new()),
},
Request::DeleteSnapshot {
tenant: other(),
aggregate_type: other(),
projection: other(),
id: other(),
},
];
for request in &mut requests {
assert!(confine(request, "acme").is_none());
let (Request::LatestVersion { tenant, .. }
| Request::StreamRoutingKey { tenant, .. }
| Request::GetSnapshot { tenant, .. }
| Request::SaveSnapshot { tenant, .. }
| Request::DeleteSnapshot { tenant, .. }) = &*request
else {
unreachable!()
};
assert_eq!(*tenant, acme(), "{request:?}");
}
}
#[test]
fn a_tenant_session_reads_its_events_in_the_default_tenant() {
let event = |tenant: &str| Event {
tenant: tenant.to_owned(),
..Default::default()
};
let mut response = Response::Read(Ok(WireReadResult {
cursors: vec![Value(String::new()), Value(String::new())],
events: vec![event("acme"), event("acme")],
page_info: Default::default(),
}));
present(&mut response);
let Response::Read(Ok(page)) = response else {
unreachable!()
};
assert!(page.events.iter().all(|e| e.tenant.is_empty()));
}
#[test]
fn a_tenant_session_refuses_replicate_and_the_subscriber_calls() {
let key = || "key".to_owned();
let worker_id = Ulid::nil();
let refusal = |mut request: Request| confine(&mut request, "acme");
assert!(matches!(
refusal(Request::Replicate {
events: vec![Event::default()]
}),
Some(Response::Write(Err(WireWriteError::Other(_))))
));
assert!(matches!(
refusal(Request::GetSubscriberCursor { key: key() }),
Some(Response::SubscriberCursor(Err(_)))
));
assert!(matches!(
refusal(Request::IsSubscriberRunning {
key: key(),
worker_id
}),
Some(Response::SubscriberRunning(Err(_)))
));
assert!(matches!(
refusal(Request::SubscriberStatus {
key: key(),
worker_id
}),
Some(Response::SubscriberStatus(Err(_)))
));
assert!(matches!(
refusal(Request::UpsertSubscriber {
key: key(),
worker_id
}),
Some(Response::Unit(Err(_)))
));
assert!(matches!(
refusal(Request::Acknowledge {
key: key(),
worker_id,
cursor: Value(String::new()),
lag: 0
}),
Some(Response::Acknowledge(Err(_)))
));
}
}
#[cfg(test)]
mod watch_tests {
use super::*;
use evento_core::{Event, EventFilter, RoutingKey};
type Wire = Framed<TcpStream, LengthDelimitedCodec>;
async fn send(wire: &mut Wire, frame: ClientFrame) {
let bytes = encode_tagged(RecordKind::ClientFrame, &frame).unwrap();
wire.send(Bytes::from(bytes)).await.unwrap();
}
async fn next(wire: &mut Wire) -> ServerFrame {
let bytes = tokio::time::timeout(Duration::from_secs(5), wire.next())
.await
.expect("a frame within 5s")
.expect("the connection is open")
.unwrap();
crate::wire::decode_tagged(RecordKind::ServerFrame, &bytes).unwrap()
}
async fn notified(wire: &mut Wire) -> Vec<u64> {
loop {
if let ServerFrame::Notify { mut watches, .. } = next(wire).await {
watches.sort_unstable();
return watches;
}
}
}
async fn settle(wire: &mut Wire, id: u64) {
send(
wire,
ClientFrame::Request {
id,
request: Request::Hello { token: None },
},
)
.await;
loop {
if let ServerFrame::Response { id: answered, .. } = next(wire).await {
if answered == id {
return;
}
}
}
}
fn watch(watch: u64, aggregate_type: &str) -> ClientFrame {
ClientFrame::Watch {
watch,
filters: Some(vec![EventFilter::by_type_raw(aggregate_type)]),
routing_key: Some(RoutingKey::Value(None)),
tenant: TenantFilter::All,
}
}
fn event(aggregate_type: &str, id: &str) -> Event {
Event {
id: ulid::Ulid::generate(),
aggregate_type: aggregate_type.to_owned(),
aggregate_id: id.to_owned(),
version: 1,
name: "Probed".to_owned(),
..Default::default()
}
}
#[tokio::test]
async fn a_notify_names_the_watches_a_write_wakes() {
let store = evento_fjall::Fjall::temporary().unwrap();
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let _server = serve(listener, store.clone());
let mut wire = Framed::new(TcpStream::connect(addr).await.unwrap(), codec());
settle(&mut wire, 0).await;
store
.write(vec![event("test/Account", "a0")])
.await
.unwrap();
let quiet = tokio::time::timeout(Duration::from_millis(300), wire.next()).await;
assert!(
quiet.is_err(),
"a connection that watches nothing is sent nothing"
);
send(&mut wire, watch(1, "test/Account")).await;
assert_eq!(notified(&mut wire).await, [1]);
send(&mut wire, watch(2, "test/Account")).await;
assert_eq!(notified(&mut wire).await, [2]);
send(&mut wire, watch(3, "test/Card")).await;
assert_eq!(notified(&mut wire).await, [3]);
store
.write(vec![event("test/Account", "a1")])
.await
.unwrap();
let mut woken = notified(&mut wire).await;
if woken.len() < 2 {
woken.extend(notified(&mut wire).await);
woken.sort_unstable();
}
assert_eq!(woken, [1, 2]);
send(&mut wire, ClientFrame::Unwatch { watch: 2 }).await;
settle(&mut wire, 1).await;
store
.write(vec![event("test/Account", "a2")])
.await
.unwrap();
assert_eq!(notified(&mut wire).await, [1]);
store.write(vec![event("test/Card", "c1")]).await.unwrap();
assert_eq!(notified(&mut wire).await, [3]);
}
}