use std::collections::HashMap;
use std::future::Future;
use std::pin::Pin;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll, Waker};
use std::thread::JoinHandle;
use iceoryx2::prelude::*;
use super::bridge::{emit_rpc_error, OwnedEmitter, ResponseEmitter, ServiceDispatcher};
use super::notify::Coalescer;
use super::open::{open_event_service, open_service, OpenMode};
use super::publish::publish_until_delivered;
use super::{
shared_node, transport_error, IoxEvent, IoxListener, IoxNotifier, IoxPubSub, IoxPublisher,
IoxSubscriber, CONSUMER_WAIT_TIMEOUT, MAX_LOANED_SAMPLES, MAX_SLICE_LEN, OPEN_RETRY_ATTEMPTS,
OPEN_RETRY_SLEEP, REQUEST_NOTIFY_SUFFIX, REQUEST_SUFFIX, RESPONSE_NOTIFY_SUFFIX,
RESPONSE_SUFFIX,
};
use crate::global::Locked;
use crate::sync::lock;
use crate::types::{
fmt_correlation_id, install_call_cancellation, BoxResponseFuture, EventKind, RpcError,
RpcHeader, CORRELATION_ID_LEN, PROTOCOL_VERSION,
};
use crate::CancellationToken;
pub(super) struct ChannelPorts {
_request_service: IoxPubSub,
_response_service: IoxPubSub,
_request_notify: IoxEvent,
_response_notify: IoxEvent,
pub(super) subscriber: IoxSubscriber,
pub(super) listener: IoxListener,
pub(super) publisher: IoxPublisher,
pub(super) response_notifier: IoxNotifier,
}
pub(super) fn open_channel_ports(channel: &str) -> Result<ChannelPorts, RpcError> {
let node = shared_node()?;
let request_service = open_service(&node, channel, REQUEST_SUFFIX, OpenMode::CreateOrOpen)?;
let response_service = open_service(&node, channel, RESPONSE_SUFFIX, OpenMode::CreateOrOpen)?;
let request_notify = open_event_service(
&node,
channel,
REQUEST_NOTIFY_SUFFIX,
OpenMode::CreateOrOpen,
)?;
let response_notify = open_event_service(
&node,
channel,
RESPONSE_NOTIFY_SUFFIX,
OpenMode::CreateOrOpen,
)?;
let subscriber = request_service
.subscriber_builder()
.create()
.map_err(|e| transport_error("request subscriber", e))?;
let listener = request_notify
.listener_builder()
.create()
.map_err(|e| transport_error("request listener", e))?;
let publisher = response_service
.publisher_builder()
.initial_max_slice_len(MAX_SLICE_LEN)
.max_loaned_samples(MAX_LOANED_SAMPLES)
.create()
.map_err(|e| transport_error("response publisher", e))?;
let response_notifier = response_notify
.notifier_builder()
.create()
.map_err(|e| transport_error("response notifier", e))?;
Ok(ChannelPorts {
_request_service: request_service,
_response_service: response_service,
_request_notify: request_notify,
_response_notify: response_notify,
subscriber,
listener,
publisher,
response_notifier,
})
}
pub fn spawn_native_service(
channel: &str,
services: Vec<ServiceDispatcher>,
stop: CancellationToken,
) -> JoinHandle<()> {
let table: HashMap<u32, ServiceDispatcher> = services
.into_iter()
.map(|dispatcher| (dispatcher.service().id, dispatcher))
.collect();
let channel = channel.to_owned();
let spawner = crate::rt::Spawner::capture();
let handle = std::thread::spawn(move || {
let ports = match open_with_retries(&channel, &stop) {
Ok(ports) => ports,
Err(e) => {
if stop.is_cancelled() {
log::debug!("[transport] '{channel}': open abandoned on shutdown: {e}");
} else {
log::error!("[transport] '{channel}': {e}");
}
return;
}
};
log::info!(
"[transport] channel '{channel}' ready ({} service(s))",
table.len()
);
let hub = Arc::new(ResponseHub::new(
channel.clone(),
ports.publisher,
ports._response_service,
ports.response_notifier,
));
super::pump::run_receive_loop(
&channel,
"request",
&ports.subscriber,
&ports.listener,
|| stop.is_cancelled(),
|header, payload| handle_request(&channel, &table, &hub, &spawner, header, payload),
);
log::info!("[transport] channel '{channel}' stopped");
});
handle
}
fn open_with_retries(channel: &str, stop: &CancellationToken) -> Result<ChannelPorts, RpcError> {
retry_open(channel, OPEN_RETRY_ATTEMPTS, stop, || {
open_channel_ports(channel)
})
}
fn retry_open<T>(
channel: &str,
attempts: u32,
stop: &CancellationToken,
mut attempt_open: impl FnMut() -> Result<T, RpcError>,
) -> Result<T, RpcError> {
let mut attempt = 0;
loop {
match attempt_open() {
Ok(opened) => return Ok(opened),
Err(e) if !e.is_retryable() || attempt >= attempts || stop.is_cancelled() => {
return Err(e)
}
Err(e) => {
attempt += 1;
log::warn!(
"[transport] '{channel}': {e} — attempt {attempt}/{attempts}, retrying in {:?}",
OPEN_RETRY_SLEEP
);
std::thread::sleep(OPEN_RETRY_SLEEP);
}
}
}
}
struct ResponseHub {
channel: String,
publisher: IoxPublisher,
service: IoxPubSub,
notifier: IoxNotifier,
seq: AtomicU64,
coalescer: Coalescer,
in_flight: Arc<InFlightCalls>,
}
#[derive(Default)]
struct InFlightCalls {
calls: Mutex<HashMap<[u8; CORRELATION_ID_LEN], CancellationToken>>,
}
impl InFlightCalls {
fn new() -> Self {
Self::default()
}
fn register(&self, cid: [u8; CORRELATION_ID_LEN]) -> CancellationToken {
let token = CancellationToken::new();
lock(&self.calls).insert(cid, token.clone());
token
}
fn cancel(&self, cid: &[u8; CORRELATION_ID_LEN]) -> bool {
match lock(&self.calls).get(cid) {
Some(token) => {
token.cancel();
true
}
None => false,
}
}
fn unregister(&self, cid: &[u8; CORRELATION_ID_LEN]) {
lock(&self.calls).remove(cid);
}
#[cfg(test)]
fn len(&self) -> usize {
lock(&self.calls).len()
}
}
impl ResponseHub {
fn new(
channel: String,
publisher: IoxPublisher,
service: IoxPubSub,
notifier: IoxNotifier,
) -> Self {
Self {
channel,
publisher,
service,
notifier,
seq: AtomicU64::new(0),
coalescer: Coalescer::new(),
in_flight: Arc::new(InFlightCalls::new()),
}
}
fn emitter(self: &Arc<Self>, request: &RpcHeader) -> CallEmitter {
CallEmitter {
hub: Arc::clone(self),
request: *request,
}
}
fn register(&self, cid: [u8; CORRELATION_ID_LEN]) -> CancellationToken {
self.in_flight.register(cid)
}
fn cancel(&self, cid: &[u8; CORRELATION_ID_LEN]) -> bool {
self.in_flight.cancel(cid)
}
fn unregister(&self, cid: &[u8; CORRELATION_ID_LEN]) {
self.in_flight.unregister(cid);
}
fn cancellable(
self: &Arc<Self>,
cid: [u8; CORRELATION_ID_LEN],
token: CancellationToken,
task: BoxResponseFuture,
) -> BoxResponseFuture {
CancellableCall::wrap(Arc::clone(&self.in_flight), cid, token, task)
}
fn publish(&self, kind: EventKind, request: &RpcHeader, payload: &[u8]) -> bool {
let header = RpcHeader::response_from(request, kind, request.service_version)
.with_seq(self.seq.fetch_add(1, Ordering::Relaxed));
match publish_until_delivered(
&self.publisher,
&self.service,
header,
payload,
CONSUMER_WAIT_TIMEOUT,
) {
Ok(()) => {
if self.coalescer.should_notify() {
let _ = self.notifier.notify_with_custom_event_id(EventId::new(0));
}
true
}
Err(e) => {
log::warn!(
"[transport] '{}': response publication failed: {e}",
self.channel
);
false
}
}
}
}
struct CallEmitter {
hub: Arc<ResponseHub>,
request: RpcHeader,
}
impl ResponseEmitter for CallEmitter {
fn emit(&mut self, kind: EventKind, payload: &[u8]) -> bool {
self.hub.publish(kind, &self.request, payload)
}
}
struct InFlightGuard {
registry: Arc<InFlightCalls>,
cid: [u8; CORRELATION_ID_LEN],
}
impl Drop for InFlightGuard {
fn drop(&mut self) {
self.registry.unregister(&self.cid);
}
}
struct CancellableCall {
_guard: InFlightGuard,
token: CancellationToken,
registered_waker: Option<Waker>,
task: BoxResponseFuture,
}
impl CancellableCall {
fn wrap(
registry: Arc<InFlightCalls>,
cid: [u8; CORRELATION_ID_LEN],
token: CancellationToken,
task: BoxResponseFuture,
) -> BoxResponseFuture {
Box::pin(Self {
_guard: InFlightGuard { registry, cid },
token,
registered_waker: None,
task,
})
}
}
impl Future for CancellableCall {
type Output = ();
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {
let this = self.get_mut();
if this.token.is_cancelled() {
return Poll::Ready(());
}
let _ambient = install_call_cancellation(this.token.clone());
if this.task.as_mut().poll(cx).is_ready() {
return Poll::Ready(());
}
if this
.token
.poll_cancelled_cached(cx, &mut this.registered_waker)
.is_ready()
{
return Poll::Ready(());
}
Poll::Pending
}
}
#[derive(Debug, PartialEq, Eq)]
enum RequestRejection {
Protocol { actual: u16 },
ServiceVersion { expected: u16, actual: u16 },
}
impl RequestRejection {
fn classify(dispatcher: &ServiceDispatcher, header: &RpcHeader) -> Option<Self> {
if header.protocol_version != PROTOCOL_VERSION {
return Some(Self::Protocol {
actual: header.protocol_version,
});
}
let expected = dispatcher.service().version;
if header.service_version != expected {
return Some(Self::ServiceVersion {
expected,
actual: header.service_version,
});
}
None
}
fn rpc_error(&self) -> RpcError {
match *self {
Self::Protocol { actual } => RpcError::ProtocolMismatch(format!(
"protocol_version {actual} != {PROTOCOL_VERSION}"
)),
Self::ServiceVersion { expected, actual } => {
RpcError::IncompatibleVersion { expected, actual }
}
}
}
}
fn handle_request(
channel: &str,
table: &HashMap<u32, ServiceDispatcher>,
hub: &Arc<ResponseHub>,
spawner: &crate::rt::Spawner,
header: &RpcHeader,
payload: &[u8],
) {
match header.event_kind() {
EventKind::Request => {}
EventKind::Cancel => {
let in_flight = hub.cancel(&header.correlation_id);
log::debug!(
"[transport] '{channel}': cancel for {} ({})",
fmt_correlation_id(&header.correlation_id),
if in_flight {
"call in flight"
} else {
"unknown call"
}
);
return;
}
other => {
log::warn!("[transport] '{channel}': unexpected {other:?} sample");
return;
}
}
let Some(dispatcher) = table.get(&header.service_id) else {
reject(
channel,
header,
hub,
RpcError::UnknownService(format!("{:#010x}", header.service_id)),
);
return;
};
if let Some(rejection) = RequestRejection::classify(dispatcher, header) {
reject(channel, header, hub, rejection.rpc_error());
return;
}
let cid = header.correlation_id;
let token = hub.register(cid);
let emitter: OwnedEmitter = Box::new(hub.emitter(header));
match dispatcher.dispatch(header.method(), *header, payload.to_vec(), emitter) {
Some(task) => spawner.run_or_spawn(hub.cancellable(cid, token, task)),
None => {
hub.unregister(&cid);
reject(
channel,
header,
hub,
RpcError::UnknownMethod(header.method().to_owned()),
);
}
}
}
fn reject(channel: &str, header: &RpcHeader, hub: &Arc<ResponseHub>, err: RpcError) {
log::warn!(
"[transport] '{channel}': rejecting '{}': {err}",
header.method()
);
let mut emitter = hub.emitter(header);
if !emit_rpc_error(err, &mut emitter) {
log::error!("[transport] '{channel}': the rejection could not be published");
}
}
struct RegisteredChannelService {
name: &'static str,
dispatcher: ServiceDispatcher,
}
struct PendingChannel {
services: Vec<RegisteredChannelService>,
stop: CancellationToken,
}
fn channel_registry() -> &'static Locked<HashMap<String, PendingChannel>> {
static REGISTRY: Locked<HashMap<String, PendingChannel>> = Locked::new();
®ISTRY
}
static CHANNELS_SEALED: AtomicBool = AtomicBool::new(false);
fn conflicting_name(entries: &[RegisteredChannelService], id: u32) -> Option<&'static str> {
entries
.iter()
.find(|svc| svc.dispatcher.service().id == id)
.map(|svc| svc.name)
}
pub fn register_native_service(
channel: &str,
service_name: &'static str,
dispatcher: ServiceDispatcher,
) -> Result<(), RpcError> {
let service_id = dispatcher.service().id;
if CHANNELS_SEALED.load(Ordering::Acquire) {
let handle = spawn_native_service(
channel,
vec![dispatcher],
crate::global_cancel_token().clone(),
);
crate::locator::ServiceLocator::global().register_shutdown_thread(handle);
return Ok(());
}
let registered = channel_registry().with(|registry| {
let pending = registry
.entry(channel.to_owned())
.or_insert_with(|| PendingChannel {
services: Vec::new(),
stop: crate::global_cancel_token().clone(),
});
if let Some(previous) = conflicting_name(&pending.services, service_id) {
return Err(format!(
"service_id collision on channel '{channel}': '{service_name}' and \
'{previous}' both map to {service_id:#010x}"
));
}
pending.services.push(RegisteredChannelService {
name: service_name,
dispatcher,
});
Ok(())
});
registered.map_err(RpcError::TransportError)?;
log::debug!("[transport] '{service_name}' registered on channel '{channel}'");
Ok(())
}
pub fn start_registered_channels() {
if CHANNELS_SEALED.swap(true, Ordering::AcqRel) {
return;
}
let channels: Vec<(String, PendingChannel)> =
channel_registry().with(|registry| registry.drain().collect());
for (channel, pending) in channels {
let services = pending
.services
.into_iter()
.map(|svc| svc.dispatcher)
.collect();
let handle = spawn_native_service(&channel, services, pending.stop);
crate::locator::ServiceLocator::global().register_shutdown_thread(handle);
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::transport::CollectEmitter;
use crate::types::{next_correlation_id, ServiceRef};
use std::sync::Mutex;
#[test]
fn a_duplicate_service_id_is_reported_with_the_previous_name() {
let entries = vec![
RegisteredChannelService {
name: "GetPerson",
dispatcher: ServiceDispatcher::new(ServiceRef::new(7, 1)),
},
RegisteredChannelService {
name: "SetPerson",
dispatcher: ServiceDispatcher::new(ServiceRef::new(9, 1)),
},
];
assert_eq!(conflicting_name(&entries, 9), Some("SetPerson"));
assert_eq!(conflicting_name(&entries, 11), None);
}
fn running() -> CancellationToken {
CancellationToken::new()
}
#[test]
fn a_retryable_open_failure_is_retried_until_it_succeeds() {
let mut attempts = 0;
let opened = retry_open("chan", 5, &running(), || {
attempts += 1;
if attempts < 3 {
Err(RpcError::TransportError("SystemInFlux".to_string()))
} else {
Ok(attempts)
}
});
assert_eq!(opened.unwrap(), 3);
assert_eq!(attempts, 3, "two failures then one success");
}
#[test]
fn a_stale_open_failure_is_not_retried() {
let mut attempts = 0;
let opened = retry_open::<()>("chan", 5, &running(), || {
attempts += 1;
Err(RpcError::ProtocolMismatch("stale service".to_string()))
});
assert!(opened.is_err());
assert_eq!(attempts, 1, "not one wasted attempt on a stale service");
}
#[test]
fn the_retry_budget_is_finite() {
let mut attempts = 0;
let opened = retry_open::<()>("chan", 2, &running(), || {
attempts += 1;
Err(RpcError::TransportError("SystemInFlux".to_string()))
});
assert!(matches!(opened, Err(RpcError::TransportError(_))));
assert_eq!(attempts, 3, "the first attempt plus the two retries");
}
#[test]
fn a_cancelled_shutdown_stops_the_retries() {
let stop = CancellationToken::new();
stop.cancel();
let mut attempts = 0;
let opened = retry_open::<()>("chan", 5, &stop, || {
attempts += 1;
Err(RpcError::TransportError("SystemInFlux".to_string()))
});
assert!(opened.is_err());
assert_eq!(attempts, 1, "a cancelled token stops before any retry");
}
#[derive(Clone, Default)]
struct SharedCollector(Arc<Mutex<CollectEmitter>>);
impl SharedCollector {
fn take(&self) -> Vec<(EventKind, Vec<u8>)> {
self.0.lock().unwrap_or_else(|e| e.into_inner()).take()
}
}
impl ResponseEmitter for SharedCollector {
fn emit(&mut self, kind: EventKind, payload: &[u8]) -> bool {
self.0
.lock()
.unwrap_or_else(|e| e.into_inner())
.emit(kind, payload)
}
}
#[test]
fn a_channel_table_routes_by_service_id() {
let mut first = ServiceDispatcher::new(ServiceRef::new(7, 1));
first.method("echo", |_header, payload, mut emitter| {
Box::pin(async move {
emitter.emit(EventKind::Next, &payload);
})
});
let mut second = ServiceDispatcher::new(ServiceRef::new(9, 1));
second.method("ping", |_header, _payload, _emitter| Box::pin(async {}));
let table: HashMap<u32, ServiceDispatcher> =
vec![(7, first), (9, second)].into_iter().collect();
let header = RpcHeader::request("echo", 7, 1);
let sink = SharedCollector::default();
let task = table
.get(&7)
.unwrap()
.dispatch("echo", header, b"x".to_vec(), Box::new(sink.clone()))
.expect("the first dispatcher has `echo`");
crate::rt::block_on(task);
assert_eq!(sink.take().len(), 1);
assert!(table
.get(&9)
.unwrap()
.dispatch("echo", header, b"x".to_vec(), Box::new(sink.clone()))
.is_none());
assert!(sink.take().is_empty());
assert!(!table.contains_key(&11));
}
#[test]
fn a_rejection_is_framed_as_a_bare_rpc_error() {
let mut emitter = CollectEmitter::new();
assert!(crate::transport::emit_rpc_error(
RpcError::UnknownMethod("nope".to_owned()),
&mut emitter
));
let samples = emitter.take();
assert_eq!(
samples.len(),
1,
"the rejection is a single terminal sample"
);
assert_eq!(samples[0].0, EventKind::RpcError);
assert!(samples[0].0.is_terminal());
let decoded: RpcError =
crate::transport::decode_aligned(&samples[0].1).expect("decode the error sample");
assert_eq!(decoded, RpcError::UnknownMethod("nope".to_owned()));
}
#[test]
fn dispatch_builds_a_task_only_for_a_known_method() {
let mut dispatcher = ServiceDispatcher::new(ServiceRef::new(7, 1));
dispatcher.method("echo", |_header, _payload, _emitter| Box::pin(async {}));
let header = RpcHeader::request("echo", 7, 1);
assert!(dispatcher
.dispatch(
"echo",
header,
b"x".to_vec(),
Box::new(CollectEmitter::new())
)
.is_some());
assert!(dispatcher
.dispatch(
"missing",
header,
b"x".to_vec(),
Box::new(CollectEmitter::new())
)
.is_none());
}
#[test]
fn an_incompatible_protocol_is_rejected_before_dispatch() {
let dispatcher = ServiceDispatcher::new(ServiceRef::new(7, 1));
let mut header = RpcHeader::request("echo", 7, 1);
header.protocol_version = PROTOCOL_VERSION.wrapping_add(1);
let rejection = RequestRejection::classify(&dispatcher, &header).expect("rejected");
assert_eq!(
rejection,
RequestRejection::Protocol {
actual: PROTOCOL_VERSION.wrapping_add(1)
}
);
assert!(matches!(
rejection.rpc_error(),
RpcError::ProtocolMismatch(_)
));
}
#[test]
fn an_incompatible_service_version_is_classified() {
let dispatcher = ServiceDispatcher::new(ServiceRef::new(7, 2));
let header = RpcHeader::request("echo", 7, 1);
let rejection = RequestRejection::classify(&dispatcher, &header).expect("rejected");
assert_eq!(
rejection,
RequestRejection::ServiceVersion {
expected: 2,
actual: 1
}
);
assert_eq!(
rejection.rpc_error(),
RpcError::IncompatibleVersion {
expected: 2,
actual: 1
}
);
}
#[test]
fn a_matching_header_is_never_rejected() {
let dispatcher = ServiceDispatcher::new(ServiceRef::new(7, 2));
let header = RpcHeader::request("echo", 7, 2);
assert_eq!(RequestRejection::classify(&dispatcher, &header), None);
}
struct Controlled {
polled: Arc<AtomicBool>,
dropped: Arc<AtomicBool>,
}
impl Controlled {
fn new() -> (Self, Arc<AtomicBool>, Arc<AtomicBool>) {
let polled = Arc::new(AtomicBool::new(false));
let dropped = Arc::new(AtomicBool::new(false));
(
Self {
polled: Arc::clone(&polled),
dropped: Arc::clone(&dropped),
},
polled,
dropped,
)
}
}
impl Future for Controlled {
type Output = ();
fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<()> {
self.get_mut().polled.store(true, Ordering::SeqCst);
Poll::Pending
}
}
impl Drop for Controlled {
fn drop(&mut self) {
self.dropped.store(true, Ordering::SeqCst);
}
}
fn poll_once(task: &mut BoxResponseFuture) -> Poll<()> {
let mut cx = Context::from_waker(std::task::Waker::noop());
task.as_mut().poll(&mut cx)
}
#[test]
fn cancelling_a_call_drops_its_task_and_releases_its_entry() {
let registry = Arc::new(InFlightCalls::new());
let cid = next_correlation_id();
let token = registry.register(cid);
let (task, polled, dropped) = Controlled::new();
let mut wrapped = CancellableCall::wrap(Arc::clone(®istry), cid, token, Box::pin(task));
assert!(
poll_once(&mut wrapped).is_pending(),
"the handler is awaiting something"
);
assert!(polled.load(Ordering::SeqCst), "the handler ran");
assert!(!dropped.load(Ordering::SeqCst));
assert_eq!(registry.len(), 1, "the call is in flight");
assert!(registry.cancel(&cid));
assert!(poll_once(&mut wrapped).is_ready(), "the call is ended");
drop(wrapped);
assert!(dropped.load(Ordering::SeqCst), "the handler was dropped");
assert_eq!(registry.len(), 0, "the entry was released");
assert!(
!registry.cancel(&cid),
"a call ended by a Cancel is no longer in flight"
);
}
#[test]
fn an_already_cancelled_call_never_reaches_its_handler() {
let registry = Arc::new(InFlightCalls::new());
let cid = next_correlation_id();
let token = registry.register(cid);
token.cancel();
let (task, polled, _dropped) = Controlled::new();
let mut wrapped = CancellableCall::wrap(Arc::clone(®istry), cid, token, Box::pin(task));
assert!(poll_once(&mut wrapped).is_ready());
assert!(
!polled.load(Ordering::SeqCst),
"a call cancelled before its first poll never runs"
);
}
#[test]
fn the_wrapped_handler_reads_the_call_token_ambiently() {
let registry = Arc::new(InFlightCalls::new());
let cid = next_correlation_id();
let token = registry.register(cid);
let task: BoxResponseFuture = Box::pin(async move {
let ambient =
crate::CallContext::cancellation().expect("the wrapper installs the call's token");
assert!(!ambient.is_cancelled(), "a fresh call is not cancelled");
ambient.cancel();
});
let mut wrapped = CancellableCall::wrap(Arc::clone(®istry), cid, token.clone(), task);
assert!(
poll_once(&mut wrapped).is_ready(),
"the handler answers inline"
);
assert!(token.is_cancelled(), "the ambient handle is the call token");
assert!(
crate::CallContext::cancellation().is_none(),
"the token is ambient for the poll only"
);
drop(wrapped);
assert_eq!(registry.len(), 0, "an answered call releases its entry");
}
#[test]
fn cancelling_an_unknown_call_does_nothing() {
let registry = InFlightCalls::new();
let cid = next_correlation_id();
let token = registry.register(cid);
assert!(!registry.cancel(&next_correlation_id()), "unknown id");
assert!(!token.is_cancelled(), "an unknown id fires no token");
assert_eq!(registry.len(), 1, "the in-flight call is untouched");
assert!(registry.cancel(&cid));
assert!(registry.cancel(&cid), "cancelling twice is fine");
registry.unregister(&cid);
assert!(!registry.cancel(&cid), "a call that ended is gone for good");
}
}